Merge cockroachdb#124889

124889: roachtest: add side-eye Start/Snapshot helpers r=dt a=dt

This is small subset of the new functionality in cockroachdb#124780, consisting of just the new SDK and helpers to install the agent using the cluster name as the env name and request a snapshot. 

Release note: none.
Epic: none.

Co-authored-by: David Taylor <[email protected]>

DEPS.bzl

@@ -2238,6 +2238,16 @@ def go_deps():
             "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/zstd/com_github_datadog_zstd-v1.5.6-0.20230824185856-869dae002e5e.zip",
         ],
     )
+    go_repository(
+        name = "com_github_dataexmachina_dev_side_eye_go",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/DataExMachina-dev/side-eye-go",
+        sha256 = "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f",
+        strip_prefix = "github.com/DataExMachina-dev/[email protected]",
+        urls = [
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip",
+        ],
+    )
     go_repository(
         name = "com_github_dave_dst",
         build_file_proto_mode = "disable_global",

build/bazelutil/distdir_files.bzl

@@ -169,6 +169,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DATA-DOG/go-sqlmock/com_github_data_dog_go_sqlmock-v1.5.0.zip": "25720bfcbd739305238408ab54263224b69ff6934923dfd9caed76d3871d0151",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/datadog-go/com_github_datadog_datadog_go-v3.2.0+incompatible.zip": "ede4a024d3c106b2f57ca04d7bfc7610e0c83f4d8a3bace2cf87b42fd5cf66cd",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/zstd/com_github_datadog_zstd-v1.5.6-0.20230824185856-869dae002e5e.zip": "e4924158bd1abf765a016d2c728fc367b32d20b86a268ef25743ba404c55e097",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip": "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GeertJohan/go.incremental/com_github_geertjohan_go_incremental-v1.0.0.zip": "ce46b3b717f8d2927046bcfb99c6f490b1b547a681e6b23240ac2c2292a891e8",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GeertJohan/go.rice/com_github_geertjohan_go_rice-v1.0.0.zip": "2fc48b9422bf356c18ed3fe32ec52f6a8b87ac168f83d2eed249afaebcc3eeb8",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GoogleCloudPlatform/cloudsql-proxy/com_github_googlecloudplatform_cloudsql_proxy-v0.0.0-20190129172621-c8b1d7a94ddf.zip": "d18ff41309efc943c71d5c8faa5b1dd792700a79fa4f61508c5e50f17fc9ca6f",

go.mod

@@ -94,6 +94,7 @@ require (
 	github.com/Azure/go-autorest/autorest/adal v0.9.15
 	github.com/BurntSushi/toml v1.2.1
 	github.com/DataDog/datadog-go v3.2.0+incompatible
+	github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d
 	github.com/IBM/sarama v1.42.1
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718

go.sum

@@ -184,6 +184,8 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf
 github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE=
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d h1:0NRhNOBaRnkXED8mftbOSCNGAf8MZhv4zu840hIUpIc=
+github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d/go.mod h1:FukCpc3od3BzYgxUtTWm3iB4ALtc4UknLNMQ0rq+V3A=
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190129172621-c8b1d7a94ddf/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo=

pkg/cmd/roachprod/flags.go

@@ -346,6 +346,8 @@ func initFlags() {
 	jaegerStartCmd.Flags().StringVar(&jaegerConfigNodes, "configure-nodes", "",
 		"the nodes on which to set the relevant CRDB cluster settings")
 
+	sideEyeRootCmd.AddCommand(sideEyeSnapCmd)
+
 	initCmd.Flags().IntVar(&startOpts.InitTarget,
 		"init-target", startOpts.InitTarget, "node on which to run initialization")
 

pkg/cmd/roachprod/main.go

@@ -1629,6 +1629,29 @@ var storageSnapshotCmd = &cobra.Command{
 	}),
 }
 
+var sideEyeRootCmd = &cobra.Command{
+	Use:   "side-eye",
+	Short: "interact with side-eye.io functionality",
+	Long: `Interact with side-eye.io functionality
+Side-Eye (app.side-eye.io) is a distributed debugger that can be used to capture
+snapshots of a CockroachDB cluster.
+`,
+	Args: cobra.MinimumNArgs(1),
+}
+var sideEyeSnapCmd = &cobra.Command{
+	Use:     "snapshot <cluster/Side-Eye environment>",
+	Aliases: []string{"snap"},
+	Short:   "capture a cluster snapshot",
+	Long: `Capture a cluster snapshot using Side-Eye
+The command will print an app.side-eye.io URL where the snapshot can be viewed.
+`,
+	Args: cobra.ExactArgs(1),
+	Run: wrap(func(cmd *cobra.Command, args []string) error {
+		roachprod.CaptureSideEyeSnapshot(context.Background(), config.Logger, args[0])
+		return nil
+	}),
+}
+
 // Before executing any command, validate and canonicalize args.
 func validateAndConfigure(cmd *cobra.Command, args []string) {
 	// Skip validation for commands that are self-sufficient.
@@ -1940,6 +1963,7 @@ func main() {
 		jaegerStartCmd,
 		jaegerStopCmd,
 		jaegerURLCmd,
+		sideEyeRootCmd,
 		fluentBitStartCmd,
 		fluentBitStopCmd,
 		opentelemetryStartCmd,

pkg/roachprod/BUILD.bazel

@@ -35,6 +35,7 @@ go_library(
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_errors//oserror",
+        "@com_github_dataexmachina_dev_side_eye_go//sideeyeclient",
         "@org_golang_x_sys//unix",
     ],
 )

pkg/roachprod/install/install.go

@@ -116,10 +116,12 @@ rm /tmp/otelcol-contrib.deb;
 // a command locally.
 var installLocalCmds = map[string]map[string]*exec.Cmd{
 	"side-eye": {
-		"%API_KEY%": exec.Command("gcloud", "secrets", "versions", "access", "latest", "--secret", "side-eye-key"),
+		"%API_KEY%": sideEyeSecretCmd,
 	},
 }
 
+var sideEyeSecretCmd = exec.Command("gcloud", "secrets", "versions", "access", "latest", "--secret", "side-eye-key")
+
 // SortedCmds TODO(peter): document
 func SortedCmds() []string {
 	cmds := make([]string, 0, len(installCmds))
@@ -174,3 +176,13 @@ func InstallTool(
 
 	return nil
 }
+
+func GetGcloudSideEyeSecret() string {
+	c := *sideEyeSecretCmd
+	c.Stderr = os.Stderr
+	out, err := c.Output()
+	if err != nil {
+		return ""
+	}
+	return string(out)
+}

pkg/roachprod/roachprod.go

@@ -32,6 +32,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/DataExMachina-dev/side-eye-go/sideeyeclient"
 	"github.com/cockroachdb/cockroach/pkg/build"
 	"github.com/cockroachdb/cockroach/pkg/cli/exit"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod/grafana"
@@ -2751,6 +2752,48 @@ func Deploy(
 	return nil
 }
 
+var sideEyeEnvToken, _ = os.LookupEnv("SIDE_EYE_API_TOKEN")
+
+// CaptureSideEyeSnapshot asks the Side-Eye service to take a snapshot of the
+// cockroach processes of this cluster. All errors are logged and swallowed, and
+// the call is a no-op if the SIDE_EYE_API_TOKEN is not in the env. The agents
+// must previously have been installed and started with the cluster's name as
+// the env name.
+func CaptureSideEyeSnapshot(ctx context.Context, l *logger.Logger, sideEyeEnv string) {
+	sideEyeToken := sideEyeEnvToken
+	if sideEyeToken == "" {
+		sideEyeToken = install.GetGcloudSideEyeSecret()
+	}
+	if sideEyeToken == "" {
+		l.PrintfCtx(ctx, "Side-Eye token is not configured via SIDE_EYE_API_TOKEN or gcloud secret, skipping snapshot")
+		return
+	}
+
+	l.PrintfCtx(ctx, "capturing snapshot of %s env with Side-Eye", sideEyeEnv)
+
+	client, err := sideeyeclient.NewSideEyeClient(sideeyeclient.WithApiToken(sideEyeToken))
+	if err != nil {
+		l.Errorf("failed to create side-eye client: %s", err)
+		return
+	}
+	defer client.Close()
+
+	// Protect against the snapshot taking too long.
+	snapCtx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+	snapRes, err := client.CaptureSnapshot(snapCtx, sideEyeEnv)
+	if err != nil {
+		msg := err.Error()
+		if errors.Is(err, sideeyeclient.NoProcessesError{}) {
+			msg += "; is cockroach running?"
+		}
+		l.PrintfCtx(ctx, "side-eye failed to capture cluster snapshot: %s", msg)
+		return
+	}
+
+	l.PrintfCtx(ctx, "captured side-eye snapshot: %s", snapRes.SnapshotURL)
+}
+
 // getClusterFromCache finds and returns a SyncedCluster from
 // the local cluster cache.
 //