From 9933b5ea9a447a5fc4306f1e7c5b35b8b10e9c99 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Tue, 25 Jun 2024 20:16:26 +0000 Subject: [PATCH 1/2] roachprod: add side-eye snap command Release note: none. Epic: none. --- DEPS.bzl | 10 ++++++++ build/bazelutil/distdir_files.bzl | 1 + go.mod | 1 + go.sum | 2 ++ pkg/cmd/roachprod/flags.go | 2 ++ pkg/cmd/roachprod/main.go | 17 ++++++++++++++ pkg/roachprod/BUILD.bazel | 1 + pkg/roachprod/roachprod.go | 39 +++++++++++++++++++++++++++++++ 8 files changed, 73 insertions(+) diff --git a/DEPS.bzl b/DEPS.bzl index 1d20ae825384..ce7c9e2f170e 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -2238,6 +2238,16 @@ def go_deps(): "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/zstd/com_github_datadog_zstd-v1.5.6-0.20230824185856-869dae002e5e.zip", ], ) + go_repository( + name = "com_github_dataexmachina_dev_side_eye_go", + build_file_proto_mode = "disable_global", + importpath = "github.com/DataExMachina-dev/side-eye-go", + sha256 = "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f", + strip_prefix = "github.com/DataExMachina-dev/side-eye-go@v0.0.0-20240528211710-5eb9c7a69e1d", + urls = [ + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip", + ], + ) go_repository( name = "com_github_dave_dst", build_file_proto_mode = "disable_global", diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index 0c34b0002ca8..fa07fb465ba3 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -169,6 +169,7 @@ DISTDIR_FILES = { "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DATA-DOG/go-sqlmock/com_github_data_dog_go_sqlmock-v1.5.0.zip": "25720bfcbd739305238408ab54263224b69ff6934923dfd9caed76d3871d0151", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/datadog-go/com_github_datadog_datadog_go-v3.2.0+incompatible.zip": "ede4a024d3c106b2f57ca04d7bfc7610e0c83f4d8a3bace2cf87b42fd5cf66cd", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/zstd/com_github_datadog_zstd-v1.5.6-0.20230824185856-869dae002e5e.zip": "e4924158bd1abf765a016d2c728fc367b32d20b86a268ef25743ba404c55e097", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip": "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GeertJohan/go.incremental/com_github_geertjohan_go_incremental-v1.0.0.zip": "ce46b3b717f8d2927046bcfb99c6f490b1b547a681e6b23240ac2c2292a891e8", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GeertJohan/go.rice/com_github_geertjohan_go_rice-v1.0.0.zip": "2fc48b9422bf356c18ed3fe32ec52f6a8b87ac168f83d2eed249afaebcc3eeb8", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GoogleCloudPlatform/cloudsql-proxy/com_github_googlecloudplatform_cloudsql_proxy-v0.0.0-20190129172621-c8b1d7a94ddf.zip": "d18ff41309efc943c71d5c8faa5b1dd792700a79fa4f61508c5e50f17fc9ca6f", diff --git a/go.mod b/go.mod index 656004e9d0b6..791e8010abad 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,7 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.9.15 github.com/BurntSushi/toml v1.2.1 github.com/DataDog/datadog-go v3.2.0+incompatible + github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d github.com/IBM/sarama v1.42.1 github.com/Masterminds/semver/v3 v3.1.1 github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 diff --git a/go.sum b/go.sum index 8790b586a5c7..dcc83eb2540b 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,8 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d h1:0NRhNOBaRnkXED8mftbOSCNGAf8MZhv4zu840hIUpIc= +github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d/go.mod h1:FukCpc3od3BzYgxUtTWm3iB4ALtc4UknLNMQ0rq+V3A= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190129172621-c8b1d7a94ddf/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= diff --git a/pkg/cmd/roachprod/flags.go b/pkg/cmd/roachprod/flags.go index 79b9e4bed3c6..9aeab6df9847 100644 --- a/pkg/cmd/roachprod/flags.go +++ b/pkg/cmd/roachprod/flags.go @@ -346,6 +346,8 @@ func initFlags() { jaegerStartCmd.Flags().StringVar(&jaegerConfigNodes, "configure-nodes", "", "the nodes on which to set the relevant CRDB cluster settings") + sideEyeRootCmd.AddCommand(sideEyeSnapCmd) + initCmd.Flags().IntVar(&startOpts.InitTarget, "init-target", startOpts.InitTarget, "node on which to run initialization") diff --git a/pkg/cmd/roachprod/main.go b/pkg/cmd/roachprod/main.go index 7058df889992..81c9988621c9 100644 --- a/pkg/cmd/roachprod/main.go +++ b/pkg/cmd/roachprod/main.go @@ -1629,6 +1629,22 @@ var storageSnapshotCmd = &cobra.Command{ }), } +var sideEyeRootCmd = &cobra.Command{ + Use: "side-eye", + Short: "interact with side-eye.io functionality", + Args: cobra.MinimumNArgs(1), +} +var sideEyeSnapCmd = &cobra.Command{ + Use: "snapshot", + Aliases: []string{"snap"}, + Short: "interact with side-eye.io functionality", + Args: cobra.ExactArgs(1), + Run: wrap(func(cmd *cobra.Command, args []string) error { + roachprod.CaptureSideEyeSnapshot(context.Background(), config.Logger, args[0]) + return nil + }), +} + // Before executing any command, validate and canonicalize args. func validateAndConfigure(cmd *cobra.Command, args []string) { // Skip validation for commands that are self-sufficient. @@ -1940,6 +1956,7 @@ func main() { jaegerStartCmd, jaegerStopCmd, jaegerURLCmd, + sideEyeRootCmd, fluentBitStartCmd, fluentBitStopCmd, opentelemetryStartCmd, diff --git a/pkg/roachprod/BUILD.bazel b/pkg/roachprod/BUILD.bazel index 0b1185853f55..e4af1abd5327 100644 --- a/pkg/roachprod/BUILD.bazel +++ b/pkg/roachprod/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "//pkg/util/timeutil", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_errors//oserror", + "@com_github_dataexmachina_dev_side_eye_go//sideeyeclient", "@org_golang_x_sys//unix", ], ) diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index d360ff286e68..4f9ea0b1c038 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -32,6 +32,7 @@ import ( "sync" "time" + "github.com/DataExMachina-dev/side-eye-go/sideeyeclient" "github.com/cockroachdb/cockroach/pkg/build" "github.com/cockroachdb/cockroach/pkg/cli/exit" "github.com/cockroachdb/cockroach/pkg/cmd/roachprod/grafana" @@ -2751,6 +2752,44 @@ func Deploy( return nil } +var sideEyeToken, _ = os.LookupEnv("SIDE_EYE_API_TOKEN") + +// CaptureSideEyeSnapshot asks the Side-Eye service to take a snapshot of the +// cockroach processes of this cluster. All errors are logged and swallowed, and +// the call is a no-op if the SIDE_EYE_API_TOKEN is not in the env. The agents +// must previously have been installed and started with the cluster's name as +// the env name. +func CaptureSideEyeSnapshot(ctx context.Context, l *logger.Logger, sideEyeEnv string) { + if sideEyeToken == "" { + l.PrintfCtx(ctx, "Side-Eye token is not configured, skipping snapshot") + return + } + + l.PrintfCtx(ctx, "capturing snapshot of %s env with Side-Eye", sideEyeEnv) + + client, err := sideeyeclient.NewSideEyeClient(sideeyeclient.WithApiToken(sideEyeToken)) + if err != nil { + l.Errorf("failed to create side-eye client: %s", err) + return + } + defer client.Close() + + // Protect against the snapshot taking too long. + snapCtx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + snapRes, err := client.CaptureSnapshot(snapCtx, sideEyeEnv) + if err != nil { + msg := err.Error() + if errors.Is(err, sideeyeclient.NoProcessesError{}) { + msg += "; is cockroach running?" + } + l.PrintfCtx(ctx, "side-eye failed to capture cluster snapshot: %s", msg) + return + } + + l.PrintfCtx(ctx, "captured side-eye snapshot: %s", snapRes.SnapshotURL) +} + // getClusterFromCache finds and returns a SyncedCluster from // the local cluster cache. // From 4246b6773b5f4f850d1df9a0db656d740f78cda7 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Tue, 25 Jun 2024 20:22:33 +0000 Subject: [PATCH 2/2] roachprod: add gcloud secret support to side-eye snap Release note: none. Epic: none. --- pkg/cmd/roachprod/main.go | 15 +++++++++++---- pkg/roachprod/install/install.go | 14 +++++++++++++- pkg/roachprod/roachprod.go | 8 ++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/cmd/roachprod/main.go b/pkg/cmd/roachprod/main.go index 81c9988621c9..79c395c763a3 100644 --- a/pkg/cmd/roachprod/main.go +++ b/pkg/cmd/roachprod/main.go @@ -1632,13 +1632,20 @@ var storageSnapshotCmd = &cobra.Command{ var sideEyeRootCmd = &cobra.Command{ Use: "side-eye", Short: "interact with side-eye.io functionality", - Args: cobra.MinimumNArgs(1), + Long: `Interact with side-eye.io functionality +Side-Eye (app.side-eye.io) is a distributed debugger that can be used to capture +snapshots of a CockroachDB cluster. +`, + Args: cobra.MinimumNArgs(1), } var sideEyeSnapCmd = &cobra.Command{ - Use: "snapshot", + Use: "snapshot ", Aliases: []string{"snap"}, - Short: "interact with side-eye.io functionality", - Args: cobra.ExactArgs(1), + Short: "capture a cluster snapshot", + Long: `Capture a cluster snapshot using Side-Eye +The command will print an app.side-eye.io URL where the snapshot can be viewed. +`, + Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { roachprod.CaptureSideEyeSnapshot(context.Background(), config.Logger, args[0]) return nil diff --git a/pkg/roachprod/install/install.go b/pkg/roachprod/install/install.go index 495583117740..7d0df2846bda 100644 --- a/pkg/roachprod/install/install.go +++ b/pkg/roachprod/install/install.go @@ -116,10 +116,12 @@ rm /tmp/otelcol-contrib.deb; // a command locally. var installLocalCmds = map[string]map[string]*exec.Cmd{ "side-eye": { - "%API_KEY%": exec.Command("gcloud", "secrets", "versions", "access", "latest", "--secret", "side-eye-key"), + "%API_KEY%": sideEyeSecretCmd, }, } +var sideEyeSecretCmd = exec.Command("gcloud", "secrets", "versions", "access", "latest", "--secret", "side-eye-key") + // SortedCmds TODO(peter): document func SortedCmds() []string { cmds := make([]string, 0, len(installCmds)) @@ -174,3 +176,13 @@ func InstallTool( return nil } + +func GetGcloudSideEyeSecret() string { + c := *sideEyeSecretCmd + c.Stderr = os.Stderr + out, err := c.Output() + if err != nil { + return "" + } + return string(out) +} diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index 4f9ea0b1c038..0eb173951df4 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -2752,7 +2752,7 @@ func Deploy( return nil } -var sideEyeToken, _ = os.LookupEnv("SIDE_EYE_API_TOKEN") +var sideEyeEnvToken, _ = os.LookupEnv("SIDE_EYE_API_TOKEN") // CaptureSideEyeSnapshot asks the Side-Eye service to take a snapshot of the // cockroach processes of this cluster. All errors are logged and swallowed, and @@ -2760,8 +2760,12 @@ var sideEyeToken, _ = os.LookupEnv("SIDE_EYE_API_TOKEN") // must previously have been installed and started with the cluster's name as // the env name. func CaptureSideEyeSnapshot(ctx context.Context, l *logger.Logger, sideEyeEnv string) { + sideEyeToken := sideEyeEnvToken if sideEyeToken == "" { - l.PrintfCtx(ctx, "Side-Eye token is not configured, skipping snapshot") + sideEyeToken = install.GetGcloudSideEyeSecret() + } + if sideEyeToken == "" { + l.PrintfCtx(ctx, "Side-Eye token is not configured via SIDE_EYE_API_TOKEN or gcloud secret, skipping snapshot") return }