Merge #77504 #78293 #78616 #78654

77504: cluster-ui: add active stmts and txns pages for CC r=jocrl a=xinhaoz

This commit adds the active statements and active transactions
pages required by cockroach cloud. The components created
in this commit are intended for use in `managed-service`.

Release note: None

78293: roachtest: sstable-corruption: collect manifests; additional logging r=jbowens a=nicktrav

Add additional logging of the contents of the data directory and collect
the Pebble manifests when the test fails to find tables to corrupt. This
will aid in diagnosing the test failures.

Touches #78121.

Release note: None.

78616: opt: disable index recommendations with PARTITION ALL BY r=rytaft a=rytaft

This commit disables index recommendations for tables with
`PARTITION ALL BY` (including `REGIONAL BY ROW` tables) because the
current recommendations are not valid. This is a backportable
fix, but a future PR will fix these recommendations and reenable
them.

Release note (sql change): Disabled index recommendations in `EXPLAIN`
output for `REGIONAL BY ROW` tables, as the previous recommendations were
not valid.

78654: roachprod: do not send status report to slack r=rail a=rail

Previously, the roachprod GC job posted cluster status on each run. This
made the #roachprod-status channel very spammy and not actionable.

This patch removes the status generation part.

Release note: None

Co-authored-by: Xin Hao Zhang <[email protected]>
Co-authored-by: Nick Travers <[email protected]>
Co-authored-by: Rebecca Taft <[email protected]>
Co-authored-by: Rail Aliiev <[email protected]>

pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior

@@ -2809,3 +2809,56 @@ SELECT * FROM [EXPLAIN SELECT * FROM regional_by_row_table_as1 LIMIT 3] OFFSET 2
           table: regional_by_row_table_as1@regional_by_row_table_as1_pkey
           spans: [/'ca-central-1' - /'us-east-1']
           limit: 3
+
+subtest index_recommendations
+
+# Enable vectorize so we get consistent EXPLAIN output. We cannot use the
+# OFFSET 2 strategy for these tests because that disables the index
+# recommendation (index recommendations are only used when EXPLAIN is the
+# root of the query tree).
+statement ok
+SET index_recommendations_enabled = true;
+SET vectorize=on
+
+statement ok
+CREATE TABLE users (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  name STRING NOT NULL,
+  email STRING NOT NULL UNIQUE,
+  INDEX (name)
+) LOCALITY REGIONAL BY ROW
+
+# Check that we don't recommend indexes that already exist.
+query T
+EXPLAIN INSERT INTO users (name, email)
+VALUES ('Craig Roacher', '[email protected]')
+----
+distribution: local
+vectorized: true
+·
+• root
+│
+├── • insert
+│   │ into: users(id, name, email, crdb_region)
+│   │
+│   └── • buffer
+│       │ label: buffer 1
+│       │
+│       └── • values
+│             size: 5 columns, 1 row
+│
+└── • constraint-check
+    │
+    └── • error if rows
+        │
+        └── • lookup join (semi)
+            │ table: users@users_email_key
+            │ lookup condition: (column2 = email) AND (crdb_region IN ('ap-southeast-2', 'ca-central-1', 'us-east-1'))
+            │ pred: (id_default != id) OR (crdb_region_default != crdb_region)
+            │
+            └── • scan buffer
+                  label: buffer 1
+
+statement ok
+SET index_recommendations_enabled = false;
+RESET vectorize

pkg/cmd/roachtest/tests/sstable_corruption.go

@@ -13,6 +13,7 @@ package tests
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -53,31 +54,65 @@ func runSSTableCorruption(ctx context.Context, t test.Test, c cluster.Cluster) {
 	opts.RoachprodOpts.Wait = true
 	c.Stop(ctx, t.L(), opts, crdbNodes)
 
-	const findTablesCmd = "" +
+	const nTables = 6
+	var dumpManifestCmd = "" +
 		// Take the latest manifest file ...
 		"ls -tr {store-dir}/MANIFEST-* | tail -n1 | " +
 		// ... dump its contents ...
 		"xargs ./cockroach debug pebble manifest dump | " +
-		// ... shuffle the files to distribute corruption over the LSM.
+		// ... filter for SSTables that contain table data.
+		"grep -v added | grep -v deleted | grep '/Table/'"
+	var findTablesCmd = dumpManifestCmd + "| " +
+		// Shuffle the files to distribute corruption over the LSM ...
 		"shuf | " +
-		// ... filter for up to six SSTables that contain table data.
-		"grep -v added | grep -v deleted | grep '/Table/' | tail -n6"
+		// ... take a fixed number of tables.
+		fmt.Sprintf("tail -n %d", nTables)
 
 	for _, node := range corruptNodes {
 		result, err := c.RunWithDetailsSingleNode(ctx, t.L(), c.Node(node), findTablesCmd)
 		if err != nil {
 			t.Fatalf("could not find tables to corrupt: %s\nstdout: %s\nstderr: %s", err, result.Stdout, result.Stderr)
 		}
 		tableSSTs := strings.Split(strings.TrimSpace(result.Stdout), "\n")
-		if len(tableSSTs) == 0 {
-			t.Fatal("expected at least one sst containing table keys only, got none")
+		if len(tableSSTs) != nTables {
+			// We couldn't find enough tables to corrupt. As there should be an
+			// abundance of tables, this warrants further investigation. To aid in
+			// such an investigation, print the contents of the data directory.
+			cmd := "ls -l {store-dir}"
+			result, err = c.RunWithDetailsSingleNode(ctx, t.L(), c.Node(node), cmd)
+			if err == nil {
+				t.Status("store dir contents:\n", result.Stdout)
+			}
+			// Fetch the MANIFEST files from this node.
+			result, err = c.RunWithDetailsSingleNode(
+				ctx, t.L(), c.Node(node),
+				"tar czf {store-dir}/manifests.tar.gz {store-dir}/MANIFEST-*",
+			)
+			if err != nil {
+				t.Fatalf("could not create manifest file archive: %s", err)
+			}
+			result, err = c.RunWithDetailsSingleNode(ctx, t.L(), c.Node(node), "echo", "-n", "{store-dir}")
+			if err != nil {
+				t.Fatalf("could not infer store directory: %s", err)
+			}
+			storeDirectory := result.Stdout
+			srcPath := filepath.Join(storeDirectory, "manifests.tar.gz")
+			dstPath := filepath.Join(t.ArtifactsDir(), fmt.Sprintf("manifests.%d.tar.gz", node))
+			err = c.Get(ctx, t.L(), srcPath, dstPath, c.Node(node))
+			if err != nil {
+				t.Fatalf("could not fetch manifest archive: %s", err)
+			}
+			t.Fatalf(
+				"expected %d SSTables containing table keys, got %d: %s",
+				nTables, len(tableSSTs), tableSSTs,
+			)
 		}
 		// Corrupt the SSTs.
 		for _, sstLine := range tableSSTs {
 			sstLine = strings.TrimSpace(sstLine)
 			firstFileIdx := strings.Index(sstLine, ":")
 			if firstFileIdx < 0 {
-				t.Fatalf("unexpected format for sst line: %s", sstLine)
+				t.Fatalf("unexpected format for sst line: %q", sstLine)
 			}
 			_, err = strconv.Atoi(sstLine[:firstFileIdx])
 			if err != nil {

pkg/roachprod/cloud/gc.go

@@ -312,11 +312,7 @@ func GCClusters(l *logger.Logger, cloud *Cloud, dryrun bool) error {
 		}
 	}
 
-	// Send out notification to #roachprod-status.
 	client := makeSlackClient()
-	channel, _ := findChannel(client, "roachprod-status", "")
-	postStatus(l, client, channel, dryrun, &s, badVMs)
-
 	// Send out user notifications if any of the user's clusters are expired or
 	// will be destroyed.
 	for user, status := range users {
@@ -330,6 +326,7 @@ func GCClusters(l *logger.Logger, cloud *Cloud, dryrun bool) error {
 		}
 	}
 
+	channel, _ := findChannel(client, "roachprod-status", "")
 	if !dryrun {
 		if len(badVMs) > 0 {
 			// Destroy bad VMs.

pkg/sql/opt/cat/table.go

@@ -136,6 +136,10 @@ type Table interface {
 
 	// Zone returns a table's zone.
 	Zone() Zone
+
+	// IsPartitionAllBy returns true if this is a PARTITION ALL BY table. This
+	// includes REGIONAL BY ROW tables.
+	IsPartitionAllBy() bool
 }
 
 // CheckConstraint contains the SQL text and the validity status for a check

pkg/sql/opt/indexrec/index_candidate_set.go

@@ -373,7 +373,14 @@ func addIndexToCandidates(
 		return
 	}
 
+	// Do not add indexes to PARTITION ALL BY tables.
+	// TODO(rytaft): Support these tables by adding implicit partitioning columns.
+	if currTable.IsPartitionAllBy() {
+		return
+	}
+
 	// Do not add indexes on spatial columns.
+	// TODO(rytaft): Support spatial predicates like st_contains() etc.
 	for _, indexCol := range newIndex {
 		colFamily := indexCol.Column.DatumType().Family()
 		if colFamily == types.GeometryFamily || colFamily == types.GeographyFamily {

pkg/sql/opt/testutils/testcat/test_catalog.go

@@ -755,6 +755,11 @@ func (tt *Table) Zone() cat.Zone {
 	return cat.AsZone(&zone)
 }
 
+// IsPartitionAllBy is part of the cat.Table interface.
+func (tt *Table) IsPartitionAllBy() bool {
+	return false
+}
+
 // FindOrdinal returns the ordinal of the column with the given name.
 func (tt *Table) FindOrdinal(name string) int {
 	for i, col := range tt.Columns {

pkg/sql/opt_catalog.go

@@ -1158,6 +1158,11 @@ func (ot *optTable) Zone() cat.Zone {
 	return ot.zone
 }
 
+// IsPartitionAllBy is part of the cat.Table interface.
+func (ot *optTable) IsPartitionAllBy() bool {
+	return ot.desc.IsPartitionAllBy()
+}
+
 // lookupColumnOrdinal returns the ordinal of the column with the given ID. A
 // cache makes the lookup O(1).
 func (ot *optTable) lookupColumnOrdinal(colID descpb.ColumnID) (int, error) {
@@ -2073,6 +2078,11 @@ func (ot *optVirtualTable) Zone() cat.Zone {
 	panic(errors.AssertionFailedf("no zone"))
 }
 
+// IsPartitionAllBy is part of the cat.Table interface.
+func (ot *optVirtualTable) IsPartitionAllBy() bool {
+	return false
+}
+
 // CollectTypes is part of the cat.DataSource interface.
 func (ot *optVirtualTable) CollectTypes(ord int) (descpb.IDs, error) {
 	col := ot.desc.AllColumns()[ord]

pkg/ui/workspaces/cluster-ui/src/statementDetails/activeStatementDetails.selectors.ts

@@ -0,0 +1,30 @@
+// Copyright 2022 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+import { createSelector } from "reselect";
+import { AppState } from "src";
+import { match, RouteComponentProps } from "react-router-dom";
+import { getMatchParamByName } from "src/util/query";
+import { executionIdAttr } from "../util/constants";
+import { getActiveStatementsFromSessions } from "../activeExecutions/activeStatementUtils";
+
+export const selectActiveStatement = createSelector(
+  (_: AppState, props: RouteComponentProps) => props.match,
+  (state: AppState) => state.adminUI.sessions,
+  (match: match, response) => {
+    if (!response.data) return null;
+
+    const executionID = getMatchParamByName(match, executionIdAttr);
+    return getActiveStatementsFromSessions(
+      response.data,
+      response.lastUpdated,
+    ).find(stmt => stmt.executionID === executionID);
+  },
+);

pkg/ui/workspaces/cluster-ui/src/statementDetails/activeStatementDetailsConnected.tsx

@@ -0,0 +1,43 @@
+// Copyright 2022 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+import { RouteComponentProps, withRouter } from "react-router-dom";
+import { connect } from "react-redux";
+import { Dispatch } from "redux";
+import { actions as sessionsActions } from "src/store/sessions";
+import { AppState } from "../store";
+import {
+  ActiveStatementDetails,
+  ActiveStatementDetailsDispatchProps,
+} from "./activeStatementDetails";
+import { selectActiveStatement } from "./activeStatementDetails.selectors";
+import { ActiveStatementDetailsStateProps } from ".";
+
+// For tenant cases, we don't show information about node, regions and
+// diagnostics.
+const mapStateToProps = (
+  state: AppState,
+  props: RouteComponentProps,
+): ActiveStatementDetailsStateProps => {
+  return {
+    statement: selectActiveStatement(state, props),
+    match: props.match,
+  };
+};
+
+const mapDispatchToProps = (
+  dispatch: Dispatch,
+): ActiveStatementDetailsDispatchProps => ({
+  refreshSessions: () => dispatch(sessionsActions.refresh()),
+});
+
+export const ActiveStatementDetailsPageConnected = withRouter(
+  connect(mapStateToProps, mapDispatchToProps)(ActiveStatementDetails),
+);

pkg/ui/workspaces/cluster-ui/src/statementDetails/index.ts

@@ -14,3 +14,4 @@ export * from "./diagnostics/diagnosticsUtils";
 export * from "./planView";
 export * from "./statementDetailsConnected";
 export * from "./activeStatementDetails";
+export * from "./activeStatementDetailsConnected";

pkg/ui/workspaces/cluster-ui/src/statementsPage/activeStatementsPage.selectors.ts

@@ -0,0 +1,90 @@
+// Copyright 2022 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+import { createSelector } from "reselect";
+import { getActiveStatementsFromSessions } from "../activeExecutions/activeStatementUtils";
+import { localStorageSelector } from "./statementsPage.selectors";
+import {
+  ActiveStatementFilters,
+  ActiveStatementsViewDispatchProps,
+  ActiveStatementsViewStateProps,
+  AppState,
+  SortSetting,
+} from "src";
+import { actions as sessionsActions } from "src/store/sessions";
+import { actions as localStorageActions } from "src/store/localStorage";
+import { Dispatch } from "redux";
+
+export const selectActiveStatements = createSelector(
+  (state: AppState) => state.adminUI.sessions,
+  response => {
+    if (!response.data) return [];
+
+    return getActiveStatementsFromSessions(response.data, response.lastUpdated);
+  },
+);
+
+export const selectSortSetting = (state: AppState): SortSetting =>
+  localStorageSelector(state)["sortSetting/ActiveStatementsPage"];
+
+export const selectFilters = (state: AppState): ActiveStatementFilters =>
+  localStorageSelector(state)["filters/ActiveStatementsPage"];
+
+const selectLocalStorageColumns = (state: AppState) => {
+  const localStorage = localStorageSelector(state);
+  return localStorage["showColumns/ActiveStatementsPage"];
+};
+
+export const selectColumns = createSelector(
+  selectLocalStorageColumns,
+  value => {
+    if (value == null) return null;
+
+    return value.split(",").map(col => col.trim());
+  },
+);
+
+export const mapStateToActiveStatementsPageProps = (
+  state: AppState,
+): ActiveStatementsViewStateProps => ({
+  statements: selectActiveStatements(state),
+  sessionsError: state.adminUI.sessions.lastError,
+  selectedColumns: selectColumns(state),
+  sortSetting: selectSortSetting(state),
+  filters: selectFilters(state),
+});
+
+export const mapDispatchToActiveStatementsPageProps = (
+  dispatch: Dispatch,
+): ActiveStatementsViewDispatchProps => ({
+  refreshSessions: () => dispatch(sessionsActions.refresh()),
+  onColumnsSelect: columns => {
+    dispatch(
+      localStorageActions.update({
+        key: "showColumns/ActiveStatementsPage",
+        value: columns.join(","),
+      }),
+    );
+  },
+  onFiltersChange: (filters: ActiveStatementFilters) =>
+    dispatch(
+      localStorageActions.update({
+        key: "filters/ActiveStatementsPage",
+        value: filters,
+      }),
+    ),
+  onSortChange: (ss: SortSetting) =>
+    dispatch(
+      localStorageActions.update({
+        key: "sortSetting/ActiveStatementsPage",
+        value: ss,
+      }),
+    ),
+});