Merge pull request cockroachdb#97470 from maryliag/backport22.2-97153…

…-97277-97312

release-22.2: show data when we hit max size limit

pkg/ui/workspaces/cluster-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cockroachlabs/cluster-ui",
-  "version": "22.2.4",
+  "version": "22.2.5",
   "description": "Cluster UI is a library of large features shared between CockroachDB and CockroachCloud",
   "repository": {
     "type": "git",

pkg/ui/workspaces/cluster-ui/src/api/insightsApi.ts

@@ -10,10 +10,13 @@
 
 import {
   executeInternalSql,
+  formatApiResult,
   INTERNAL_SQL_API_APP,
+  isMaxSizeError,
   LARGE_RESULT_SIZE,
   LONG_TIMEOUT,
   sqlApiErrorMessage,
+  SqlApiResponse,
   SqlExecutionRequest,
   SqlExecutionResponse,
   sqlResultsAreEmpty,
@@ -201,7 +204,9 @@ const makeInsightsSqlRequest = (queries: string[]): SqlExecutionRequest => ({
 });
 
 // getTransactionInsightEventState is the API function that executes the queries and returns the results.
-export async function getTransactionInsightEventState(): Promise<TransactionInsightEventsResponse> {
+export async function getTransactionInsightEventState(): Promise<
+  SqlApiResponse<TransactionInsightEventsResponse>
+> {
   // Note that any errors encountered fetching these results are caught
   // earlier in the call stack.
 
@@ -211,15 +216,20 @@ export async function getTransactionInsightEventState(): Promise<TransactionInsi
     await executeInternalSql<TransactionContentionResponseColumns>(
       makeInsightsSqlRequest([txnContentionQuery]),
     );
-  if (contentionResults.error) {
+  const maxSizeReached = isMaxSizeError(contentionResults.error?.message);
+  if (contentionResults.error && !maxSizeReached) {
     throw new Error(
       `Error while retrieving contention information: ${sqlApiErrorMessage(
         contentionResults.error.message,
       )}`,
     );
   }
   if (sqlResultsAreEmpty(contentionResults)) {
-    return [];
+    return formatApiResult(
+      [],
+      contentionResults.error,
+      "retrieving contention information",
+    );
   }
 
   // Step 2: Fetch the stmt fingerprints in the contended transactions.
@@ -236,15 +246,22 @@ export async function getTransactionInsightEventState(): Promise<TransactionInsi
         txnStmtFingerprintsQuery(Array.from(txnFingerprintIDs)),
       ]),
     );
-  if (txnStmtFingerprintResults.error) {
+  const maxSizeTxnFingerprintReached = isMaxSizeError(
+    txnStmtFingerprintResults.error?.message,
+  );
+  if (txnStmtFingerprintResults.error && !maxSizeTxnFingerprintReached) {
     throw new Error(
       `Error while retrieving statements information: ${sqlApiErrorMessage(
         txnStmtFingerprintResults.error.message,
       )}`,
     );
   }
   if (sqlResultsAreEmpty(txnStmtFingerprintResults)) {
-    return [];
+    return formatApiResult(
+      [],
+      contentionResults.error,
+      "retrieving statements information",
+    );
   }
 
   // Step 3: Get all query strings for statement fingerprints.
@@ -259,7 +276,10 @@ export async function getTransactionInsightEventState(): Promise<TransactionInsi
     await executeInternalSql<FingerprintStmtsResponseColumns>(
       fingerprintStmtsRequest,
     );
-  if (fingerprintStmtResults.error) {
+  const maxSizeStmtFingerprintReached = isMaxSizeError(
+    fingerprintStmtResults.error?.message,
+  );
+  if (fingerprintStmtResults.error && !maxSizeStmtFingerprintReached) {
     throw new Error(
       `Error while retrieving statements information: ${sqlApiErrorMessage(
         fingerprintStmtResults.error.message,
@@ -271,20 +291,27 @@ export async function getTransactionInsightEventState(): Promise<TransactionInsi
     transactionContentionResultsToEventState(contentionResults),
     txnStmtFingerprintsResultsToEventState(txnStmtFingerprintResults),
     fingerprintStmtsResultsToEventState(fingerprintStmtResults),
+    maxSizeReached ||
+      maxSizeTxnFingerprintReached ||
+      maxSizeStmtFingerprintReached,
   );
 }
 
 export function combineTransactionInsightEventState(
   txnContentionState: TransactionContentionEventsResponse,
   txnFingerprintState: TxnStmtFingerprintEventsResponse,
   fingerprintToQuery: StmtFingerprintToQueryRecord,
-): TransactionInsightEventState[] {
+  maxSizeReached: boolean,
+): SqlApiResponse<TransactionInsightEventState[]> {
   if (
     !txnContentionState.length ||
     !txnFingerprintState.length ||
     !fingerprintToQuery.size
   ) {
-    return [];
+    return {
+      maxSizeReached: maxSizeReached,
+      results: [],
+    };
   }
   const txnsWithStmtQueries = txnFingerprintState.map(txnRow => ({
     fingerprintID: txnRow.fingerprintID,
@@ -307,7 +334,10 @@ export function combineTransactionInsightEventState(
     }
   });
 
-  return res;
+  return {
+    maxSizeReached: maxSizeReached,
+    results: res,
+  };
 }
 
 // Transaction insight details.
@@ -455,7 +485,7 @@ function transactionContentionDetailsResultsToEventState(
 // getTransactionInsightEventState is the API function that executes the queries and returns the results.
 export async function getTransactionInsightEventDetailsState(
   req: TransactionInsightEventDetailsRequest,
-): Promise<TransactionInsightEventDetailsResponse> {
+): Promise<SqlApiResponse<TransactionInsightEventDetailsResponse>> {
   // Note that any errors encountered fetching these results are caught // earlier in the call stack.
   //
   // There are 3 api requests/queries in this process.
@@ -471,15 +501,20 @@ export async function getTransactionInsightEventDetailsState(
     await executeInternalSql<TxnContentionDetailsResponseColumns>(
       txnContentionDetailsRequest,
     );
-  if (contentionResults.error) {
+  const maxSizeReached = isMaxSizeError(contentionResults.error?.message);
+  if (contentionResults.error && !maxSizeReached) {
     throw new Error(
       `Error while retrieving contention information: ${sqlApiErrorMessage(
         contentionResults.error.message,
       )}`,
     );
   }
   if (sqlResultsAreEmpty(contentionResults)) {
-    return;
+    return formatApiResult(
+      [],
+      contentionResults.error,
+      "retrieving contention information",
+    );
   }
 
   // Collect all txn fingerprints involved.
@@ -504,7 +539,10 @@ export async function getTransactionInsightEventDetailsState(
     await executeInternalSql<TxnStmtFingerprintsResponseColumns>(
       makeInsightsSqlRequest([txnStmtFingerprintsQuery(txnFingerprintIDs)]),
     );
-  if (getStmtFingerprintsResponse.error) {
+  const maxSizeStmtFingerprintReached = isMaxSizeError(
+    getStmtFingerprintsResponse.error?.message,
+  );
+  if (getStmtFingerprintsResponse.error && !maxSizeStmtFingerprintReached) {
     throw new Error(
       `Error while retrieving statements information: ${sqlApiErrorMessage(
         getStmtFingerprintsResponse.error.message,
@@ -523,7 +561,10 @@ export async function getTransactionInsightEventDetailsState(
         fingerprintStmtsQuery(Array.from(stmtFingerprintIDs)),
       ]),
     );
-  if (stmtQueriesResponse.error) {
+  const maxSizeQueriesReached = isMaxSizeError(
+    stmtQueriesResponse.error?.message,
+  );
+  if (stmtQueriesResponse.error && !maxSizeQueriesReached) {
     throw new Error(
       `Error while retrieving statements information: ${sqlApiErrorMessage(
         stmtQueriesResponse.error.message,
@@ -535,20 +576,25 @@ export async function getTransactionInsightEventDetailsState(
     transactionContentionDetailsResultsToEventState(contentionResults),
     txnStmtFingerprintsResultsToEventState(getStmtFingerprintsResponse),
     fingerprintStmtsResultsToEventState(stmtQueriesResponse),
+    maxSizeReached || maxSizeStmtFingerprintReached || maxSizeQueriesReached,
   );
 }
 
 export function combineTransactionInsightEventDetailsState(
   txnContentionDetailsState: TransactionContentionEventDetailsResponse,
   txnsWithStmtFingerprints: TxnStmtFingerprintEventsResponse,
   stmtFingerprintToQuery: StmtFingerprintToQueryRecord,
-): TransactionInsightEventDetailsState {
+  maxSizeReached: boolean,
+): SqlApiResponse<TransactionInsightEventDetailsState> {
   if (
     !txnContentionDetailsState &&
     !txnsWithStmtFingerprints.length &&
     !stmtFingerprintToQuery.size
   ) {
-    return null;
+    return {
+      maxSizeReached: maxSizeReached,
+      results: null,
+    };
   }
 
   txnContentionDetailsState.blockingContentionDetails.forEach(blockedRow => {
@@ -557,7 +603,10 @@ export function combineTransactionInsightEventDetailsState(
     );
 
     if (!currBlockedFingerprintStmts) {
-      return;
+      return {
+        maxSizeReached: maxSizeReached,
+        results: null,
+      };
     }
 
     blockedRow.blockingQueries = currBlockedFingerprintStmts.queryIDs.map(
@@ -575,7 +624,10 @@ export function combineTransactionInsightEventDetailsState(
     queries: waitingTxn.queryIDs.map(id => stmtFingerprintToQuery.get(id)),
   };
 
-  return res;
+  return {
+    maxSizeReached: maxSizeReached,
+    results: res,
+  };
 }
 
 // Statements
@@ -698,7 +750,9 @@ const statementInsightsQuery: InsightQuery<
   toState: getStatementInsightsFromClusterExecutionInsightsResponse,
 };
 
-export async function getStatementInsightsApi(): Promise<StatementInsights> {
+export async function getStatementInsightsApi(): Promise<
+  SqlApiResponse<StatementInsights>
+> {
   const request: SqlExecutionRequest = {
     statements: [
       {
@@ -712,13 +766,17 @@ export async function getStatementInsightsApi(): Promise<StatementInsights> {
   const result = await executeInternalSql<ExecutionInsightsResponseRow>(
     request,
   );
-  if (result.error) {
+  const maxSizeReached = isMaxSizeError(result.error?.message);
+  if (result.error && !maxSizeReached) {
     throw new Error(
       `Error while retrieving insights information: ${sqlApiErrorMessage(
         result.error.message,
       )}`,
     );
   }
-
-  return statementInsightsQuery.toState(result);
+  return formatApiResult(
+    statementInsightsQuery.toState(result),
+    result.error,
+    "retrieving insights information",
+  );
 }

pkg/ui/workspaces/cluster-ui/src/api/schemaInsightsApi.ts

@@ -14,7 +14,9 @@ import {
   executeInternalSql,
   LONG_TIMEOUT,
   sqlResultsAreEmpty,
-  sqlApiErrorMessage,
+  LARGE_RESULT_SIZE,
+  SqlApiResponse,
+  formatApiResult,
 } from "./sqlApi";
 import {
   InsightRecommendation,
@@ -128,23 +130,37 @@ function createIndexRecommendationsToSchemaInsight(
   return results;
 }
 
+// This query have an ORDER BY for the cases where we reach the limit of the sql-api
+// and want to return the most used ones as a priority.
 const dropUnusedIndexQuery: SchemaInsightQuery<ClusterIndexUsageStatistic> = {
   name: "DropIndex",
-  query: `SELECT
-            us.table_id,
-            us.index_id,
-            us.last_read,
-            ti.created_at,
-            ti.index_name,
-            t.name as table_name,
-            t.parent_id as database_id,
-            t.database_name,
-            t.schema_name,
-            (SELECT value FROM crdb_internal.cluster_settings WHERE variable = 'sql.index_recommendation.drop_unused_duration') AS unused_threshold
-          FROM "".crdb_internal.index_usage_statistics AS us
-                 JOIN "".crdb_internal.table_indexes as ti ON us.index_id = ti.index_id AND us.table_id = ti.descriptor_id
-                 JOIN "".crdb_internal.tables as t ON t.table_id = ti.descriptor_id and t.name = ti.descriptor_name
-          WHERE t.database_name != 'system' AND ti.index_type != 'primary';`,
+  query: `WITH cs AS (
+    SELECT value 
+        FROM crdb_internal.cluster_settings 
+    WHERE variable = 'sql.index_recommendation.drop_unused_duration'
+    )
+    SELECT * FROM (SELECT us.table_id,
+                          us.index_id,
+                          us.last_read,
+                          us.total_reads,
+                          ti.created_at,
+                          ti.index_name,
+                          t.name      as table_name,
+                          t.parent_id as database_id,
+                          t.database_name,
+                          t.schema_name,
+                          cs.value as unused_threshold,
+                          cs.value::interval as interval_threshold, 
+                          now() - COALESCE(us.last_read AT TIME ZONE 'UTC', COALESCE(ti.created_at, '0001-01-01')) as unused_interval
+                   FROM "".crdb_internal.index_usage_statistics AS us
+                            JOIN "".crdb_internal.table_indexes as ti
+                                 ON us.index_id = ti.index_id AND us.table_id = ti.descriptor_id
+                            JOIN "".crdb_internal.tables as t
+                                 ON t.table_id = ti.descriptor_id and t.name = ti.descriptor_name
+                            CROSS JOIN cs
+                   WHERE t.database_name != 'system' AND ti.index_type != 'primary')
+          WHERE unused_interval > interval_threshold
+          ORDER BY total_reads DESC;`,
   toSchemaInsight: clusterIndexUsageStatsToSchemaInsight,
 };
 
@@ -181,26 +197,22 @@ const schemaInsightQueries: SchemaInsightQuery<SchemaInsightResponse>[] = [
 
 // getSchemaInsights makes requests over the SQL API and transforms the corresponding
 // SQL responses into schema insights.
-export async function getSchemaInsights(): Promise<InsightRecommendation[]> {
+export async function getSchemaInsights(): Promise<
+  SqlApiResponse<InsightRecommendation[]>
+> {
   const request: SqlExecutionRequest = {
     statements: schemaInsightQueries.map(insightQuery => ({
       sql: insightQuery.query,
     })),
     execute: true,
+    max_result_size: LARGE_RESULT_SIZE,
     timeout: LONG_TIMEOUT,
   };
   const result = await executeInternalSql<SchemaInsightResponse>(request);
-  if (result.error) {
-    throw new Error(
-      `Error while retrieving insights information: ${sqlApiErrorMessage(
-        result.error.message,
-      )}`,
-    );
-  }
+
   const results: InsightRecommendation[] = [];
   if (sqlResultsAreEmpty(result)) {
-    // No data.
-    return results;
+    return formatApiResult([], result.error, "retrieving insights information");
   }
   result.execution.txn_results.map(txn_result => {
     // Note: txn_result.statement values begin at 1, not 0.
@@ -210,5 +222,9 @@ export async function getSchemaInsights(): Promise<InsightRecommendation[]> {
       results.push(...insightQuery.toSchemaInsight(txn_result));
     }
   });
-  return results;
+  return formatApiResult(
+    results,
+    result.error,
+    "retrieving insights information",
+  );
 }