Merge pull request #86869 from maryliag/backport22.1-86679

release-22.1: server,ui: show internal stats with new cluster setting

docs/generated/settings/settings-for-tenants.txt

@@ -170,6 +170,7 @@ sql.stats.multi_column_collection.enabled	boolean	true	multi-column statistics c
 sql.stats.persisted_rows.max	integer	1000000	maximum number of rows of statement and transaction statistics that will be persisted in the system tables
 sql.stats.post_events.enabled	boolean	false	if set, an event is logged for every CREATE STATISTICS job
 sql.stats.response.max	integer	20000	the maximum number of statements and transaction stats returned in a CombinedStatements request
+sql.stats.response.show_internal.enabled	boolean	false	controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity
 sql.telemetry.query_sampling.enabled	boolean	false	when set to true, executed queries will emit an event on the telemetry logging channel
 sql.temp_object_cleaner.cleanup_interval	duration	30m0s	how often to clean up orphaned temporary objects
 sql.temp_object_cleaner.wait_interval	duration	30m0s	how long after creation a temporary object will be cleaned up

docs/generated/settings/settings.html

@@ -186,6 +186,7 @@
 <tr><td><code>sql.stats.persisted_rows.max</code></td><td>integer</td><td><code>1000000</code></td><td>maximum number of rows of statement and transaction statistics that will be persisted in the system tables</td></tr>
 <tr><td><code>sql.stats.post_events.enabled</code></td><td>boolean</td><td><code>false</code></td><td>if set, an event is logged for every CREATE STATISTICS job</td></tr>
 <tr><td><code>sql.stats.response.max</code></td><td>integer</td><td><code>20000</code></td><td>the maximum number of statements and transaction stats returned in a CombinedStatements request</td></tr>
+<tr><td><code>sql.stats.response.show_internal.enabled</code></td><td>boolean</td><td><code>false</code></td><td>controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity</td></tr>
 <tr><td><code>sql.telemetry.query_sampling.enabled</code></td><td>boolean</td><td><code>false</code></td><td>when set to true, executed queries will emit an event on the telemetry logging channel</td></tr>
 <tr><td><code>sql.temp_object_cleaner.cleanup_interval</code></td><td>duration</td><td><code>30m0s</code></td><td>how often to clean up orphaned temporary objects</td></tr>
 <tr><td><code>sql.temp_object_cleaner.wait_interval</code></td><td>duration</td><td><code>30m0s</code></td><td>how long after creation a temporary object will be cleaned up</td></tr>

pkg/server/cluster_settings.go

@@ -21,3 +21,13 @@ var SQLStatsResponseMax = settings.RegisterIntSetting(
 	20000,
 	settings.NonNegativeInt,
 ).WithPublic()
+
+// SQLStatsShowInternal controls if statistics for internal executions should be returned by the
+// CombinedStatements endpoint.
+var SQLStatsShowInternal = settings.RegisterBoolSetting(
+	settings.TenantWritable,
+	"sql.stats.response.show_internal.enabled",
+	"controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This "+
+		"endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity",
+	false,
+).WithPublic()

pkg/server/combined_statement_stats.go

@@ -70,7 +70,9 @@ func getCombinedStatementStats(
 	startTime := getTimeFromSeconds(req.Start)
 	endTime := getTimeFromSeconds(req.End)
 	limit := SQLStatsResponseMax.Get(&settings.SV)
-	whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs(startTime, endTime, limit, testingKnobs)
+	showInternal := SQLStatsShowInternal.Get(&settings.SV)
+	whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs(
+		startTime, endTime, limit, testingKnobs, showInternal)
 	statements, err := collectCombinedStatements(ctx, ie, whereClause, args, orderAndLimit)
 	if err != nil {
 		return nil, serverError(ctx, err)
@@ -98,13 +100,17 @@ func getCombinedStatementStats(
 // The whereClause will be in the format `WHERE A = $1 AND B = $2` and
 // args will return the list of arguments in order that will replace the actual values.
 func getCombinedStatementsQueryClausesAndArgs(
-	start, end *time.Time, limit int64, testingKnobs *sqlstats.TestingKnobs,
+	start, end *time.Time, limit int64, testingKnobs *sqlstats.TestingKnobs, showInternal bool,
 ) (whereClause string, orderAndLimitClause string, args []interface{}) {
 	var buffer strings.Builder
 	buffer.WriteString(testingKnobs.GetAOSTClause())
 
-	// Filter out internal statements by app name.
-	buffer.WriteString(fmt.Sprintf(" WHERE app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	if showInternal {
+		buffer.WriteString(" WHERE true")
+	} else {
+		// Filter out internal statements by app name.
+		buffer.WriteString(fmt.Sprintf(" WHERE app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	}
 
 	if start != nil {
 		buffer.WriteString(" AND aggregated_ts >= $1")
@@ -357,7 +363,8 @@ func getStatementDetails(
 	testingKnobs *sqlstats.TestingKnobs,
 ) (*serverpb.StatementDetailsResponse, error) {
 	limit := SQLStatsResponseMax.Get(&settings.SV)
-	whereClause, args, err := getStatementDetailsQueryClausesAndArgs(req, testingKnobs)
+	showInternal := SQLStatsShowInternal.Get(&settings.SV)
+	whereClause, args, err := getStatementDetailsQueryClausesAndArgs(req, testingKnobs, showInternal)
 	if err != nil {
 		return nil, serverError(ctx, err)
 	}
@@ -407,7 +414,7 @@ func getStatementDetails(
 // The whereClause will be in the format `WHERE A = $1 AND B = $2` and
 // args will return the list of arguments in order that will replace the actual values.
 func getStatementDetailsQueryClausesAndArgs(
-	req *serverpb.StatementDetailsRequest, testingKnobs *sqlstats.TestingKnobs,
+	req *serverpb.StatementDetailsRequest, testingKnobs *sqlstats.TestingKnobs, showInternal bool,
 ) (whereClause string, args []interface{}, err error) {
 	var buffer strings.Builder
 	buffer.WriteString(testingKnobs.GetAOSTClause())
@@ -420,18 +427,24 @@ func getStatementDetailsQueryClausesAndArgs(
 	args = append(args, sqlstatsutil.EncodeUint64ToBytes(fingerprintID))
 	buffer.WriteString(fmt.Sprintf(" WHERE fingerprint_id = $%d", len(args)))
 
-	// Filter out internal statements by app name.
-	buffer.WriteString(fmt.Sprintf(" AND app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	if !showInternal {
+		// Filter out internal statements by app name.
+		buffer.WriteString(fmt.Sprintf(" AND app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	}
 
 	// Statements are grouped ignoring the app name in the Statements/Transactions page, so when
 	// calling for the Statement Details endpoint, this value can be empty or a list of app names.
 	if len(req.AppNames) > 0 {
 		if !(len(req.AppNames) == 1 && req.AppNames[0] == "") {
+			hasInternal := false
 			buffer.WriteString(" AND (")
 			for i, app := range req.AppNames {
 				if app == "(unset)" {
 					app = ""
 				}
+				if strings.Contains(app, catconstants.InternalAppNamePrefix) {
+					hasInternal = true
+				}
 				if i != 0 {
 					args = append(args, app)
 					buffer.WriteString(fmt.Sprintf(" OR app_name = $%d", len(args)))
@@ -440,6 +453,9 @@ func getStatementDetailsQueryClausesAndArgs(
 					buffer.WriteString(fmt.Sprintf(" app_name = $%d", len(args)))
 				}
 			}
+			if hasInternal {
+				buffer.WriteString(fmt.Sprintf(" OR app_name LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+			}
 			buffer.WriteString(" )")
 		}
 	}

pkg/ui/workspaces/cluster-ui/src/statementsPage/statementsPage.selectors.ts

@@ -62,27 +62,28 @@ export const selectApps = createSelector(sqlStatsSelector, sqlStatsState => {
   }
 
   let sawBlank = false;
+  let sawInternal = false;
   const apps: { [app: string]: boolean } = {};
   sqlStatsState.data.statements.forEach(
     (statement: ICollectedStatementStatistics) => {
-      const isNotInternalApp =
+      if (
         sqlStatsState.data.internal_app_name_prefix &&
-        !statement.key.key_data.app.startsWith(
+        statement.key.key_data.app.startsWith(
           sqlStatsState.data.internal_app_name_prefix,
-        );
-      if (
-        sqlStatsState.data.internal_app_name_prefix == undefined ||
-        isNotInternalApp
+        )
       ) {
-        if (statement.key.key_data.app) {
-          apps[statement.key.key_data.app] = true;
-        } else {
-          sawBlank = true;
-        }
+        sawInternal = true;
+      } else if (statement.key.key_data.app) {
+        apps[statement.key.key_data.app] = true;
+      } else {
+        sawBlank = true;
       }
     },
   );
-  return [].concat(sawBlank ? [unset] : []).concat(Object.keys(apps).sort());
+  return []
+    .concat(sawInternal ? [sqlStatsState.data.internal_app_name_prefix] : [])
+    .concat(sawBlank ? [unset] : [])
+    .concat(Object.keys(apps).sort());
 });
 
 // selectDatabases returns the array of all databases with statement statistics present

pkg/ui/workspaces/cluster-ui/src/transactionsPage/utils.ts

@@ -40,9 +40,13 @@ export const getTrxAppFilterOptions = (
   prefix: string,
 ): string[] => {
   const uniqueAppNames = new Set(
-    transactions
-      .filter(t => !t.stats_data.app.startsWith(prefix))
-      .map(t => (t.stats_data.app ? t.stats_data.app : unset)),
+    transactions.map(t =>
+      t.stats_data.app
+        ? t.stats_data.app.startsWith(prefix)
+          ? prefix
+          : t.stats_data.app
+        : unset,
+    ),
   );
 
   return Array.from(uniqueAppNames).sort();

pkg/ui/workspaces/db-console/src/views/statements/statementsPage.tsx

@@ -168,27 +168,26 @@ export const selectApps = createSelector(
     }
 
     let sawBlank = false;
+    let sawInternal = false;
     const apps: { [app: string]: boolean } = {};
     state.data.statements.forEach(
       (statement: ICollectedStatementStatistics) => {
-        const isNotInternalApp =
+        if (
           state.data.internal_app_name_prefix &&
-          !statement.key.key_data.app.startsWith(
+          statement.key.key_data.app.startsWith(
             state.data.internal_app_name_prefix,
-          );
-        if (
-          state.data.internal_app_name_prefix == undefined ||
-          isNotInternalApp
+          )
         ) {
-          if (statement.key.key_data.app) {
-            apps[statement.key.key_data.app] = true;
-          } else {
-            sawBlank = true;
-          }
+          sawInternal = true;
+        } else if (statement.key.key_data.app) {
+          apps[statement.key.key_data.app] = true;
+        } else {
+          sawBlank = true;
         }
       },
     );
     return []
+      .concat(sawInternal ? [state.data.internal_app_name_prefix] : [])
       .concat(sawBlank ? [unset] : [])
       .concat(Object.keys(apps))
       .sort();