server,ui: show internal stats with new cluster setting

docs/generated/settings/settings-for-tenants.txt

@@ -269,6 +269,7 @@ sql.stats.non_default_columns.min_retention_period	duration	24h0m0s	minimum rete
 sql.stats.persisted_rows.max	integer	1000000	maximum number of rows of statement and transaction statistics that will be persisted in the system tables
 sql.stats.post_events.enabled	boolean	false	if set, an event is logged for every CREATE STATISTICS job
 sql.stats.response.max	integer	20000	the maximum number of statements and transaction stats returned in a CombinedStatements request
+sql.stats.response.show_internal.enabled	boolean	false	controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity
 sql.stats.system_tables.enabled	boolean	true	when true, enables use of statistics on system tables by the query optimizer
 sql.stats.system_tables_autostats.enabled	boolean	true	when true, enables automatic collection of statistics on system tables
 sql.telemetry.query_sampling.enabled	boolean	false	when set to true, executed queries will emit an event on the telemetry logging channel

docs/generated/settings/settings.html

@@ -200,6 +200,7 @@
 <tr><td><code>sql.stats.persisted_rows.max</code></td><td>integer</td><td><code>1000000</code></td><td>maximum number of rows of statement and transaction statistics that will be persisted in the system tables</td></tr>
 <tr><td><code>sql.stats.post_events.enabled</code></td><td>boolean</td><td><code>false</code></td><td>if set, an event is logged for every CREATE STATISTICS job</td></tr>
 <tr><td><code>sql.stats.response.max</code></td><td>integer</td><td><code>20000</code></td><td>the maximum number of statements and transaction stats returned in a CombinedStatements request</td></tr>
+<tr><td><code>sql.stats.response.show_internal.enabled</code></td><td>boolean</td><td><code>false</code></td><td>controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity</td></tr>
 <tr><td><code>sql.stats.system_tables.enabled</code></td><td>boolean</td><td><code>true</code></td><td>when true, enables use of statistics on system tables by the query optimizer</td></tr>
 <tr><td><code>sql.stats.system_tables_autostats.enabled</code></td><td>boolean</td><td><code>true</code></td><td>when true, enables automatic collection of statistics on system tables</td></tr>
 <tr><td><code>sql.telemetry.query_sampling.enabled</code></td><td>boolean</td><td><code>false</code></td><td>when set to true, executed queries will emit an event on the telemetry logging channel</td></tr>

pkg/server/cluster_settings.go

@@ -21,3 +21,13 @@ var SQLStatsResponseMax = settings.RegisterIntSetting(
 	20000,
 	settings.NonNegativeInt,
 ).WithPublic()
+
+// SQLStatsShowInternal controls if statistics for internal executions should be returned by the
+// CombinedStatements endpoint.
+var SQLStatsShowInternal = settings.RegisterBoolSetting(
+	settings.TenantWritable,
+	"sql.stats.response.show_internal.enabled",
+	"controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This "+
+		"endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity",
+	false,
+).WithPublic()

pkg/server/combined_statement_stats.go

@@ -70,7 +70,9 @@ func getCombinedStatementStats(
 	startTime := getTimeFromSeconds(req.Start)
 	endTime := getTimeFromSeconds(req.End)
 	limit := SQLStatsResponseMax.Get(&settings.SV)
-	whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs(startTime, endTime, limit, testingKnobs)
+	showInternal := SQLStatsShowInternal.Get(&settings.SV)
+	whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs(
+		startTime, endTime, limit, testingKnobs, showInternal)
 	statements, err := collectCombinedStatements(ctx, ie, whereClause, args, orderAndLimit)
 	if err != nil {
 		return nil, serverError(ctx, err)
@@ -98,13 +100,17 @@ func getCombinedStatementStats(
 // The whereClause will be in the format `WHERE A = $1 AND B = $2` and
 // args will return the list of arguments in order that will replace the actual values.
 func getCombinedStatementsQueryClausesAndArgs(
-	start, end *time.Time, limit int64, testingKnobs *sqlstats.TestingKnobs,
+	start, end *time.Time, limit int64, testingKnobs *sqlstats.TestingKnobs, showInternal bool,
 ) (whereClause string, orderAndLimitClause string, args []interface{}) {
 	var buffer strings.Builder
 	buffer.WriteString(testingKnobs.GetAOSTClause())
 
-	// Filter out internal statements by app name.
-	buffer.WriteString(fmt.Sprintf(" WHERE app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	if showInternal {
+		buffer.WriteString(" WHERE true")
+	} else {
+		// Filter out internal statements by app name.
+		buffer.WriteString(fmt.Sprintf(" WHERE app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	}
 
 	if start != nil {
 		buffer.WriteString(" AND aggregated_ts >= $1")
@@ -357,7 +363,8 @@ func getStatementDetails(
 	testingKnobs *sqlstats.TestingKnobs,
 ) (*serverpb.StatementDetailsResponse, error) {
 	limit := SQLStatsResponseMax.Get(&settings.SV)
-	whereClause, args, err := getStatementDetailsQueryClausesAndArgs(req, testingKnobs)
+	showInternal := SQLStatsShowInternal.Get(&settings.SV)
+	whereClause, args, err := getStatementDetailsQueryClausesAndArgs(req, testingKnobs, showInternal)
 	if err != nil {
 		return nil, serverError(ctx, err)
 	}
@@ -407,7 +414,7 @@ func getStatementDetails(
 // The whereClause will be in the format `WHERE A = $1 AND B = $2` and
 // args will return the list of arguments in order that will replace the actual values.
 func getStatementDetailsQueryClausesAndArgs(
-	req *serverpb.StatementDetailsRequest, testingKnobs *sqlstats.TestingKnobs,
+	req *serverpb.StatementDetailsRequest, testingKnobs *sqlstats.TestingKnobs, showInternal bool,
 ) (whereClause string, args []interface{}, err error) {
 	var buffer strings.Builder
 	buffer.WriteString(testingKnobs.GetAOSTClause())
@@ -420,18 +427,24 @@ func getStatementDetailsQueryClausesAndArgs(
 	args = append(args, sqlstatsutil.EncodeUint64ToBytes(fingerprintID))
 	buffer.WriteString(fmt.Sprintf(" WHERE fingerprint_id = $%d", len(args)))
 
-	// Filter out internal statements by app name.
-	buffer.WriteString(fmt.Sprintf(" AND app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	if !showInternal {
+		// Filter out internal statements by app name.
+		buffer.WriteString(fmt.Sprintf(" AND app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+	}
 
 	// Statements are grouped ignoring the app name in the Statements/Transactions page, so when
 	// calling for the Statement Details endpoint, this value can be empty or a list of app names.
 	if len(req.AppNames) > 0 {
 		if !(len(req.AppNames) == 1 && req.AppNames[0] == "") {
+			hasInternal := false
 			buffer.WriteString(" AND (")
 			for i, app := range req.AppNames {
 				if app == "(unset)" {
 					app = ""
 				}
+				if strings.Contains(app, catconstants.InternalAppNamePrefix) {
+					hasInternal = true
+				}
 				if i != 0 {
 					args = append(args, app)
 					buffer.WriteString(fmt.Sprintf(" OR app_name = $%d", len(args)))
@@ -440,6 +453,9 @@ func getStatementDetailsQueryClausesAndArgs(
 					buffer.WriteString(fmt.Sprintf(" app_name = $%d", len(args)))
 				}
 			}
+			if hasInternal {
+				buffer.WriteString(fmt.Sprintf(" OR app_name LIKE '%s%%'", catconstants.InternalAppNamePrefix))
+			}
 			buffer.WriteString(" )")
 		}
 	}

pkg/ui/workspaces/cluster-ui/src/statementsPage/statementsPage.selectors.ts

@@ -59,27 +59,28 @@ export const selectApps = createSelector(sqlStatsSelector, sqlStatsState => {
   }
 
   let sawBlank = false;
+  let sawInternal = false;
   const apps: { [app: string]: boolean } = {};
   sqlStatsState.data.statements.forEach(
     (statement: ICollectedStatementStatistics) => {
-      const isNotInternalApp =
+      if (
         sqlStatsState.data.internal_app_name_prefix &&
-        !statement.key.key_data.app.startsWith(
+        statement.key.key_data.app.startsWith(
           sqlStatsState.data.internal_app_name_prefix,
-        );
-      if (
-        sqlStatsState.data.internal_app_name_prefix == undefined ||
-        isNotInternalApp
+        )
       ) {
-        if (statement.key.key_data.app) {
-          apps[statement.key.key_data.app] = true;
-        } else {
-          sawBlank = true;
-        }
+        sawInternal = true;
+      } else if (statement.key.key_data.app) {
+        apps[statement.key.key_data.app] = true;
+      } else {
+        sawBlank = true;
       }
     },
   );
-  return [].concat(sawBlank ? [unset] : []).concat(Object.keys(apps).sort());
+  return []
+    .concat(sawInternal ? [sqlStatsState.data.internal_app_name_prefix] : [])
+    .concat(sawBlank ? [unset] : [])
+    .concat(Object.keys(apps).sort());
 });
 
 // selectDatabases returns the array of all databases with statement statistics present

pkg/ui/workspaces/cluster-ui/src/transactionsPage/utils.ts

@@ -42,9 +42,13 @@ export const getTrxAppFilterOptions = (
   prefix: string,
 ): string[] => {
   const uniqueAppNames = new Set(
-    transactions
-      .filter(t => !t.stats_data.app.startsWith(prefix))
-      .map(t => (t.stats_data.app ? t.stats_data.app : unset)),
+    transactions.map(t =>
+      t.stats_data.app
+        ? t.stats_data.app.startsWith(prefix)
+          ? prefix
+          : t.stats_data.app
+        : unset,
+    ),
   );
 
   return Array.from(uniqueAppNames).sort();

pkg/ui/workspaces/db-console/src/views/statements/statementsPage.tsx

@@ -181,27 +181,26 @@ export const selectApps = createSelector(
     }
 
     let sawBlank = false;
+    let sawInternal = false;
     const apps: { [app: string]: boolean } = {};
     state.data.statements.forEach(
       (statement: ICollectedStatementStatistics) => {
-        const isNotInternalApp =
+        if (
           state.data.internal_app_name_prefix &&
-          !statement.key.key_data.app.startsWith(
+          statement.key.key_data.app.startsWith(
             state.data.internal_app_name_prefix,
-          );
-        if (
-          state.data.internal_app_name_prefix == undefined ||
-          isNotInternalApp
+          )
         ) {
-          if (statement.key.key_data.app) {
-            apps[statement.key.key_data.app] = true;
-          } else {
-            sawBlank = true;
-          }
+          sawInternal = true;
+        } else if (statement.key.key_data.app) {
+          apps[statement.key.key_data.app] = true;
+        } else {
+          sawBlank = true;
         }
       },
     );
     return []
+      .concat(sawInternal ? [state.data.internal_app_name_prefix] : [])
       .concat(sawBlank ? [unset] : [])
       .concat(Object.keys(apps))
       .sort();