From 37508211347c042779d6451b544236d9b0f4582c Mon Sep 17 00:00:00 2001 From: Marylia Gutierrez Date: Tue, 23 Aug 2022 12:56:15 -0400 Subject: [PATCH] server,ui: show internal stats with new cluster setting Previously, we were not showing internal results on fingerprint options on SQL Activity. A new cluster setting created `sql.stats.response.show_internal` can be set to `true` and internal statistics will be displayed on SQL Activity page. Fixes #79547 Release justification: low risk, high benefit change Release note (sql change): New cluster setting `sql.stats.response.show_internal` with default value of `false` can be set to true, to display information about internal stats on SQL Activity page, with fingerprint option. --- .../settings/settings-for-tenants.txt | 1 + docs/generated/settings/settings.html | 1 + pkg/server/cluster_settings.go | 10 ++++++ pkg/server/combined_statement_stats.go | 32 ++++++++++++++----- .../statementsPage.selectors.ts | 25 ++++++++------- .../cluster-ui/src/transactionsPage/utils.ts | 10 ++++-- .../src/views/statements/statementsPage.tsx | 21 ++++++------ 7 files changed, 66 insertions(+), 34 deletions(-) diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt index 79caafabcb23..e8ae47347d94 100644 --- a/docs/generated/settings/settings-for-tenants.txt +++ b/docs/generated/settings/settings-for-tenants.txt @@ -269,6 +269,7 @@ sql.stats.non_default_columns.min_retention_period duration 24h0m0s minimum rete sql.stats.persisted_rows.max integer 1000000 maximum number of rows of statement and transaction statistics that will be persisted in the system tables sql.stats.post_events.enabled boolean false if set, an event is logged for every CREATE STATISTICS job sql.stats.response.max integer 20000 the maximum number of statements and transaction stats returned in a CombinedStatements request +sql.stats.response.show_internal.enabled boolean false controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity sql.stats.system_tables.enabled boolean true when true, enables use of statistics on system tables by the query optimizer sql.stats.system_tables_autostats.enabled boolean true when true, enables automatic collection of statistics on system tables sql.telemetry.query_sampling.enabled boolean false when set to true, executed queries will emit an event on the telemetry logging channel diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index e710183ad1cc..1b891bcab456 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -200,6 +200,7 @@ sql.stats.persisted_rows.maxinteger1000000maximum number of rows of statement and transaction statistics that will be persisted in the system tables sql.stats.post_events.enabledbooleanfalseif set, an event is logged for every CREATE STATISTICS job sql.stats.response.maxinteger20000the maximum number of statements and transaction stats returned in a CombinedStatements request +sql.stats.response.show_internal.enabledbooleanfalsecontrols if statistics for internal executions should be returned by the CombinedStatements endpoint. This endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity sql.stats.system_tables.enabledbooleantruewhen true, enables use of statistics on system tables by the query optimizer sql.stats.system_tables_autostats.enabledbooleantruewhen true, enables automatic collection of statistics on system tables sql.telemetry.query_sampling.enabledbooleanfalsewhen set to true, executed queries will emit an event on the telemetry logging channel diff --git a/pkg/server/cluster_settings.go b/pkg/server/cluster_settings.go index da7643356946..b15b119d18aa 100644 --- a/pkg/server/cluster_settings.go +++ b/pkg/server/cluster_settings.go @@ -21,3 +21,13 @@ var SQLStatsResponseMax = settings.RegisterIntSetting( 20000, settings.NonNegativeInt, ).WithPublic() + +// SQLStatsShowInternal controls if statistics for internal executions should be returned by the +// CombinedStatements endpoint. +var SQLStatsShowInternal = settings.RegisterBoolSetting( + settings.TenantWritable, + "sql.stats.response.show_internal.enabled", + "controls if statistics for internal executions should be returned by the CombinedStatements endpoint. This "+ + "endpoint is used to display statistics on the Statement and Transaction fingerprint pages under SQL Activity", + false, +).WithPublic() diff --git a/pkg/server/combined_statement_stats.go b/pkg/server/combined_statement_stats.go index 299f13422459..b50b7e3244d8 100644 --- a/pkg/server/combined_statement_stats.go +++ b/pkg/server/combined_statement_stats.go @@ -70,7 +70,9 @@ func getCombinedStatementStats( startTime := getTimeFromSeconds(req.Start) endTime := getTimeFromSeconds(req.End) limit := SQLStatsResponseMax.Get(&settings.SV) - whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs(startTime, endTime, limit, testingKnobs) + showInternal := SQLStatsShowInternal.Get(&settings.SV) + whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs( + startTime, endTime, limit, testingKnobs, showInternal) statements, err := collectCombinedStatements(ctx, ie, whereClause, args, orderAndLimit) if err != nil { return nil, serverError(ctx, err) @@ -98,13 +100,17 @@ func getCombinedStatementStats( // The whereClause will be in the format `WHERE A = $1 AND B = $2` and // args will return the list of arguments in order that will replace the actual values. func getCombinedStatementsQueryClausesAndArgs( - start, end *time.Time, limit int64, testingKnobs *sqlstats.TestingKnobs, + start, end *time.Time, limit int64, testingKnobs *sqlstats.TestingKnobs, showInternal bool, ) (whereClause string, orderAndLimitClause string, args []interface{}) { var buffer strings.Builder buffer.WriteString(testingKnobs.GetAOSTClause()) - // Filter out internal statements by app name. - buffer.WriteString(fmt.Sprintf(" WHERE app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix)) + if showInternal { + buffer.WriteString(" WHERE true") + } else { + // Filter out internal statements by app name. + buffer.WriteString(fmt.Sprintf(" WHERE app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix)) + } if start != nil { buffer.WriteString(" AND aggregated_ts >= $1") @@ -357,7 +363,8 @@ func getStatementDetails( testingKnobs *sqlstats.TestingKnobs, ) (*serverpb.StatementDetailsResponse, error) { limit := SQLStatsResponseMax.Get(&settings.SV) - whereClause, args, err := getStatementDetailsQueryClausesAndArgs(req, testingKnobs) + showInternal := SQLStatsShowInternal.Get(&settings.SV) + whereClause, args, err := getStatementDetailsQueryClausesAndArgs(req, testingKnobs, showInternal) if err != nil { return nil, serverError(ctx, err) } @@ -407,7 +414,7 @@ func getStatementDetails( // The whereClause will be in the format `WHERE A = $1 AND B = $2` and // args will return the list of arguments in order that will replace the actual values. func getStatementDetailsQueryClausesAndArgs( - req *serverpb.StatementDetailsRequest, testingKnobs *sqlstats.TestingKnobs, + req *serverpb.StatementDetailsRequest, testingKnobs *sqlstats.TestingKnobs, showInternal bool, ) (whereClause string, args []interface{}, err error) { var buffer strings.Builder buffer.WriteString(testingKnobs.GetAOSTClause()) @@ -420,18 +427,24 @@ func getStatementDetailsQueryClausesAndArgs( args = append(args, sqlstatsutil.EncodeUint64ToBytes(fingerprintID)) buffer.WriteString(fmt.Sprintf(" WHERE fingerprint_id = $%d", len(args))) - // Filter out internal statements by app name. - buffer.WriteString(fmt.Sprintf(" AND app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix)) + if !showInternal { + // Filter out internal statements by app name. + buffer.WriteString(fmt.Sprintf(" AND app_name NOT LIKE '%s%%'", catconstants.InternalAppNamePrefix)) + } // Statements are grouped ignoring the app name in the Statements/Transactions page, so when // calling for the Statement Details endpoint, this value can be empty or a list of app names. if len(req.AppNames) > 0 { if !(len(req.AppNames) == 1 && req.AppNames[0] == "") { + hasInternal := false buffer.WriteString(" AND (") for i, app := range req.AppNames { if app == "(unset)" { app = "" } + if strings.Contains(app, catconstants.InternalAppNamePrefix) { + hasInternal = true + } if i != 0 { args = append(args, app) buffer.WriteString(fmt.Sprintf(" OR app_name = $%d", len(args))) @@ -440,6 +453,9 @@ func getStatementDetailsQueryClausesAndArgs( buffer.WriteString(fmt.Sprintf(" app_name = $%d", len(args))) } } + if hasInternal { + buffer.WriteString(fmt.Sprintf(" OR app_name LIKE '%s%%'", catconstants.InternalAppNamePrefix)) + } buffer.WriteString(" )") } } diff --git a/pkg/ui/workspaces/cluster-ui/src/statementsPage/statementsPage.selectors.ts b/pkg/ui/workspaces/cluster-ui/src/statementsPage/statementsPage.selectors.ts index 510f6e9db8df..51039b1d601b 100644 --- a/pkg/ui/workspaces/cluster-ui/src/statementsPage/statementsPage.selectors.ts +++ b/pkg/ui/workspaces/cluster-ui/src/statementsPage/statementsPage.selectors.ts @@ -59,27 +59,28 @@ export const selectApps = createSelector(sqlStatsSelector, sqlStatsState => { } let sawBlank = false; + let sawInternal = false; const apps: { [app: string]: boolean } = {}; sqlStatsState.data.statements.forEach( (statement: ICollectedStatementStatistics) => { - const isNotInternalApp = + if ( sqlStatsState.data.internal_app_name_prefix && - !statement.key.key_data.app.startsWith( + statement.key.key_data.app.startsWith( sqlStatsState.data.internal_app_name_prefix, - ); - if ( - sqlStatsState.data.internal_app_name_prefix == undefined || - isNotInternalApp + ) ) { - if (statement.key.key_data.app) { - apps[statement.key.key_data.app] = true; - } else { - sawBlank = true; - } + sawInternal = true; + } else if (statement.key.key_data.app) { + apps[statement.key.key_data.app] = true; + } else { + sawBlank = true; } }, ); - return [].concat(sawBlank ? [unset] : []).concat(Object.keys(apps).sort()); + return [] + .concat(sawInternal ? [sqlStatsState.data.internal_app_name_prefix] : []) + .concat(sawBlank ? [unset] : []) + .concat(Object.keys(apps).sort()); }); // selectDatabases returns the array of all databases with statement statistics present diff --git a/pkg/ui/workspaces/cluster-ui/src/transactionsPage/utils.ts b/pkg/ui/workspaces/cluster-ui/src/transactionsPage/utils.ts index fc82280c33a7..db98af8cca22 100644 --- a/pkg/ui/workspaces/cluster-ui/src/transactionsPage/utils.ts +++ b/pkg/ui/workspaces/cluster-ui/src/transactionsPage/utils.ts @@ -42,9 +42,13 @@ export const getTrxAppFilterOptions = ( prefix: string, ): string[] => { const uniqueAppNames = new Set( - transactions - .filter(t => !t.stats_data.app.startsWith(prefix)) - .map(t => (t.stats_data.app ? t.stats_data.app : unset)), + transactions.map(t => + t.stats_data.app + ? t.stats_data.app.startsWith(prefix) + ? prefix + : t.stats_data.app + : unset, + ), ); return Array.from(uniqueAppNames).sort(); diff --git a/pkg/ui/workspaces/db-console/src/views/statements/statementsPage.tsx b/pkg/ui/workspaces/db-console/src/views/statements/statementsPage.tsx index 1e96cc8ecdb4..1ff027ffdbce 100644 --- a/pkg/ui/workspaces/db-console/src/views/statements/statementsPage.tsx +++ b/pkg/ui/workspaces/db-console/src/views/statements/statementsPage.tsx @@ -181,27 +181,26 @@ export const selectApps = createSelector( } let sawBlank = false; + let sawInternal = false; const apps: { [app: string]: boolean } = {}; state.data.statements.forEach( (statement: ICollectedStatementStatistics) => { - const isNotInternalApp = + if ( state.data.internal_app_name_prefix && - !statement.key.key_data.app.startsWith( + statement.key.key_data.app.startsWith( state.data.internal_app_name_prefix, - ); - if ( - state.data.internal_app_name_prefix == undefined || - isNotInternalApp + ) ) { - if (statement.key.key_data.app) { - apps[statement.key.key_data.app] = true; - } else { - sawBlank = true; - } + sawInternal = true; + } else if (statement.key.key_data.app) { + apps[statement.key.key_data.app] = true; + } else { + sawBlank = true; } }, ); return [] + .concat(sawInternal ? [state.data.internal_app_name_prefix] : []) .concat(sawBlank ? [unset] : []) .concat(Object.keys(apps)) .sort();