Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release-22.2: sql: add crdb_internal.statement_statisticsV22_1 for upgrades #96454

Merged
merged 1 commit into from
Feb 7, 2023
Merged

release-22.2: sql: add crdb_internal.statement_statisticsV22_1 for upgrades #96454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ crdb_internal schema_changes table admin NULL NULL
crdb_internal session_trace table admin NULL NULL
crdb_internal session_variables table admin NULL NULL
crdb_internal statement_statistics view admin NULL NULL
crdb_internal statement_statistics_v22_1 view admin NULL NULL
crdb_internal super_regions table admin NULL NULL
crdb_internal table_columns table admin NULL NULL
crdb_internal table_indexes table admin NULL NULL
Expand Down
1 change: 1 addition & 0 deletions pkg/cli/zip_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ table_name NOT IN (
'cluster_statement_statistics',
'cluster_transaction_statistics',
'statement_statistics',
'statement_statistics_v22_1',
'transaction_statistics',
'tenant_usage_details',
'pg_catalog_table_is_implemented'
Expand Down
60 changes: 42 additions & 18 deletions pkg/server/combined_statement_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,11 @@ func getCombinedStatementStats(
endTime := getTimeFromSeconds(req.End)
limit := SQLStatsResponseMax.Get(&settings.SV)
showInternal := SQLStatsShowInternal.Get(&settings.SV)

whereClause, orderAndLimit, args := getCombinedStatementsQueryClausesAndArgs(
startTime, endTime, limit, testingKnobs, showInternal)
statements, err := collectCombinedStatements(ctx, ie, whereClause, args, orderAndLimit)

statements, err := collectCombinedStatements(ctx, ie, whereClause, args, orderAndLimit, settings)
if err != nil {
return nil, serverError(ctx, err)
}
Expand Down Expand Up @@ -138,8 +140,12 @@ func collectCombinedStatements(
whereClause string,
args []interface{},
orderAndLimit string,
settings *cluster.Settings,
) ([]serverpb.StatementsResponse_CollectedStatementStatistics, error) {

table := "crdb_internal.statement_statistics"
if !settings.Version.IsActive(ctx, clusterversion.AlterSystemStatementStatisticsAddIndexRecommendations) {
table = "crdb_internal.statement_statistics_v22_1"
}
query := fmt.Sprintf(
`SELECT
fingerprint_id,
Expand All @@ -150,14 +156,14 @@ func collectCombinedStatements(
crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics,
max(sampled_plan) AS sampled_plan,
aggregation_interval
FROM crdb_internal.statement_statistics %s
FROM %s %s
GROUP BY
fingerprint_id,
transaction_fingerprint_id,
app_name,
metadata,
aggregation_interval
%s`, whereClause, orderAndLimit)
%s`, table, whereClause, orderAndLimit)

const expectedNumDatums = 8

Expand Down Expand Up @@ -374,15 +380,26 @@ func getStatementDetails(
return nil, serverError(ctx, err)
}

statementTotal, err := getTotalStatementDetails(ctx, ie, whereClause, args)
withIndexRecs := settings.Version.IsActive(ctx, clusterversion.AlterSystemStatementStatisticsAddIndexRecommendations)
stmtsTable := "crdb_internal.statement_statistics"

if !withIndexRecs {
stmtsTable = "crdb_internal.statement_statistics_v22_1"
}

// (xinhaoz) I don't think this propagation of the table name is ideal, but
// since this is only for the 22.2 release branch it's not really worth the
// effort to refactor these calls.
statementTotal, err := getTotalStatementDetails(ctx, ie, whereClause, args, stmtsTable)
if err != nil {
return nil, serverError(ctx, err)
}
statementStatisticsPerAggregatedTs, err := getStatementDetailsPerAggregatedTs(ctx, ie, whereClause, args, limit)
statementStatisticsPerAggregatedTs, err :=
getStatementDetailsPerAggregatedTs(ctx, ie, whereClause, args, limit, stmtsTable)
if err != nil {
return nil, serverError(ctx, err)
}
statementStatisticsPerPlanHash, err := getStatementDetailsPerPlanHash(ctx, ie, whereClause, args, limit, settings)
statementStatisticsPerPlanHash, err := getStatementDetailsPerPlanHash(ctx, ie, whereClause, args, limit, withIndexRecs)
if err != nil {
return nil, serverError(ctx, err)
}
Expand Down Expand Up @@ -482,7 +499,11 @@ func getStatementDetailsQueryClausesAndArgs(

// getTotalStatementDetails return all the statistics for the selected statement combined.
func getTotalStatementDetails(
ctx context.Context, ie *sql.InternalExecutor, whereClause string, args []interface{},
ctx context.Context,
ie *sql.InternalExecutor,
whereClause string,
args []interface{},
table string,
) (serverpb.StatementDetailsResponse_CollectedStatementSummary, error) {
query := fmt.Sprintf(
`SELECT
Expand All @@ -492,11 +513,11 @@ func getTotalStatementDetails(
crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics,
max(sampled_plan) as sampled_plan,
encode(fingerprint_id, 'hex') as fingerprint_id
FROM crdb_internal.statement_statistics %s
FROM %s %s
GROUP BY
aggregation_interval,
fingerprint_id
LIMIT 1`, whereClause)
LIMIT 1`, table, whereClause)

const expectedNumDatums = 6
var statement serverpb.StatementDetailsResponse_CollectedStatementSummary
Expand Down Expand Up @@ -574,6 +595,7 @@ func getStatementDetailsPerAggregatedTs(
whereClause string,
args []interface{},
limit int64,
table string,
) ([]serverpb.StatementDetailsResponse_CollectedStatementGroupedByAggregatedTs, error) {
query := fmt.Sprintf(
`SELECT
Expand All @@ -582,12 +604,12 @@ func getStatementDetailsPerAggregatedTs(
crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics,
max(sampled_plan) as sampled_plan,
aggregation_interval
FROM crdb_internal.statement_statistics %s
FROM %s %s
GROUP BY
aggregated_ts,
aggregation_interval
ORDER BY aggregated_ts ASC
LIMIT $%d`, whereClause, len(args)+1)
LIMIT $%d`, table, whereClause, len(args)+1)

args = append(args, limit)
const expectedNumDatums = 5
Expand Down Expand Up @@ -702,7 +724,7 @@ func getStatementDetailsPerPlanHash(
whereClause string,
args []interface{},
limit int64,
settings *cluster.Settings,
withIndexRecs bool,
) ([]serverpb.StatementDetailsResponse_CollectedStatementGroupedByPlanHash, error) {

query := fmt.Sprintf(
Expand All @@ -713,15 +735,15 @@ func getStatementDetailsPerPlanHash(
crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics,
max(sampled_plan) as sampled_plan,
aggregation_interval
FROM crdb_internal.statement_statistics %s
FROM crdb_internal.statement_statistics_v22_1 %s
GROUP BY
plan_hash,
plan_gist,
aggregation_interval
LIMIT $%d`, whereClause, len(args)+1)
expectedNumDatums := 6

if settings.Version.IsActive(ctx, clusterversion.AlterSystemStatementStatisticsAddIndexRecommendations) {
if withIndexRecs {
query = fmt.Sprintf(
`SELECT
plan_hash,
Expand Down Expand Up @@ -801,10 +823,12 @@ func getStatementDetailsPerPlanHash(
metadata.Stats.SensitiveInfo.MostRecentPlanDescription = *plan
aggInterval := tree.MustBeDInterval(row[5]).Duration

recommendations := tree.MustBeDArray(row[6])
var idxRecommendations []string
for _, s := range recommendations.Array {
idxRecommendations = util.CombineUniqueString(idxRecommendations, []string{string(tree.MustBeDString(s))})
if withIndexRecs {
recommendations := tree.MustBeDArray(row[6])
for _, s := range recommendations.Array {
idxRecommendations = util.CombineUniqueString(idxRecommendations, []string{string(tree.MustBeDString(s))})
}
}

// A metadata is unique for each plan, meaning if any of the counts are greater than zero,
Expand Down
63 changes: 63 additions & 0 deletions pkg/sql/crdb_internal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ var crdbInternal = virtualSchema{
catconstants.CrdbInternalSessionTraceTableID: crdbInternalSessionTraceTable,
catconstants.CrdbInternalSessionVariablesTableID: crdbInternalSessionVariablesTable,
catconstants.CrdbInternalStmtStatsTableID: crdbInternalStmtStatsView,
catconstants.CrdbInternalStmtStatsV22_1TableID: crdbInternalStmtStatsViewV22_1,
catconstants.CrdbInternalTableColumnsTableID: crdbInternalTableColumnsTable,
catconstants.CrdbInternalTableIndexesTableID: crdbInternalTableIndexesTable,
catconstants.CrdbInternalTablesTableLastStatsID: crdbInternalTablesTableLastStats,
Expand Down Expand Up @@ -5775,6 +5776,68 @@ GROUP BY
},
}

// Copy of crdb_internal.statement_statistics for previous versions during upgrade.
// See crdb_internal.statement_statistics for more details.
var crdbInternalStmtStatsViewV22_1 = virtualSchemaView{
schema: `
CREATE VIEW crdb_internal.statement_statistics_v22_1 AS
SELECT
aggregated_ts,
fingerprint_id,
transaction_fingerprint_id,
plan_hash,
app_name,
max(metadata) as metadata,
crdb_internal.merge_statement_stats(array_agg(DISTINCT statistics)),
max(sampled_plan),
aggregation_interval
FROM (
SELECT
aggregated_ts,
fingerprint_id,
transaction_fingerprint_id,
plan_hash,
app_name,
metadata,
statistics,
sampled_plan,
aggregation_interval
FROM
crdb_internal.cluster_statement_statistics
UNION ALL
SELECT
aggregated_ts,
fingerprint_id,
transaction_fingerprint_id,
plan_hash,
app_name,
metadata,
statistics,
plan,
agg_interval
FROM
system.statement_statistics
)
GROUP BY
aggregated_ts,
fingerprint_id,
transaction_fingerprint_id,
plan_hash,
app_name,
aggregation_interval`,
resultColumns: colinfo.ResultColumns{
{Name: "aggregated_ts", Typ: types.TimestampTZ},
{Name: "fingerprint_id", Typ: types.Bytes},
{Name: "transaction_fingerprint_id", Typ: types.Bytes},
{Name: "plan_hash", Typ: types.Bytes},
{Name: "app_name", Typ: types.String},
{Name: "metadata", Typ: types.Jsonb},
{Name: "statistics", Typ: types.Jsonb},
{Name: "sampled_plan", Typ: types.Jsonb},
{Name: "aggregation_interval", Typ: types.Interval},
},
}

var crdbInternalActiveRangeFeedsTable = virtualSchemaTable{
comment: `node-level table listing all currently running range feeds`,
// NB: startTS is exclusive; consider renaming to startAfter.
Expand Down
7 changes: 7 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ crdb_internal schema_changes table admin NULL NULL
crdb_internal session_trace table admin NULL NULL
crdb_internal session_variables table admin NULL NULL
crdb_internal statement_statistics view admin NULL NULL
crdb_internal statement_statistics_v22_1 view admin NULL NULL
crdb_internal super_regions table admin NULL NULL
crdb_internal table_columns table admin NULL NULL
crdb_internal table_indexes table admin NULL NULL
Expand Down Expand Up @@ -791,6 +792,12 @@ SELECT start_pretty, end_pretty FROM crdb_internal.ranges
WHERE split_enforced_until IS NOT NULL
AND (start_pretty LIKE '/Table/112/1%' OR start_pretty LIKE '/Table/112/2%')
----
/Table/112/1/1 /Table/112/1/2
/Table/112/1/2 /Table/112/1/3
/Table/112/1/3 /Table/112/2/1
/Table/112/2/1 /Table/112/2/2
/Table/112/2/2 /Table/112/2/3
/Table/112/2/3 /Table/112/3/1

statement ok
ALTER TABLE foo SPLIT AT VALUES (1), (2), (3)
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/logictest/testdata/logic_test/grant_table
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ test crdb_internal schema_changes public
test crdb_internal session_trace public SELECT false
test crdb_internal session_variables public SELECT false
test crdb_internal statement_statistics public SELECT false
test crdb_internal statement_statistics_v22_1 public SELECT false
test crdb_internal super_regions public SELECT false
test crdb_internal table_columns public SELECT false
test crdb_internal table_indexes public SELECT false
Expand Down
5 changes: 5 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/information_schema
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,7 @@ crdb_internal schema_changes
crdb_internal session_trace
crdb_internal session_variables
crdb_internal statement_statistics
crdb_internal statement_statistics_v22_1
crdb_internal super_regions
crdb_internal table_columns
crdb_internal table_indexes
Expand Down Expand Up @@ -781,6 +782,7 @@ schema_changes
session_trace
session_variables
statement_statistics
statement_statistics_v22_1
super_regions
table_columns
table_indexes
Expand Down Expand Up @@ -1146,6 +1148,7 @@ system crdb_internal schema_changes SYSTEM
system crdb_internal session_trace SYSTEM VIEW NO 1
system crdb_internal session_variables SYSTEM VIEW NO 1
system crdb_internal statement_statistics SYSTEM VIEW NO 1
system crdb_internal statement_statistics_v22_1 SYSTEM VIEW NO 1
system crdb_internal super_regions SYSTEM VIEW NO 1
system crdb_internal table_columns SYSTEM VIEW NO 1
system crdb_internal table_indexes SYSTEM VIEW NO 1
Expand Down Expand Up @@ -2807,6 +2810,7 @@ NULL public system crdb_internal schema_changes
NULL public system crdb_internal session_trace SELECT NO YES
NULL public system crdb_internal session_variables SELECT NO YES
NULL public system crdb_internal statement_statistics SELECT NO YES
NULL public system crdb_internal statement_statistics_v22_1 SELECT NO YES
NULL public system crdb_internal super_regions SELECT NO YES
NULL public system crdb_internal table_columns SELECT NO YES
NULL public system crdb_internal table_indexes SELECT NO YES
Expand Down Expand Up @@ -3379,6 +3383,7 @@ NULL public system crdb_internal schema_changes
NULL public system crdb_internal session_trace SELECT NO YES
NULL public system crdb_internal session_variables SELECT NO YES
NULL public system crdb_internal statement_statistics SELECT NO YES
NULL public system crdb_internal statement_statistics_v22_1 SELECT NO YES
NULL public system crdb_internal super_regions SELECT NO YES
NULL public system crdb_internal table_columns SELECT NO YES
NULL public system crdb_internal table_indexes SELECT NO YES
Expand Down
Loading