Skip to content

Commit

Permalink
sql: add new index_recommendation column
Browse files Browse the repository at this point in the history 
This commit adds a new column index_recommendations
STRING[] to:
crdb_internal.node_statement_statistics
crdb_internal.cluster_statement_statistics
system.statement_statistics
crdb_internal.statement_statistics

Part of #83782

Release note (sql change): Adding new column index_recommendations
to crdb_internal.node_statement_statistics,
crdb_internal.cluster_statement_statistics, system.statement_statistics
and crdb_internal.statement_statistics
  • Loading branch information
@maryliag
maryliag committed Jul 22, 2022
1 parent c73adb1 commit 6ca9d27
Show file tree
Hide file tree
Showing 29 changed files with 405 additions and 66 deletions.
2 changes: 1 addition & 1 deletion docs/generated/settings/settings-for-tenants.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,4 +283,4 @@ trace.jaeger.agent string the address of a Jaeger agent to receive traces using
trace.opentelemetry.collector string address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.
trace.span_registry.enabled boolean true if set, ongoing traces can be seen at https://<ui>/#/debug/tracez
trace.zipkin.collector string the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.
version version 22.1-28 set the active cluster version in the format '<major>.<minor>'
version version 22.1-30 set the active cluster version in the format '<major>.<minor>'
2 changes: 1 addition & 1 deletion docs/generated/settings/settings.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,6 @@
<tr><td><code>trace.opentelemetry.collector</code></td><td>string</td><td><code></code></td><td>address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.</td></tr>
<tr><td><code>trace.span_registry.enabled</code></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://<ui>/#/debug/tracez</td></tr>
<tr><td><code>trace.zipkin.collector</code></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.</td></tr>
<tr><td><code>version</code></td><td>version</td><td><code>22.1-28</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
<tr><td><code>version</code></td><td>version</td><td><code>22.1-30</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
</tbody>
</table>
4 changes: 2 additions & 2 deletions pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,10 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
----
node_id table_id name parent_id expiration deleted

query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTT colnames
query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTTT colnames
SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
----
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id index_recommendations

query ITTTIIRRRRRRRRRRRRRRRRRR colnames
SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
Expand Down
7 changes: 7 additions & 0 deletions pkg/clusterversion/cockroach_versions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,9 @@ const (
AlterSystemSQLInstancesAddLocality
// SystemExternalConnectionsTable adds system.external_connections table.
SystemExternalConnectionsTable
// AlterSystemStatementStatisticsAddIndexRecommendations adds an
// index_recommendations column to the system.statement_statistics table.
AlterSystemStatementStatisticsAddIndexRecommendations

// *************************************************
// Step (1): Add new versions here.
Expand Down Expand Up @@ -564,6 +567,10 @@ var versionsSingleton = keyedVersions{
Key: SystemExternalConnectionsTable,
Version: roachpb.Version{Major: 22, Minor: 1, Internal: 28},
},
{
Key: AlterSystemStatementStatisticsAddIndexRecommendations,
Version: roachpb.Version{Major: 22, Minor: 1, Internal: 30},
},

// *************************************************
// Step (2): Add new versions here.
Expand Down
5 changes: 3 additions & 2 deletions pkg/clusterversion/key_string.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/roachpb/app_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ func (s *StatementStatistics) Add(other *StatementStatistics) {
s.RowsWritten.Add(other.RowsWritten, s.Count, other.Count)
s.Nodes = util.CombineUniqueInt64(s.Nodes, other.Nodes)
s.PlanGists = util.CombineUniqueString(s.PlanGists, other.PlanGists)
s.IndexRecommendations = util.CombineUniqueString(s.IndexRecommendations, other.IndexRecommendations)

s.ExecStats.Add(other.ExecStats)

Expand Down
5 changes: 4 additions & 1 deletion pkg/roachpb/app_stats.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,15 @@ message StatementStatistics {
// Nodes is the ordered list of nodes ids on which the statement was executed.
repeated int64 nodes = 24;

// plan_gists is list of a compressed version of plan that can be converted (lossily)
// plan_gists is the list of a compressed version of plan that can be converted (lossily)
// back into a logical plan.
// Each statement contain only one plan gist, but the same statement fingerprint id
// can contain more than one value.
repeated string plan_gists = 26;

// index_recommendations is the list of index recommendations generated for the statement fingerprint.
repeated string index_recommendations = 27;

// Note: be sure to update `sql/app_stats.go` when adding/removing fields here!

reserved 13, 14, 17, 18, 19, 20;
Expand Down
10 changes: 7 additions & 3 deletions pkg/sql/catalog/systemschema/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,8 @@ CREATE TABLE system.statement_statistics (
mod(fnv32(crdb_internal.datums_to_bytes(aggregated_ts, app_name, fingerprint_id, node_id, plan_hash, transaction_fingerprint_id)), 8:::INT8)
) STORED,
index_recommendations STRING[],
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id)
USING HASH WITH (bucket_count=8),
INDEX "fingerprint_stats_idx" (fingerprint_id, transaction_fingerprint_id),
Expand All @@ -495,7 +497,8 @@ CREATE TABLE system.statement_statistics (
agg_interval,
metadata,
statistics,
plan
plan,
index_recommendations
)
)
`
Expand Down Expand Up @@ -2009,6 +2012,7 @@ var (
ComputeExpr: &sqlStmtHashComputeExpr,
Hidden: true,
},
{Name: "index_recommendations", ID: 12, Type: types.StringArray, Nullable: true},
},
[]descpb.ColumnFamilyDescriptor{
{
Expand All @@ -2017,9 +2021,9 @@ var (
ColumnNames: []string{
"crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8",
"aggregated_ts", "fingerprint_id", "transaction_fingerprint_id", "plan_hash", "app_name", "node_id",
"agg_interval", "metadata", "statistics", "plan",
"agg_interval", "metadata", "statistics", "plan", "index_recommendations",
},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12},
DefaultColumnID: 0,
},
},
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/catalog/systemschema_test/testdata/bootstrap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ CREATE TABLE public.statement_statistics (
statistics JSONB NOT NULL,
plan JSONB NOT NULL,
crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(aggregated_ts, app_name, fingerprint_id, node_id, plan_hash, transaction_fingerprint_id)), 8:::INT8)) STORED,
index_recommendations STRING[] NULL,
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts ASC, fingerprint_id ASC, transaction_fingerprint_id ASC, plan_hash ASC, app_name ASC, node_id ASC) USING HASH WITH (bucket_count=8),
INDEX fingerprint_stats_idx (fingerprint_id ASC, transaction_fingerprint_id ASC)
);
Expand Down
35 changes: 29 additions & 6 deletions pkg/sql/crdb_internal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1103,7 +1103,8 @@ CREATE TABLE crdb_internal.node_statement_statistics (
sample_plan JSONB,
database_name STRING NOT NULL,
exec_node_ids INT[] NOT NULL,
txn_fingerprint_id STRING
txn_fingerprint_id STRING,
index_recommendations STRING[]
)`,
populate: func(ctx context.Context, p *planner, _ catalog.DatabaseDescriptor, addRow func(...tree.Datum) error) error {
hasViewActivityOrViewActivityRedacted, err := p.HasViewActivityOrViewActivityRedactedRole(ctx)
Expand Down Expand Up @@ -1156,6 +1157,13 @@ CREATE TABLE crdb_internal.node_statement_statistics (

}

indexRecommendations := tree.NewDArray(types.String)
for _, recommendation := range stats.Stats.IndexRecommendations {
if err := indexRecommendations.Append(tree.NewDString(recommendation)); err != nil {
return err
}
}

err := addRow(
tree.NewDInt(tree.DInt(nodeID)), // node_id
tree.NewDString(stats.Key.App), // application_name
Expand Down Expand Up @@ -1199,6 +1207,7 @@ CREATE TABLE crdb_internal.node_statement_statistics (
tree.NewDString(stats.Key.Database), // database_name
execNodeIDs, // exec_node_ids
txnFingerprintID, // txn_fingerprint_id
indexRecommendations, // index_recommendations
)
if err != nil {
return err
Expand Down Expand Up @@ -5368,7 +5377,8 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
metadata JSONB NOT NULL,
statistics JSONB NOT NULL,
sampled_plan JSONB NOT NULL,
aggregation_interval INTERVAL NOT NULL
aggregation_interval INTERVAL NOT NULL,
index_recommendations STRING[]
);`,
generator: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, stopper *stop.Stopper) (virtualTableGenerator, cleanupFunc, error) {
// TODO(azhng): we want to eventually implement memory accounting within the
Expand Down Expand Up @@ -5399,7 +5409,7 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
curAggTs := s.ComputeAggregatedTs()
aggInterval := s.GetAggregationInterval()

row := make(tree.Datums, 8 /* number of columns for this virtual table */)
row := make(tree.Datums, 9 /* number of columns for this virtual table */)
worker := func(ctx context.Context, pusher rowPusher) error {
return memSQLStats.IterateStatementStats(ctx, &sqlstats.IteratorOptions{
SortedAppNames: true,
Expand Down Expand Up @@ -5434,6 +5444,13 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
duration.MakeDuration(aggInterval.Nanoseconds(), 0, 0),
types.DefaultIntervalTypeMetadata)

indexRecommendations := tree.NewDArray(types.String)
for _, recommendation := range statistics.Stats.IndexRecommendations {
if err := indexRecommendations.Append(tree.NewDString(recommendation)); err != nil {
return err
}
}

row = row[:0]
row = append(row,
aggregatedTs, // aggregated_ts
Expand All @@ -5445,6 +5462,7 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
tree.NewDJSON(statisticsJSON), // statistics
tree.NewDJSON(plan), // plan
aggInterval, // aggregation_interval
indexRecommendations, // index_recommendations
)

return pusher.pushRow(row...)
Expand All @@ -5470,7 +5488,8 @@ SELECT
max(metadata) as metadata,
crdb_internal.merge_statement_stats(array_agg(statistics)),
max(sampled_plan),
aggregation_interval
aggregation_interval,
array_remove(array_agg(index_rec), NULL) AS index_recommendations
FROM (
SELECT
aggregated_ts,
Expand All @@ -5481,7 +5500,8 @@ FROM (
metadata,
statistics,
sampled_plan,
aggregation_interval
aggregation_interval,
index_recommendations
FROM
crdb_internal.cluster_statement_statistics
UNION ALL
Expand All @@ -5494,10 +5514,12 @@ FROM (
metadata,
statistics,
plan,
agg_interval
agg_interval,
index_recommendations
FROM
system.statement_statistics
)
LEFT JOIN LATERAL unnest(index_recommendations) AS index_rec ON true
GROUP BY
aggregated_ts,
fingerprint_id,
Expand All @@ -5515,6 +5537,7 @@ GROUP BY
{Name: "statistics", Typ: types.Jsonb},
{Name: "sampled_plan", Typ: types.Jsonb},
{Name: "aggregation_interval", Typ: types.Interval},
{Name: "index_recommendations", Typ: types.StringArray},
},
}

Expand Down
35 changes: 18 additions & 17 deletions pkg/sql/executor_statement_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,23 +168,24 @@ func (ex *connExecutor) recordStatementSummary(
}

recordedStmtStats := sqlstats.RecordedStmtStats{
SessionID: ex.sessionID,
StatementID: planner.stmt.QueryID,
AutoRetryCount: automaticRetryCount,
RowsAffected: rowsAffected,
ParseLatency: parseLat,
PlanLatency: planLat,
RunLatency: runLat,
ServiceLatency: svcLat,
OverheadLatency: execOverhead,
BytesRead: stats.bytesRead,
RowsRead: stats.rowsRead,
RowsWritten: stats.rowsWritten,
Nodes: getNodesFromPlanner(planner),
StatementType: stmt.AST.StatementType(),
Plan: planner.instrumentation.PlanForStats(ctx),
PlanGist: planner.instrumentation.planGist.String(),
StatementError: stmtErr,
SessionID: ex.sessionID,
StatementID: planner.stmt.QueryID,
AutoRetryCount: automaticRetryCount,
RowsAffected: rowsAffected,
ParseLatency: parseLat,
PlanLatency: planLat,
RunLatency: runLat,
ServiceLatency: svcLat,
OverheadLatency: execOverhead,
BytesRead: stats.bytesRead,
RowsRead: stats.rowsRead,
RowsWritten: stats.rowsWritten,
Nodes: getNodesFromPlanner(planner),
StatementType: stmt.AST.StatementType(),
Plan: planner.instrumentation.PlanForStats(ctx),
PlanGist: planner.instrumentation.planGist.String(),
StatementError: stmtErr,
IndexRecommendations: planner.instrumentation.indexRecommendations,
}

stmtFingerprintID, err :=
Expand Down
4 changes: 2 additions & 2 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,10 +315,10 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
----
node_id table_id name parent_id expiration deleted

query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTT colnames
query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTTT colnames
SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
----
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id index_recommendations

query ITTTIIRRRRRRRRRRRRRRRRRR colnames
SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
Expand Down
Loading

0 comments on commit 6ca9d27

Please sign in to comment.