Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sql: add new index_recommendation column #84618

Merged
merged 1 commit into from
Jul 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/generated/settings/settings-for-tenants.txt
Original file line number Diff line number Diff line change
Expand Up @@ -283,4 +283,4 @@ trace.jaeger.agent string the address of a Jaeger agent to receive traces using
trace.opentelemetry.collector string address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.
trace.span_registry.enabled boolean true if set, ongoing traces can be seen at https://<ui>/#/debug/tracez
trace.zipkin.collector string the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.
version version 22.1-30 set the active cluster version in the format '<major>.<minor>'
version version 22.1-32 set the active cluster version in the format '<major>.<minor>'
2 changes: 1 addition & 1 deletion docs/generated/settings/settings.html
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,6 @@
<tr><td><code>trace.opentelemetry.collector</code></td><td>string</td><td><code></code></td><td>address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.</td></tr>
<tr><td><code>trace.span_registry.enabled</code></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://<ui>/#/debug/tracez</td></tr>
<tr><td><code>trace.zipkin.collector</code></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.</td></tr>
<tr><td><code>version</code></td><td>version</td><td><code>22.1-30</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
<tr><td><code>version</code></td><td>version</td><td><code>22.1-32</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
</tbody>
</table>
4 changes: 2 additions & 2 deletions pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,10 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
----
node_id table_id name parent_id expiration deleted

query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTT colnames
query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTTT colnames
SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
----
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id index_recommendations

query ITTTIIRRRRRRRRRRRRRRRRRR colnames
SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
Expand Down
7 changes: 7 additions & 0 deletions pkg/clusterversion/cockroach_versions.go
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,9 @@ const (
AlterSystemSQLInstancesAddLocality
// SystemExternalConnectionsTable adds system.external_connections table.
SystemExternalConnectionsTable
// AlterSystemStatementStatisticsAddIndexRecommendations adds an
// index_recommendations column to the system.statement_statistics table.
AlterSystemStatementStatisticsAddIndexRecommendations

// *************************************************
// Step (1): Add new versions here.
Expand Down Expand Up @@ -572,6 +575,10 @@ var versionsSingleton = keyedVersions{
Key: SystemExternalConnectionsTable,
Version: roachpb.Version{Major: 22, Minor: 1, Internal: 30},
},
{
Key: AlterSystemStatementStatisticsAddIndexRecommendations,
Version: roachpb.Version{Major: 22, Minor: 1, Internal: 32},
},

// *************************************************
// Step (2): Add new versions here.
Expand Down
5 changes: 3 additions & 2 deletions pkg/clusterversion/key_string.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/roachpb/app_stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ func (s *StatementStatistics) Add(other *StatementStatistics) {
s.RowsWritten.Add(other.RowsWritten, s.Count, other.Count)
s.Nodes = util.CombineUniqueInt64(s.Nodes, other.Nodes)
s.PlanGists = util.CombineUniqueString(s.PlanGists, other.PlanGists)
s.IndexRecommendations = util.CombineUniqueString(s.IndexRecommendations, other.IndexRecommendations)

s.ExecStats.Add(other.ExecStats)

Expand Down
5 changes: 4 additions & 1 deletion pkg/roachpb/app_stats.proto
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,15 @@ message StatementStatistics {
// Nodes is the ordered list of nodes ids on which the statement was executed.
repeated int64 nodes = 24;

// plan_gists is list of a compressed version of plan that can be converted (lossily)
// plan_gists is the list of a compressed version of plan that can be converted (lossily)
// back into a logical plan.
// Each statement contain only one plan gist, but the same statement fingerprint id
// can contain more than one value.
repeated string plan_gists = 26;

// index_recommendations is the list of index recommendations generated for the statement fingerprint.
repeated string index_recommendations = 27;

// Note: be sure to update `sql/app_stats.go` when adding/removing fields here!

reserved 13, 14, 17, 18, 19, 20;
Expand Down
12 changes: 9 additions & 3 deletions pkg/sql/catalog/systemschema/system.go
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,8 @@ CREATE TABLE system.statement_statistics (
mod(fnv32(crdb_internal.datums_to_bytes(aggregated_ts, app_name, fingerprint_id, node_id, plan_hash, transaction_fingerprint_id)), 8:::INT8)
) STORED,
index_recommendations STRING[] NOT NULL DEFAULT (array[]::STRING[]),
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id)
USING HASH WITH (bucket_count=8),
INDEX "fingerprint_stats_idx" (fingerprint_id, transaction_fingerprint_id),
Expand All @@ -495,7 +497,8 @@ CREATE TABLE system.statement_statistics (
agg_interval,
metadata,
statistics,
plan
plan,
index_recommendations
)
)
`
Expand Down Expand Up @@ -1983,6 +1986,8 @@ var (
},
))

defaultIndexRec = "ARRAY[]:::STRING[]"

// StatementStatisticsTable is the descriptor for the SQL statement stats table.
// It stores statistics for statement fingerprints.
StatementStatisticsTable = registerSystemTable(
Expand All @@ -2009,6 +2014,7 @@ var (
ComputeExpr: &sqlStmtHashComputeExpr,
Hidden: true,
},
{Name: "index_recommendations", ID: 12, Type: types.StringArray, Nullable: false, DefaultExpr: &defaultIndexRec},
},
[]descpb.ColumnFamilyDescriptor{
{
Expand All @@ -2017,9 +2023,9 @@ var (
ColumnNames: []string{
"crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8",
"aggregated_ts", "fingerprint_id", "transaction_fingerprint_id", "plan_hash", "app_name", "node_id",
"agg_interval", "metadata", "statistics", "plan",
"agg_interval", "metadata", "statistics", "plan", "index_recommendations",
},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12},
DefaultColumnID: 0,
},
},
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/catalog/systemschema_test/testdata/bootstrap
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ CREATE TABLE public.statement_statistics (
statistics JSONB NOT NULL,
plan JSONB NOT NULL,
crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(aggregated_ts, app_name, fingerprint_id, node_id, plan_hash, transaction_fingerprint_id)), 8:::INT8)) STORED,
index_recommendations STRING[] NOT NULL DEFAULT ARRAY[]:::STRING[],
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts ASC, fingerprint_id ASC, transaction_fingerprint_id ASC, plan_hash ASC, app_name ASC, node_id ASC) USING HASH WITH (bucket_count=8),
INDEX fingerprint_stats_idx (fingerprint_id ASC, transaction_fingerprint_id ASC)
);
Expand Down
35 changes: 29 additions & 6 deletions pkg/sql/crdb_internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -1103,7 +1103,8 @@ CREATE TABLE crdb_internal.node_statement_statistics (
sample_plan JSONB,
database_name STRING NOT NULL,
exec_node_ids INT[] NOT NULL,
txn_fingerprint_id STRING
txn_fingerprint_id STRING,
index_recommendations STRING[] NOT NULL
)`,
populate: func(ctx context.Context, p *planner, _ catalog.DatabaseDescriptor, addRow func(...tree.Datum) error) error {
hasViewActivityOrViewActivityRedacted, err := p.HasViewActivityOrViewActivityRedactedRole(ctx)
Expand Down Expand Up @@ -1156,6 +1157,13 @@ CREATE TABLE crdb_internal.node_statement_statistics (

}

indexRecommendations := tree.NewDArray(types.String)
for _, recommendation := range stats.Stats.IndexRecommendations {
if err := indexRecommendations.Append(tree.NewDString(recommendation)); err != nil {
return err
}
}

err := addRow(
tree.NewDInt(tree.DInt(nodeID)), // node_id
tree.NewDString(stats.Key.App), // application_name
Expand Down Expand Up @@ -1199,6 +1207,7 @@ CREATE TABLE crdb_internal.node_statement_statistics (
tree.NewDString(stats.Key.Database), // database_name
execNodeIDs, // exec_node_ids
txnFingerprintID, // txn_fingerprint_id
indexRecommendations, // index_recommendations
)
if err != nil {
return err
Expand Down Expand Up @@ -5371,7 +5380,8 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
metadata JSONB NOT NULL,
statistics JSONB NOT NULL,
sampled_plan JSONB NOT NULL,
aggregation_interval INTERVAL NOT NULL
aggregation_interval INTERVAL NOT NULL,
index_recommendations STRING[] NOT NULL
);`,
generator: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, stopper *stop.Stopper) (virtualTableGenerator, cleanupFunc, error) {
// TODO(azhng): we want to eventually implement memory accounting within the
Expand Down Expand Up @@ -5402,7 +5412,7 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
curAggTs := s.ComputeAggregatedTs()
aggInterval := s.GetAggregationInterval()

row := make(tree.Datums, 8 /* number of columns for this virtual table */)
row := make(tree.Datums, 9 /* number of columns for this virtual table */)
worker := func(ctx context.Context, pusher rowPusher) error {
return memSQLStats.IterateStatementStats(ctx, &sqlstats.IteratorOptions{
SortedAppNames: true,
Expand Down Expand Up @@ -5437,6 +5447,13 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
duration.MakeDuration(aggInterval.Nanoseconds(), 0, 0),
types.DefaultIntervalTypeMetadata)

indexRecommendations := tree.NewDArray(types.String)
for _, recommendation := range statistics.Stats.IndexRecommendations {
if err := indexRecommendations.Append(tree.NewDString(recommendation)); err != nil {
return err
}
}

row = row[:0]
row = append(row,
aggregatedTs, // aggregated_ts
Expand All @@ -5448,6 +5465,7 @@ CREATE TABLE crdb_internal.cluster_statement_statistics (
tree.NewDJSON(statisticsJSON), // statistics
tree.NewDJSON(plan), // plan
aggInterval, // aggregation_interval
indexRecommendations, // index_recommendations
)

return pusher.pushRow(row...)
Expand All @@ -5473,7 +5491,8 @@ SELECT
max(metadata) as metadata,
crdb_internal.merge_statement_stats(array_agg(statistics)),
max(sampled_plan),
aggregation_interval
aggregation_interval,
array_remove(array_agg(index_rec), NULL) AS index_recommendations
FROM (
SELECT
aggregated_ts,
Expand All @@ -5484,7 +5503,8 @@ FROM (
metadata,
statistics,
sampled_plan,
aggregation_interval
aggregation_interval,
index_recommendations
FROM
crdb_internal.cluster_statement_statistics
UNION ALL
Expand All @@ -5497,10 +5517,12 @@ FROM (
metadata,
statistics,
plan,
agg_interval
agg_interval,
index_recommendations
FROM
system.statement_statistics
)
LEFT JOIN LATERAL unnest(index_recommendations) AS index_rec ON true
GROUP BY
aggregated_ts,
fingerprint_id,
Expand All @@ -5518,6 +5540,7 @@ GROUP BY
{Name: "statistics", Typ: types.Jsonb},
{Name: "sampled_plan", Typ: types.Jsonb},
{Name: "aggregation_interval", Typ: types.Interval},
{Name: "index_recommendations", Typ: types.StringArray},
},
}

Expand Down
35 changes: 18 additions & 17 deletions pkg/sql/executor_statement_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,23 +168,24 @@ func (ex *connExecutor) recordStatementSummary(
}

recordedStmtStats := sqlstats.RecordedStmtStats{
SessionID: ex.sessionID,
StatementID: planner.stmt.QueryID,
AutoRetryCount: automaticRetryCount,
RowsAffected: rowsAffected,
ParseLatency: parseLat,
PlanLatency: planLat,
RunLatency: runLat,
ServiceLatency: svcLat,
OverheadLatency: execOverhead,
BytesRead: stats.bytesRead,
RowsRead: stats.rowsRead,
RowsWritten: stats.rowsWritten,
Nodes: getNodesFromPlanner(planner),
StatementType: stmt.AST.StatementType(),
Plan: planner.instrumentation.PlanForStats(ctx),
PlanGist: planner.instrumentation.planGist.String(),
StatementError: stmtErr,
SessionID: ex.sessionID,
StatementID: planner.stmt.QueryID,
AutoRetryCount: automaticRetryCount,
RowsAffected: rowsAffected,
ParseLatency: parseLat,
PlanLatency: planLat,
RunLatency: runLat,
ServiceLatency: svcLat,
OverheadLatency: execOverhead,
BytesRead: stats.bytesRead,
RowsRead: stats.rowsRead,
RowsWritten: stats.rowsWritten,
Nodes: getNodesFromPlanner(planner),
StatementType: stmt.AST.StatementType(),
Plan: planner.instrumentation.PlanForStats(ctx),
PlanGist: planner.instrumentation.planGist.String(),
StatementError: stmtErr,
IndexRecommendations: planner.instrumentation.indexRecommendations,
}

stmtFingerprintID, err :=
Expand Down
4 changes: 2 additions & 2 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal
Original file line number Diff line number Diff line change
Expand Up @@ -315,10 +315,10 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
----
node_id table_id name parent_id expiration deleted

query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTT colnames
query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBBTTTTT colnames
SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
----
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id
node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan sample_plan database_name exec_node_ids txn_fingerprint_id index_recommendations

query ITTTIIRRRRRRRRRRRRRRRRRR colnames
SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
Expand Down
Loading