Skip to content

Commit

Permalink
sql: add p99 computed column and index to system.statement_statistics…
Browse files Browse the repository at this point in the history 
…, system.transaction_statistics

Part of cockroachdb#98624.
Follows cockroachdb#99042.

This commit adds a new computed column (p99_latency) and partial
index (p99_latency_idx) to the system.statement_statistics and
system.transaction_statistics tables.

The commit also increases the hard-coded fingerprint execution
count for the flush benchmark to the maximum for in-memory
statement stats that trigger a flush.

Epic: none

Release note: None
  • Loading branch information
@ericharmeling
ericharmeling committed Mar 21, 2023
1 parent b26b338 commit 6e4ca2d
Show file tree
Hide file tree
Showing 10 changed files with 477 additions and 195 deletions.
12 changes: 6 additions & 6 deletions pkg/sql/catalog/bootstrap/testdata/testdata
View file

Large diffs are not rendered by default.

61 changes: 56 additions & 5 deletions pkg/sql/catalog/systemschema/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ CREATE SEQUENCE system.role_id_seq START 100 MINVALUE 100 MAXVALUE 2147483647;`
cpuSqlNanosComputeExpr = `(((statistics->'execution_statistics':::STRING)->'cpuSQLNanos':::STRING)->'mean':::STRING)::FLOAT8`
contentionTimeComputeExpr = `(((statistics->'execution_statistics':::STRING)->'contentionTime':::STRING)->'mean':::STRING)::FLOAT8`
totalEstimatedExecutionTimeExpr = `((statistics->'statistics':::STRING)->'cnt':::STRING)::FLOAT8 * (((statistics->'statistics':::STRING)->'svcLat':::STRING)->>'mean':::STRING)::FLOAT8`
p99LatencyComputeExpr = `(((statistics->'statistics':::STRING)->'latencyInfo':::STRING)->'p99':::STRING)::FLOAT8`
)

var indexUsageComputeExprStr = indexUsageComputeExpr
Expand All @@ -134,6 +135,7 @@ var serviceLatencyComputeExprStr = serviceLatencyComputeExpr
var cpuSqlNanosComputeExprStr = cpuSqlNanosComputeExpr
var contentionTimeComputeExprStr = contentionTimeComputeExpr
var totalEstimatedExecutionTimeExprStr = totalEstimatedExecutionTimeExpr
var p99LatencyComputeExprStr = p99LatencyComputeExpr

// These system tables are not part of the system config.
const (
Expand Down Expand Up @@ -543,6 +545,7 @@ CREATE TABLE system.statement_statistics (
cpu_sql_nanos FLOAT AS (` + cpuSqlNanosComputeExpr + `) STORED,
contention_time FLOAT AS (` + contentionTimeComputeExpr + `) STORED,
total_estimated_execution_time FLOAT AS (` + totalEstimatedExecutionTimeExpr + `) STORED,
p99_latency FLOAT8 AS (` + p99LatencyComputeExpr + `) STORED,
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id)
USING HASH WITH (bucket_count=8),
Expand All @@ -553,6 +556,7 @@ CREATE TABLE system.statement_statistics (
INDEX "cpu_sql_nanos_idx" (aggregated_ts, app_name, cpu_sql_nanos DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "contention_time_idx" (aggregated_ts, app_name, contention_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "total_estimated_execution_time_idx" (aggregated_ts, app_name, total_estimated_execution_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "p99_latency_idx" (aggregated_ts, app_name, p99_latency DESC) WHERE app_name NOT LIKE '$ internal%',
FAMILY "primary" (
crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8,
aggregated_ts,
Expand All @@ -570,7 +574,8 @@ CREATE TABLE system.statement_statistics (
service_latency,
cpu_sql_nanos,
contention_time,
total_estimated_execution_time
total_estimated_execution_time,
p99_latency
)
)
`
Expand All @@ -594,6 +599,7 @@ CREATE TABLE system.transaction_statistics (
cpu_sql_nanos FLOAT AS (` + cpuSqlNanosComputeExpr + `) STORED,
contention_time FLOAT AS (` + contentionTimeComputeExpr + `) STORED,
total_estimated_execution_time FLOAT AS (` + totalEstimatedExecutionTimeExpr + `) STORED,
p99_latency FLOAT8 AS (` + p99LatencyComputeExpr + `) STORED,
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts, fingerprint_id, app_name, node_id)
USING HASH WITH (bucket_count=8),
Expand All @@ -603,6 +609,7 @@ CREATE TABLE system.transaction_statistics (
INDEX "cpu_sql_nanos_idx" (aggregated_ts, app_name, cpu_sql_nanos DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "contention_time_idx" (aggregated_ts, app_name, contention_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "total_estimated_execution_time_idx" (aggregated_ts, app_name, total_estimated_execution_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "p99_latency_idx" (aggregated_ts, app_name, p99_latency DESC) WHERE app_name NOT LIKE '$ internal%',
FAMILY "primary" (
crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8,
aggregated_ts,
Expand All @@ -616,7 +623,8 @@ CREATE TABLE system.transaction_statistics (
service_latency,
cpu_sql_nanos,
contention_time,
total_estimated_execution_time
total_estimated_execution_time,
p99_latency
)
);
`
Expand Down Expand Up @@ -2484,6 +2492,7 @@ var (
{Name: "cpu_sql_nanos", ID: 16, Type: types.Float, Nullable: true, ComputeExpr: &cpuSqlNanosComputeExprStr},
{Name: "contention_time", ID: 17, Type: types.Float, Nullable: true, ComputeExpr: &contentionTimeComputeExprStr},
{Name: "total_estimated_execution_time", ID: 18, Type: types.Float, Nullable: true, ComputeExpr: &totalEstimatedExecutionTimeExprStr},
{Name: "p99_latency", ID: 19, Type: types.Float, Nullable: true, ComputeExpr: &p99LatencyComputeExprStr},
},
[]descpb.ColumnFamilyDescriptor{
{
Expand All @@ -2494,8 +2503,9 @@ var (
"aggregated_ts", "fingerprint_id", "transaction_fingerprint_id", "plan_hash", "app_name", "node_id",
"agg_interval", "metadata", "statistics", "plan", "index_recommendations", "execution_count",
"service_latency", "cpu_sql_nanos", "contention_time", "total_estimated_execution_time",
"p99_latency",
},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19},
DefaultColumnID: 0,
},
},
Expand Down Expand Up @@ -2668,6 +2678,26 @@ var (
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
descpb.IndexDescriptor{
Name: "p99_latency_idx",
ID: 9,
Unique: false,
KeyColumnNames: []string{
"aggregated_ts",
"app_name",
"p99_latency",
},
KeyColumnDirections: []catenumpb.IndexColumn_Direction{
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_DESC,
},
KeyColumnIDs: []descpb.ColumnID{1, 5, 19},
KeySuffixColumnIDs: []descpb.ColumnID{11, 2, 3, 4, 6},
CompositeColumnIDs: []descpb.ColumnID{19},
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
),
func(tbl *descpb.TableDescriptor) {
tbl.Checks = []*descpb.TableDescriptor_CheckConstraint{{
Expand Down Expand Up @@ -2709,6 +2739,7 @@ var (
{Name: "cpu_sql_nanos", ID: 11, Type: types.Float, Nullable: true, ComputeExpr: &cpuSqlNanosComputeExprStr},
{Name: "contention_time", ID: 12, Type: types.Float, Nullable: true, ComputeExpr: &contentionTimeComputeExprStr},
{Name: "total_estimated_execution_time", ID: 13, Type: types.Float, Nullable: true, ComputeExpr: &totalEstimatedExecutionTimeExprStr},
{Name: "p99_latency", ID: 14, Type: types.Float, Nullable: true, ComputeExpr: &p99LatencyComputeExprStr},
},
[]descpb.ColumnFamilyDescriptor{
{
Expand All @@ -2718,9 +2749,9 @@ var (
"crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8",
"aggregated_ts", "fingerprint_id", "app_name", "node_id",
"agg_interval", "metadata", "statistics", "execution_count", "service_latency", "cpu_sql_nanos",
"contention_time", "total_estimated_execution_time",
"contention_time", "total_estimated_execution_time", "p99_latency",
},
ColumnIDs: []descpb.ColumnID{8, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13},
ColumnIDs: []descpb.ColumnID{8, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14},
DefaultColumnID: 0,
},
},
Expand Down Expand Up @@ -2869,6 +2900,26 @@ var (
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
descpb.IndexDescriptor{
Name: "p99_latency_idx",
ID: 8,
Unique: false,
KeyColumnNames: []string{
"aggregated_ts",
"app_name",
"p99_latency",
},
KeyColumnDirections: []catenumpb.IndexColumn_Direction{
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_DESC,
},
KeyColumnIDs: []descpb.ColumnID{1, 3, 14},
KeySuffixColumnIDs: []descpb.ColumnID{8, 2, 4},
CompositeColumnIDs: []descpb.ColumnID{14},
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
),
func(tbl *descpb.TableDescriptor) {
tbl.Checks = []*descpb.TableDescriptor_CheckConstraint{{
Expand Down
16 changes: 10 additions & 6 deletions pkg/sql/catalog/systemschema_test/testdata/bootstrap
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/information_schema
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2420,6 +2420,7 @@ system public statement_statistics index_recommendati
system public statement_statistics indexes_usage 13
system public statement_statistics metadata 8
system public statement_statistics node_id 6
system public statement_statistics p99_latency 19
system public statement_statistics plan 10
system public statement_statistics plan_hash 4
system public statement_statistics service_latency 15
Expand Down Expand Up @@ -2487,6 +2488,7 @@ system public transaction_statistics execution_count
system public transaction_statistics fingerprint_id 2
system public transaction_statistics metadata 6
system public transaction_statistics node_id 4
system public transaction_statistics p99_latency 14
system public transaction_statistics service_latency 10
system public transaction_statistics statistics 7
system public transaction_statistics total_estimated_execution_time 13
Expand Down
8 changes: 8 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/pg_catalog
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1130,6 +1130,7 @@ indexrelid indrelid indnatts indisunique indnullsnotdistinct indisprimary
663840564 42 1 false false false false false false true false false true false 13 0 0 2 NULL NULL 1
663840565 42 2 false false false false false false true false false true false 2 3 0 0 0 0 2 2 NULL NULL 2
663840566 42 6 true false true false true false true false false true false 1 2 3 4 5 6 0 0 0 0 3403232968 0 0 0 0 0 0 0 2 2 2 2 2 2 NULL NULL 6
663840574 42 3 false false false false false false true false false true false 1 5 19 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
663840575 42 3 false false false false false false true false false true false 1 5 18 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
710236230 58 1 true false true false true false true false false true false 1 3403232968 0 2 NULL NULL 1
803027558 26 3 true false true false true false true false false true false 1 2 3 0 0 3403232968 0 0 0 2 2 2 NULL NULL 3
Expand Down Expand Up @@ -1189,6 +1190,7 @@ indexrelid indrelid indnatts indisunique indnullsnotdistinct indisprimary
3613730852 43 1 false false false false false false true false false true false 2 0 0 2 NULL NULL 1
3613730853 43 3 false false false false false false true false false true false 1 3 9 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
3613730855 43 4 true false true false true false true false false true false 1 2 3 4 0 0 3403232968 0 0 0 0 0 2 2 2 2 NULL NULL 4
3613730862 43 3 false false false false false false true false false true false 1 3 14 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
3660126519 59 3 true false true false true false true false false true false 1 2 3 0 3403232968 0 0 0 0 2 2 2 NULL NULL 3
3706522180 11 5 true false true false true false true false false true false 5 1 2 4 3 0 0 0 0 0 0 0 0 0 0 2 2 2 2 2 NULL NULL 5
3752917847 27 2 true false true false true false true false false true false 1 2 0 0 0 0 2 2 NULL NULL 2
Expand Down Expand Up @@ -1244,6 +1246,9 @@ indexrelid operator_argument_type_oid operator_argument_position
663840566 0 4
663840566 0 5
663840566 0 6
663840574 0 1
663840574 0 2
663840574 0 3
663840575 0 1
663840575 0 2
663840575 0 3
Expand Down Expand Up @@ -1345,6 +1350,9 @@ indexrelid operator_argument_type_oid operator_argument_position
3613730855 0 2
3613730855 0 3
3613730855 0 4
3613730862 0 1
3613730862 0 2
3613730862 0 3
3660126519 0 1
3660126519 0 2
3660126519 0 3
Expand Down
Loading

0 comments on commit 6e4ca2d

Please sign in to comment.