Skip to content

Commit

Permalink
sql: add p99 computed column and index to system.statement_statistics…
Browse files Browse the repository at this point in the history 
…, system.transaction_statistics

Part of #98624.
Follows #99042.

This commit adds a new computed column (p99_latency) and partial
index (p99_latency_idx) to the system.statement_statistics and
system.transaction_statistics tables.

The commit also increases the hard-coded fingerprint execution
count for the flush benchmark to the maximum for in-memory
statement stats that trigger a flush.

Epic: none

Release note: None
  • Loading branch information
@ericharmeling
ericharmeling committed Mar 21, 2023
1 parent 1352ad2 commit 9dc146f
Show file tree
Hide file tree
Showing 11 changed files with 499 additions and 217 deletions.
12 changes: 6 additions & 6 deletions pkg/sql/catalog/bootstrap/testdata/testdata
View file

Large diffs are not rendered by default.

61 changes: 56 additions & 5 deletions pkg/sql/catalog/systemschema/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ CREATE SEQUENCE system.role_id_seq START 100 MINVALUE 100 MAXVALUE 2147483647;`
cpuSqlNanosComputeExpr = `(((statistics->'execution_statistics':::STRING)->'cpuSQLNanos':::STRING)->'mean':::STRING)::FLOAT8`
contentionTimeComputeExpr = `(((statistics->'execution_statistics':::STRING)->'contentionTime':::STRING)->'mean':::STRING)::FLOAT8`
totalEstimatedExecutionTimeExpr = `((statistics->'statistics':::STRING)->'cnt':::STRING)::FLOAT8 * (((statistics->'statistics':::STRING)->'svcLat':::STRING)->>'mean':::STRING)::FLOAT8`
p99LatencyComputeExpr = `(((statistics->'statistics':::STRING)->'latencyInfo':::STRING)->'p99':::STRING)::FLOAT8`
)

var indexUsageComputeExprStr = indexUsageComputeExpr
Expand All @@ -134,6 +135,7 @@ var serviceLatencyComputeExprStr = serviceLatencyComputeExpr
var cpuSqlNanosComputeExprStr = cpuSqlNanosComputeExpr
var contentionTimeComputeExprStr = contentionTimeComputeExpr
var totalEstimatedExecutionTimeExprStr = totalEstimatedExecutionTimeExpr
var p99LatencyComputeExprStr = p99LatencyComputeExpr

// These system tables are not part of the system config.
const (
Expand Down Expand Up @@ -543,6 +545,7 @@ CREATE TABLE system.statement_statistics (
cpu_sql_nanos FLOAT AS (` + cpuSqlNanosComputeExpr + `) STORED,
contention_time FLOAT AS (` + contentionTimeComputeExpr + `) STORED,
total_estimated_execution_time FLOAT AS (` + totalEstimatedExecutionTimeExpr + `) STORED,
p99_latency FLOAT8 AS (` + p99LatencyComputeExpr + `) STORED,
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id)
USING HASH WITH (bucket_count=8),
Expand All @@ -553,6 +556,7 @@ CREATE TABLE system.statement_statistics (
INDEX "cpu_sql_nanos_idx" (aggregated_ts, app_name, cpu_sql_nanos DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "contention_time_idx" (aggregated_ts, app_name, contention_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "total_estimated_execution_time_idx" (aggregated_ts, app_name, total_estimated_execution_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "p99_latency_idx" (aggregated_ts, app_name, p99_latency DESC) WHERE app_name NOT LIKE '$ internal%',
FAMILY "primary" (
crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8,
aggregated_ts,
Expand All @@ -570,7 +574,8 @@ CREATE TABLE system.statement_statistics (
service_latency,
cpu_sql_nanos,
contention_time,
total_estimated_execution_time
total_estimated_execution_time,
p99_latency
)
)
`
Expand All @@ -594,6 +599,7 @@ CREATE TABLE system.transaction_statistics (
cpu_sql_nanos FLOAT AS (` + cpuSqlNanosComputeExpr + `) STORED,
contention_time FLOAT AS (` + contentionTimeComputeExpr + `) STORED,
total_estimated_execution_time FLOAT AS (` + totalEstimatedExecutionTimeExpr + `) STORED,
p99_latency FLOAT8 AS (` + p99LatencyComputeExpr + `) STORED,
CONSTRAINT "primary" PRIMARY KEY (aggregated_ts, fingerprint_id, app_name, node_id)
USING HASH WITH (bucket_count=8),
Expand All @@ -603,6 +609,7 @@ CREATE TABLE system.transaction_statistics (
INDEX "cpu_sql_nanos_idx" (aggregated_ts, app_name, cpu_sql_nanos DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "contention_time_idx" (aggregated_ts, app_name, contention_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "total_estimated_execution_time_idx" (aggregated_ts, app_name, total_estimated_execution_time DESC) WHERE app_name NOT LIKE '$ internal%',
INDEX "p99_latency_idx" (aggregated_ts, app_name, p99_latency DESC) WHERE app_name NOT LIKE '$ internal%',
FAMILY "primary" (
crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8,
aggregated_ts,
Expand All @@ -616,7 +623,8 @@ CREATE TABLE system.transaction_statistics (
service_latency,
cpu_sql_nanos,
contention_time,
total_estimated_execution_time
total_estimated_execution_time,
p99_latency
)
);
`
Expand Down Expand Up @@ -2585,6 +2593,7 @@ var (
{Name: "cpu_sql_nanos", ID: 16, Type: types.Float, Nullable: true, ComputeExpr: &cpuSqlNanosComputeExprStr},
{Name: "contention_time", ID: 17, Type: types.Float, Nullable: true, ComputeExpr: &contentionTimeComputeExprStr},
{Name: "total_estimated_execution_time", ID: 18, Type: types.Float, Nullable: true, ComputeExpr: &totalEstimatedExecutionTimeExprStr},
{Name: "p99_latency", ID: 19, Type: types.Float, Nullable: true, ComputeExpr: &p99LatencyComputeExprStr},
},
[]descpb.ColumnFamilyDescriptor{
{
Expand All @@ -2595,8 +2604,9 @@ var (
"aggregated_ts", "fingerprint_id", "transaction_fingerprint_id", "plan_hash", "app_name", "node_id",
"agg_interval", "metadata", "statistics", "plan", "index_recommendations", "execution_count",
"service_latency", "cpu_sql_nanos", "contention_time", "total_estimated_execution_time",
"p99_latency",
},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18},
ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19},
DefaultColumnID: 0,
},
},
Expand Down Expand Up @@ -2769,6 +2779,26 @@ var (
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
descpb.IndexDescriptor{
Name: "p99_latency_idx",
ID: 9,
Unique: false,
KeyColumnNames: []string{
"aggregated_ts",
"app_name",
"p99_latency",
},
KeyColumnDirections: []catenumpb.IndexColumn_Direction{
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_DESC,
},
KeyColumnIDs: []descpb.ColumnID{1, 5, 19},
KeySuffixColumnIDs: []descpb.ColumnID{11, 2, 3, 4, 6},
CompositeColumnIDs: []descpb.ColumnID{19},
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
),
func(tbl *descpb.TableDescriptor) {
tbl.Checks = []*descpb.TableDescriptor_CheckConstraint{{
Expand Down Expand Up @@ -2810,6 +2840,7 @@ var (
{Name: "cpu_sql_nanos", ID: 11, Type: types.Float, Nullable: true, ComputeExpr: &cpuSqlNanosComputeExprStr},
{Name: "contention_time", ID: 12, Type: types.Float, Nullable: true, ComputeExpr: &contentionTimeComputeExprStr},
{Name: "total_estimated_execution_time", ID: 13, Type: types.Float, Nullable: true, ComputeExpr: &totalEstimatedExecutionTimeExprStr},
{Name: "p99_latency", ID: 14, Type: types.Float, Nullable: true, ComputeExpr: &p99LatencyComputeExprStr},
},
[]descpb.ColumnFamilyDescriptor{
{
Expand All @@ -2819,9 +2850,9 @@ var (
"crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8",
"aggregated_ts", "fingerprint_id", "app_name", "node_id",
"agg_interval", "metadata", "statistics", "execution_count", "service_latency", "cpu_sql_nanos",
"contention_time", "total_estimated_execution_time",
"contention_time", "total_estimated_execution_time", "p99_latency",
},
ColumnIDs: []descpb.ColumnID{8, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13},
ColumnIDs: []descpb.ColumnID{8, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14},
DefaultColumnID: 0,
},
},
Expand Down Expand Up @@ -2970,6 +3001,26 @@ var (
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
descpb.IndexDescriptor{
Name: "p99_latency_idx",
ID: 8,
Unique: false,
KeyColumnNames: []string{
"aggregated_ts",
"app_name",
"p99_latency",
},
KeyColumnDirections: []catenumpb.IndexColumn_Direction{
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_ASC,
catenumpb.IndexColumn_DESC,
},
KeyColumnIDs: []descpb.ColumnID{1, 3, 14},
KeySuffixColumnIDs: []descpb.ColumnID{8, 2, 4},
CompositeColumnIDs: []descpb.ColumnID{14},
Version: descpb.StrictIndexColumnIDGuaranteesVersion,
Predicate: "app_name NOT LIKE '$ internal%':::STRING",
},
),
func(tbl *descpb.TableDescriptor) {
tbl.Checks = []*descpb.TableDescriptor_CheckConstraint{{
Expand Down
20 changes: 12 additions & 8 deletions pkg/sql/catalog/systemschema_test/testdata/bootstrap
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog
View file

Large diffs are not rendered by default.

40 changes: 20 additions & 20 deletions pkg/sql/logictest/testdata/logic_test/gen_test_objects
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,26 +192,26 @@ WHERE table_catalog = 'newdb2' AND table_schema = 'public'
ORDER BY table_name, column_name
LIMIT 20
----
"comm%pents" " object_%2cid" bigint
"comm%pents" "%vcommEnt" text
"comm%pents" rowid bigint
"comm%pents" sub_id bigint
"comm%pents" "typ(e" bigint
migrations completed_at timestamp with time zone
migrations internal bigint
migrations "majo,r" bigint
migrations "mino*😫r" bigint
migrations patch bigint
migrations rowid bigint
role_members """member" text
role_members "isA!dmin" boolean
role_members "mem b"" er_id " oid
role_members "role" text
role_members role_id oid
role_members rowid bigint
role_options """user_id" oid
role_options "op tion" text
role_options rowid bigint
"comm%pents" "'sub__i😌d" bigint
"comm%pents" "Typ'e" bigint
"comm%pents" "commen t" text
"comm%pents" "objec\\u6EEFt_id" bigint
"comm%pents" rowid bigint
migrations "Patch" bigint
migrations completed_at timestamp with time zone
migrations internal bigint
migrations major bigint
migrations minor bigint
migrations rowid bigint
role_members " member_id" oid
role_members """ role_ id" oid
role_members "isAdmin%p" boolean
role_members member text
role_members "roLe" text
role_members rowid bigint
role_options "o""ption" text
role_options rowid bigint
role_options user_id oid

subtest templates/different_templates_in_each_db

Expand Down
2 changes: 2 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/information_schema
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2518,6 +2518,7 @@ system public statement_statistics index_recommendati
system public statement_statistics indexes_usage 13
system public statement_statistics metadata 8
system public statement_statistics node_id 6
system public statement_statistics p99_latency 19
system public statement_statistics plan 10
system public statement_statistics plan_hash 4
system public statement_statistics service_latency 15
Expand Down Expand Up @@ -2600,6 +2601,7 @@ system public transaction_statistics execution_count
system public transaction_statistics fingerprint_id 2
system public transaction_statistics metadata 6
system public transaction_statistics node_id 4
system public transaction_statistics p99_latency 14
system public transaction_statistics service_latency 10
system public transaction_statistics statistics 7
system public transaction_statistics total_estimated_execution_time 13
Expand Down
8 changes: 8 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/pg_catalog
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1130,6 +1130,7 @@ indexrelid indrelid indnatts indisunique indnullsnotdistinct indisprimary
663840564 42 1 false false false false false false true false false true false 13 0 0 2 NULL NULL 1
663840565 42 2 false false false false false false true false false true false 2 3 0 0 0 0 2 2 NULL NULL 2
663840566 42 6 true false true false true false true false false true false 1 2 3 4 5 6 0 0 0 0 3403232968 0 0 0 0 0 0 0 2 2 2 2 2 2 NULL NULL 6
663840574 42 3 false false false false false false true false false true false 1 5 19 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
663840575 42 3 false false false false false false true false false true false 1 5 18 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
710236230 58 1 true false true false true false true false false true false 1 3403232968 0 2 NULL NULL 1
803027558 26 3 true false true false true false true false false true false 1 2 3 0 0 3403232968 0 0 0 2 2 2 NULL NULL 3
Expand Down Expand Up @@ -1197,6 +1198,7 @@ indexrelid indrelid indnatts indisunique indnullsnotdistinct indisprimary
3613730852 43 1 false false false false false false true false false true false 2 0 0 2 NULL NULL 1
3613730853 43 3 false false false false false false true false false true false 1 3 9 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
3613730855 43 4 true false true false true false true false false true false 1 2 3 4 0 0 3403232968 0 0 0 0 0 2 2 2 2 NULL NULL 4
3613730862 43 3 false false false false false false true false false true false 1 3 14 0 3403232968 0 0 0 0 2 2 1 NULL app_name NOT LIKE '$ internal%'::STRING 3
3660126519 59 3 true false true false true false true false false true false 1 2 3 0 3403232968 0 0 0 0 2 2 2 NULL NULL 3
3706522180 11 5 true false true false true false true false false true false 5 1 2 4 3 0 0 0 0 0 0 0 0 0 0 2 2 2 2 2 NULL NULL 5
3752917847 27 2 true false true false true false true false false true false 1 2 0 0 0 0 2 2 NULL NULL 2
Expand Down Expand Up @@ -1260,6 +1262,9 @@ indexrelid operator_argument_type_oid operator_argument_position
663840566 0 4
663840566 0 5
663840566 0 6
663840574 0 1
663840574 0 2
663840574 0 3
663840575 0 1
663840575 0 2
663840575 0 3
Expand Down Expand Up @@ -1381,6 +1386,9 @@ indexrelid operator_argument_type_oid operator_argument_position
3613730855 0 2
3613730855 0 3
3613730855 0 4
3613730862 0 1
3613730862 0 2
3613730862 0 3
3660126519 0 1
3660126519 0 2
3660126519 0 3
Expand Down
Loading

0 comments on commit 9dc146f

Please sign in to comment.