From a445110fa78bd72fcf1d0d50a8d9c6f4bbf1c380 Mon Sep 17 00:00:00 2001 From: Azhng Date: Tue, 20 Jul 2021 21:19:47 -0400 Subject: [PATCH] sql: remove `count` from stmt/txn stats system table Previously, system.statement_statistics and system.transaction_statistics table includes a `count` column that corresponds to `roachpb.StatementStatistics.Count` and `roachpb.TransactionStatistics.Count` fields respectively. The objective for that column is to make `INSERT ON CONFLICT DO UPDATE` style query easy. However, since early prototyping have shown that `INSERT ON CONFLICT DO UPDATE` style statement is quite inefficient, the SQL Stats flush mechanism will be implemented using separate queries INSERT and UPDATE statements. This column is no longer userful and it would require special handling. Removing this column simplifies the flush logic and removes the need for special handlings. Release note (sql change): count column is removed from system.statement_statistics and system.transaction_statistics tables. --- pkg/sql/catalog/systemschema/system.go | 58 ++++++++----------- .../testdata/logic_test/information_schema | 38 +++++------- .../testdata/logic_test/statement_statistics | 6 ++ 3 files changed, 46 insertions(+), 56 deletions(-) diff --git a/pkg/sql/catalog/systemschema/system.go b/pkg/sql/catalog/systemschema/system.go index 00f2fa1fceb8..1c7856ba2786 100644 --- a/pkg/sql/catalog/systemschema/system.go +++ b/pkg/sql/catalog/systemschema/system.go @@ -377,16 +377,13 @@ CREATE TABLE system.join_tokens ( CREATE TABLE system.statement_statistics ( aggregated_ts TIMESTAMPTZ NOT NULL, fingerprint_id BYTES NOT NULL, - plan_hash INT NOT NULL, + plan_hash INT8 NOT NULL, app_name STRING NOT NULL, - node_id INT NOT NULL, + node_id INT8 NOT NULL, - count INT NOT NULL, agg_interval INTERVAL NOT NULL, - metadata JSONB NOT NULL, statistics JSONB NOT NULL, - plan JSONB NOT NULL, PRIMARY KEY (aggregated_ts, fingerprint_id, plan_hash, app_name, node_id) @@ -399,7 +396,6 @@ CREATE TABLE system.statement_statistics ( plan_hash, app_name, node_id, - count, agg_interval, metadata, statistics, @@ -413,11 +409,9 @@ CREATE TABLE system.transaction_statistics ( aggregated_ts TIMESTAMPTZ NOT NULL, fingerprint_id BYTES NOT NULL, app_name STRING NOT NULL, - node_id INT NOT NULL, + node_id INT8 NOT NULL, - count INT NOT NULL, agg_interval INTERVAL NOT NULL, - metadata JSONB NOT NULL, statistics JSONB NOT NULL, @@ -430,7 +424,6 @@ CREATE TABLE system.transaction_statistics ( fingerprint_id, app_name, node_id, - count, agg_interval, metadata, statistics @@ -1928,21 +1921,20 @@ var ( {Name: "plan_hash", ID: 3, Type: types.Int, Nullable: false}, {Name: "app_name", ID: 4, Type: types.String, Nullable: false}, {Name: "node_id", ID: 5, Type: types.Int, Nullable: false}, - {Name: "count", ID: 6, Type: types.Int, Nullable: false}, - {Name: "agg_interval", ID: 7, Type: types.Interval, Nullable: false}, - {Name: "metadata", ID: 8, Type: types.Jsonb, Nullable: false}, - {Name: "statistics", ID: 9, Type: types.Jsonb, Nullable: false}, - {Name: "plan", ID: 10, Type: types.Jsonb, Nullable: false}, + {Name: "agg_interval", ID: 6, Type: types.Interval, Nullable: false}, + {Name: "metadata", ID: 7, Type: types.Jsonb, Nullable: false}, + {Name: "statistics", ID: 8, Type: types.Jsonb, Nullable: false}, + {Name: "plan", ID: 9, Type: types.Jsonb, Nullable: false}, { Name: "crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_shard_8", - ID: 11, + ID: 10, Type: types.Int4, Nullable: false, ComputeExpr: &sqlStmtHashComputeExpr, Hidden: true, }, }, - NextColumnID: 12, + NextColumnID: 11, Families: []descpb.ColumnFamilyDescriptor{ { Name: "primary", @@ -1950,9 +1942,9 @@ var ( ColumnNames: []string{ "crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_shard_8", "aggregated_ts", "fingerprint_id", "plan_hash", "app_name", "node_id", - "count", "agg_interval", "metadata", "statistics", "plan", + "agg_interval", "metadata", "statistics", "plan", }, - ColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + ColumnIDs: []descpb.ColumnID{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, DefaultColumnID: 0, }, }, @@ -1977,7 +1969,7 @@ var ( descpb.IndexDescriptor_ASC, descpb.IndexDescriptor_ASC, }, - KeyColumnIDs: []descpb.ColumnID{11, 1, 2, 3, 4, 5}, + KeyColumnIDs: []descpb.ColumnID{10, 1, 2, 3, 4, 5}, Version: descpb.StrictIndexColumnIDGuaranteesVersion, Sharded: descpb.ShardedDescriptor{ IsSharded: true, @@ -1997,7 +1989,7 @@ var ( Expr: "crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_shard_8 IN (0:::INT8, 1:::INT8, 2:::INT8, 3:::INT8, 4:::INT8, 5:::INT8, 6:::INT8, 7:::INT8)", Name: "check_crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_shard_8", Validity: descpb.ConstraintValidity_Validated, - ColumnIDs: []descpb.ColumnID{11}, + ColumnIDs: []descpb.ColumnID{10}, IsNonNullConstraint: false, Hidden: true, }, @@ -2022,7 +2014,7 @@ var ( descpb.IndexDescriptor_ASC, }, KeyColumnIDs: []descpb.ColumnID{2, 1, 3, 4, 5}, - KeySuffixColumnIDs: []descpb.ColumnID{11}, + KeySuffixColumnIDs: []descpb.ColumnID{10}, Version: descpb.StrictIndexColumnIDGuaranteesVersion, }, }, @@ -2046,20 +2038,19 @@ var ( {Name: "fingerprint_id", ID: 2, Type: types.Bytes, Nullable: false}, {Name: "app_name", ID: 3, Type: types.String, Nullable: false}, {Name: "node_id", ID: 4, Type: types.Int, Nullable: false}, - {Name: "count", ID: 5, Type: types.Int, Nullable: false}, - {Name: "agg_interval", ID: 6, Type: types.Interval, Nullable: false}, - {Name: "metadata", ID: 7, Type: types.Jsonb, Nullable: false}, - {Name: "statistics", ID: 8, Type: types.Jsonb, Nullable: false}, + {Name: "agg_interval", ID: 5, Type: types.Interval, Nullable: false}, + {Name: "metadata", ID: 6, Type: types.Jsonb, Nullable: false}, + {Name: "statistics", ID: 7, Type: types.Jsonb, Nullable: false}, { Name: "crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8", - ID: 9, + ID: 8, Type: types.Int4, Nullable: false, ComputeExpr: &sqlTxnHashComputeExpr, Hidden: true, }, }, - NextColumnID: 10, + NextColumnID: 9, Families: []descpb.ColumnFamilyDescriptor{ { Name: "primary", @@ -2067,10 +2058,9 @@ var ( ColumnNames: []string{ "crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8", "aggregated_ts", "fingerprint_id", "app_name", "node_id", - "count", "agg_interval", "metadata", "statistics", + "agg_interval", "metadata", "statistics", }, - ColumnIDs: []descpb.ColumnID{9, 1, 2, 3, 4, 5, 6, 7, 8}, - + ColumnIDs: []descpb.ColumnID{8, 1, 2, 3, 4, 5, 6, 7}, DefaultColumnID: 0, }, }, @@ -2093,7 +2083,7 @@ var ( descpb.IndexDescriptor_ASC, descpb.IndexDescriptor_ASC, }, - KeyColumnIDs: []descpb.ColumnID{9, 1, 2, 3, 4}, + KeyColumnIDs: []descpb.ColumnID{8, 1, 2, 3, 4}, Version: descpb.StrictIndexColumnIDGuaranteesVersion, Sharded: descpb.ShardedDescriptor{ IsSharded: true, @@ -2112,7 +2102,7 @@ var ( Expr: "crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8 IN (0:::INT8, 1:::INT8, 2:::INT8, 3:::INT8, 4:::INT8, 5:::INT8, 6:::INT8, 7:::INT8)", Name: "check_crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8", Validity: descpb.ConstraintValidity_Validated, - ColumnIDs: []descpb.ColumnID{9}, + ColumnIDs: []descpb.ColumnID{8}, IsNonNullConstraint: false, Hidden: true, }, @@ -2135,7 +2125,7 @@ var ( descpb.IndexDescriptor_ASC, }, KeyColumnIDs: []descpb.ColumnID{2, 1, 3, 4}, - KeySuffixColumnIDs: []descpb.ColumnID{9}, + KeySuffixColumnIDs: []descpb.ColumnID{8}, Version: descpb.StrictIndexColumnIDGuaranteesVersion, }, }, diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index 0f4d3328a2c3..4d06109f9579 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -1533,7 +1533,6 @@ system public 630200280_35_3_not_null system public 630200280_35_5_not_null system public statement_diagnostics_requests CHECK NO NO system public primary system public statement_diagnostics_requests PRIMARY KEY NO NO system public 630200280_42_10_not_null system public statement_statistics CHECK NO NO -system public 630200280_42_11_not_null system public statement_statistics CHECK NO NO system public 630200280_42_1_not_null system public statement_statistics CHECK NO NO system public 630200280_42_2_not_null system public statement_statistics CHECK NO NO system public 630200280_42_3_not_null system public statement_statistics CHECK NO NO @@ -1568,7 +1567,6 @@ system public 630200280_43_5_not_null system public 630200280_43_6_not_null system public transaction_statistics CHECK NO NO system public 630200280_43_7_not_null system public transaction_statistics CHECK NO NO system public 630200280_43_8_not_null system public transaction_statistics CHECK NO NO -system public 630200280_43_9_not_null system public transaction_statistics CHECK NO NO system public check_crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8 system public transaction_statistics CHECK NO NO system public primary system public transaction_statistics PRIMARY KEY NO NO system public 630200280_14_1_not_null system public ui CHECK NO NO @@ -1711,24 +1709,22 @@ system public 630200280_40_5_not_null system public 630200280_41_1_not_null id IS NOT NULL system public 630200280_41_2_not_null secret IS NOT NULL system public 630200280_41_3_not_null expiration IS NOT NULL -system public 630200280_42_10_not_null plan IS NOT NULL system public 630200280_42_1_not_null aggregated_ts IS NOT NULL system public 630200280_42_2_not_null fingerprint_id IS NOT NULL system public 630200280_42_3_not_null plan_hash IS NOT NULL system public 630200280_42_4_not_null app_name IS NOT NULL system public 630200280_42_5_not_null node_id IS NOT NULL -system public 630200280_42_6_not_null count IS NOT NULL -system public 630200280_42_7_not_null agg_interval IS NOT NULL -system public 630200280_42_8_not_null metadata IS NOT NULL -system public 630200280_42_9_not_null statistics IS NOT NULL +system public 630200280_42_6_not_null agg_interval IS NOT NULL +system public 630200280_42_7_not_null metadata IS NOT NULL +system public 630200280_42_8_not_null statistics IS NOT NULL +system public 630200280_42_9_not_null plan IS NOT NULL system public 630200280_43_1_not_null aggregated_ts IS NOT NULL system public 630200280_43_2_not_null fingerprint_id IS NOT NULL system public 630200280_43_3_not_null app_name IS NOT NULL system public 630200280_43_4_not_null node_id IS NOT NULL -system public 630200280_43_5_not_null count IS NOT NULL -system public 630200280_43_6_not_null agg_interval IS NOT NULL -system public 630200280_43_7_not_null metadata IS NOT NULL -system public 630200280_43_8_not_null statistics IS NOT NULL +system public 630200280_43_5_not_null agg_interval IS NOT NULL +system public 630200280_43_6_not_null metadata IS NOT NULL +system public 630200280_43_7_not_null statistics IS NOT NULL system public 630200280_44_1_not_null database_id IS NOT NULL system public 630200280_44_2_not_null role_name IS NOT NULL system public 630200280_44_3_not_null settings IS NOT NULL @@ -2052,17 +2048,16 @@ system public statement_diagnostics_requests id system public statement_diagnostics_requests requested_at 5 system public statement_diagnostics_requests statement_diagnostics_id 4 system public statement_diagnostics_requests statement_fingerprint 3 -system public statement_statistics agg_interval 7 +system public statement_statistics agg_interval 6 system public statement_statistics aggregated_ts 1 system public statement_statistics app_name 4 -system public statement_statistics count 6 -system public statement_statistics crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_shard_8 11 +system public statement_statistics crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_shard_8 10 system public statement_statistics fingerprint_id 2 -system public statement_statistics metadata 8 +system public statement_statistics metadata 7 system public statement_statistics node_id 5 -system public statement_statistics plan 10 +system public statement_statistics plan 9 system public statement_statistics plan_hash 3 -system public statement_statistics statistics 9 +system public statement_statistics statistics 8 system public table_statistics columnIDs 4 system public table_statistics createdAt 5 system public table_statistics distinctCount 7 @@ -2092,15 +2087,14 @@ system public tenant_usage total_write_reques system public tenants active 2 system public tenants id 1 system public tenants info 3 -system public transaction_statistics agg_interval 6 +system public transaction_statistics agg_interval 5 system public transaction_statistics aggregated_ts 1 system public transaction_statistics app_name 3 -system public transaction_statistics count 5 -system public transaction_statistics crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8 9 +system public transaction_statistics crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8 8 system public transaction_statistics fingerprint_id 2 -system public transaction_statistics metadata 7 +system public transaction_statistics metadata 6 system public transaction_statistics node_id 4 -system public transaction_statistics statistics 8 +system public transaction_statistics statistics 7 system public ui key 1 system public ui lastUpdated 3 system public ui value 2 diff --git a/pkg/sql/logictest/testdata/logic_test/statement_statistics b/pkg/sql/logictest/testdata/logic_test/statement_statistics index c6ce62af54b1..547ae8faeb98 100644 --- a/pkg/sql/logictest/testdata/logic_test/statement_statistics +++ b/pkg/sql/logictest/testdata/logic_test/statement_statistics @@ -1,5 +1,11 @@ # LogicTest: local !3node-tenant(52763) +# Disable SQL Stats flush to prevents stats from being cleared from the +# in-memory store. + +statement ok +SET CLUSTER SETTING sql.stats.flush.enabled = false; + # Check that node_statement_statistics report per application statement ok