From 9ee243aeb7bac0348a4ccce49c7c0a157482f301 Mon Sep 17 00:00:00 2001 From: j82w Date: Tue, 26 Sep 2023 16:02:54 -0400 Subject: [PATCH 1/2] sql: reduce memory usage of activity update job Problem: The `array_agg` was causing a large amount of memory. Solution: This adds 3 new aggregate functions to avoid `array_agg` . The functions are `merge_stats_metadata` ,`merge_statement_stats` , `merge_transaction_stats`. They only create a single object and keep adding to the single object instead of allocating an array of all the JSON objects and then passing it to a function to aggregate. The new functions are undocumented to avoid customers from taking a dependency on this logic. Fixes: #111866 Release note (sql change): Reduces the sql activity job memory usage. --- pkg/sql/BUILD.bazel | 1 + pkg/sql/appstatspb/app_stats.go | 30 + pkg/sql/distsql/columnar_operators_test.go | 8 + pkg/sql/execinfrapb/aggregate_funcs.go | 3 + pkg/sql/execinfrapb/processors_sql.proto | 3 + .../exec/execbuilder/testdata/observability | 660 ++++++++---------- pkg/sql/opt/memo/typing.go | 4 + pkg/sql/opt/operator.go | 110 +-- pkg/sql/opt/ops/scalar.opt | 15 + pkg/sql/opt/optbuilder/groupby.go | 6 + pkg/sql/sem/builtins/aggregate_builtins.go | 269 +++++++ pkg/sql/sem/builtins/builtins.go | 40 +- pkg/sql/sem/builtins/fixed_oids.go | 3 + pkg/sql/sql_activity_update_job_test.go | 122 ++++ .../sqlstatsutil/json_decoding.go | 10 + .../sqlstatsutil/json_impl.go | 11 + 16 files changed, 844 insertions(+), 451 deletions(-) diff --git a/pkg/sql/BUILD.bazel b/pkg/sql/BUILD.bazel index b60d931a3cef..be79f7a7df71 100644 --- a/pkg/sql/BUILD.bazel +++ b/pkg/sql/BUILD.bazel @@ -829,6 +829,7 @@ go_test( "//pkg/sql/sqlliveness/sqllivenesstestutils", "//pkg/sql/sqlstats", "//pkg/sql/sqlstats/persistedsqlstats", + "//pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil", "//pkg/sql/sqltestutils", "//pkg/sql/stats", "//pkg/sql/stmtdiagnostics", diff --git a/pkg/sql/appstatspb/app_stats.go b/pkg/sql/appstatspb/app_stats.go index aa61f740f0f8..29edb63aafad 100644 --- a/pkg/sql/appstatspb/app_stats.go +++ b/pkg/sql/appstatspb/app_stats.go @@ -138,6 +138,36 @@ func (t *TransactionStatistics) Add(other *TransactionStatistics) { t.Count += other.Count } +// Add combines CollectedStatementStatistics into a single AggregatedStatementMetadata. +func (s *AggregatedStatementMetadata) Add(other *CollectedStatementStatistics) { + // Only set the value if it hasn't already been set. + if s.Query == "" || s.QuerySummary == "" { + s.ImplicitTxn = other.Key.ImplicitTxn + s.Query = other.Key.Query + s.QuerySummary = other.Key.QuerySummary + s.StmtType = other.Stats.SQLType + } + + // Avoid creating the array if the db names match. + if len(s.Databases) != 1 || s.Databases[0] != other.Key.Database { + s.Databases = util.CombineUniqueString(s.Databases, []string{other.Key.Database}) + } + + if other.Key.DistSQL { + s.DistSQLCount++ + } + if other.Key.Failed { + s.FailedCount++ + } + if other.Key.FullScan { + s.FullScanCount++ + } + if other.Key.Vec { + s.VecCount++ + } + s.TotalCount++ +} + // Add combines other into this StatementStatistics. func (s *StatementStatistics) Add(other *StatementStatistics) { s.FirstAttemptCount += other.FirstAttemptCount diff --git a/pkg/sql/distsql/columnar_operators_test.go b/pkg/sql/distsql/columnar_operators_test.go index a3799047407b..d40b8a15d9be 100644 --- a/pkg/sql/distsql/columnar_operators_test.go +++ b/pkg/sql/distsql/columnar_operators_test.go @@ -105,6 +105,9 @@ var aggregateFuncToNumArguments = map[execinfrapb.AggregatorSpec_Func]int{ execinfrapb.FinalCorr: 1, execinfrapb.FinalSqrdiff: 3, execinfrapb.ArrayCatAgg: 1, + execinfrapb.MergeStatsMetadata: 1, + execinfrapb.MergeStatementStats: 1, + execinfrapb.MergeTransactionStats: 1, } // TestAggregateFuncToNumArguments ensures that all aggregate functions are @@ -183,6 +186,11 @@ func TestAggregatorAgainstProcessor(t *testing.T) { execinfrapb.PercentileContImpl: // We skip percentile functions because those can only be // planned as window functions. + case execinfrapb.MergeStatsMetadata, + execinfrapb.MergeStatementStats, + execinfrapb.MergeTransactionStats: + // We skip merge statistics functions because they + // require custom JSON objects. default: found = true } diff --git a/pkg/sql/execinfrapb/aggregate_funcs.go b/pkg/sql/execinfrapb/aggregate_funcs.go index 7068a7a335be..63572e0ad7fa 100644 --- a/pkg/sql/execinfrapb/aggregate_funcs.go +++ b/pkg/sql/execinfrapb/aggregate_funcs.go @@ -74,4 +74,7 @@ const ( FinalCorr = AggregatorSpec_FINAL_CORR FinalSqrdiff = AggregatorSpec_FINAL_SQRDIFF ArrayCatAgg = AggregatorSpec_ARRAY_CAT_AGG + MergeStatsMetadata = AggregatorSpec_MERGE_STATS_METADATA + MergeStatementStats = AggregatorSpec_MERGE_STATEMENT_STATS + MergeTransactionStats = AggregatorSpec_MERGE_TRANSACTION_STATS ) diff --git a/pkg/sql/execinfrapb/processors_sql.proto b/pkg/sql/execinfrapb/processors_sql.proto index d3e64c803ff9..5d345d9b9863 100644 --- a/pkg/sql/execinfrapb/processors_sql.proto +++ b/pkg/sql/execinfrapb/processors_sql.proto @@ -864,6 +864,9 @@ message AggregatorSpec { FINAL_CORR = 59; FINAL_SQRDIFF = 60; ARRAY_CAT_AGG = 61; + MERGE_STATS_METADATA = 62; + MERGE_STATEMENT_STATS = 63; + MERGE_TRANSACTION_STATS = 64; } enum Type { diff --git a/pkg/sql/opt/exec/execbuilder/testdata/observability b/pkg/sql/opt/exec/execbuilder/testdata/observability index 9996e0b7594b..459a208867db 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/observability +++ b/pkg/sql/opt/exec/execbuilder/testdata/observability @@ -44,8 +44,8 @@ EXPLAIN (VERBOSE) UPSERT INTO system.public.transaction_activity app_name, fingerprint_id, agg_interval, - crdb_internal.merge_stats_metadata(array_agg(metadata)) AS metadata, - crdb_internal.merge_transaction_stats(array_agg(statistics)) AS statistics + merge_stats_metadata(metadata) AS metadata, + merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics WHERE aggregated_ts = '2023-04-10 16:00:00.000000 +00:00' and app_name not like '$ internal%' @@ -64,67 +64,58 @@ vectorized: true │ arbiter indexes: primary │ └── • project - │ columns: (max, fingerprint_id, app_name, agg_interval, metadata, statistics, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, metadata, statistics, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) + │ columns: (max, fingerprint_id, app_name, agg_interval, merge_stats_metadata, merge_transaction_stats, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, merge_stats_metadata, merge_transaction_stats, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) │ └── • lookup join (left outer) - │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, metadata, statistics, aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) + │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats, aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) │ estimated row count: 3 (missing stats) │ table: transaction_activity@primary │ equality: (max, fingerprint_id, app_name) = (aggregated_ts,fingerprint_id,app_name) │ equality cols are key │ └── • distinct - │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, metadata, statistics) + │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats) │ estimated row count: 3 (missing stats) │ distinct on: fingerprint_id, app_name, max │ nulls are distinct │ error on duplicate │ └── • render - │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, metadata, statistics) + │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats) │ render query: '' - │ render int8: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render ?column?: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render int8: ((merge_transaction_stats->'execution_statistics')->>'cnt')::INT8 + │ render ?column?: ((merge_transaction_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_transaction_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 │ render execution_total_cluster_seconds: 100.0 - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render float8: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render coalesce: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_transaction_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_transaction_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render float8: (((merge_transaction_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render coalesce: COALESCE((((merge_transaction_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) │ render fingerprint_id: fingerprint_id │ render app_name: app_name │ render agg_interval: agg_interval │ render max: max - │ render metadata: metadata - │ render statistics: statistics + │ render merge_stats_metadata: merge_stats_metadata + │ render merge_transaction_stats: merge_transaction_stats │ - └── • render - │ columns: (metadata, statistics, fingerprint_id, app_name, agg_interval, max) - │ render metadata: crdb_internal.merge_stats_metadata(array_agg) - │ render statistics: crdb_internal.merge_transaction_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render app_name: app_name - │ render agg_interval: agg_interval - │ render max: max + └── • group (hash) + │ columns: (fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats) + │ estimated row count: 3 (missing stats) + │ aggregate 0: max(aggregated_ts) + │ aggregate 1: merge_stats_metadata(metadata) + │ aggregate 2: merge_transaction_stats(statistics) + │ group by: fingerprint_id, app_name, agg_interval │ - └── • group (hash) - │ columns: (fingerprint_id, app_name, agg_interval, max, array_agg, array_agg) + └── • index join + │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics) │ estimated row count: 3 (missing stats) - │ aggregate 0: max(aggregated_ts) - │ aggregate 1: array_agg(metadata) - │ aggregate 2: array_agg(statistics) - │ group by: fingerprint_id, app_name, agg_interval + │ table: transaction_statistics@primary + │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, aggregated_ts, fingerprint_id, app_name, node_id │ - └── • index join - │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics) - │ estimated row count: 3 (missing stats) - │ table: transaction_statistics@primary - │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, aggregated_ts, fingerprint_id, app_name, node_id - │ - └── • scan - columns: (aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8) - estimated row count: 3 (missing stats) - table: transaction_statistics@execution_count_idx (partial index) - spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z + └── • scan + columns: (aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8) + estimated row count: 3 (missing stats) + table: transaction_statistics@execution_count_idx (partial index) + spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z # Upsert all statement_activity query T retry @@ -159,8 +150,8 @@ EXPLAIN (VERBOSE) UPSERT plan_hash, app_name, agg_interval, - crdb_internal.merge_stats_metadata(array_agg(metadata)) AS metadata, - crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics, + merge_stats_metadata(metadata) AS metadata, + merge_statement_stats(statistics) AS statistics, plan, index_recommendations FROM system.public.statement_statistics @@ -185,31 +176,31 @@ vectorized: true │ arbiter indexes: primary │ └── • project - │ columns: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, metadata, statistics, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) + │ columns: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, merge_stats_metadata, merge_statement_stats, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, merge_stats_metadata, merge_statement_stats, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) │ └── • lookup join (left outer) - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) │ estimated row count: 3 (missing stats) │ table: statement_activity@primary │ equality: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name) = (aggregated_ts,fingerprint_id,transaction_fingerprint_id,plan_hash,app_name) │ equality cols are key │ └── • distinct - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics) + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) │ estimated row count: 3 (missing stats) │ distinct on: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, max │ nulls are distinct │ error on duplicate │ └── • render - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics) - │ render int8: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render ?column?: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) + │ render int8: ((merge_statement_stats->'execution_statistics')->>'cnt')::INT8 + │ render ?column?: ((merge_statement_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 │ render execution_total_cluster_seconds: 100.0 - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render float8: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render coalesce: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_statement_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_statement_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render float8: (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render coalesce: COALESCE((((merge_statement_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) │ render fingerprint_id: fingerprint_id │ render transaction_fingerprint_id: transaction_fingerprint_id │ render plan_hash: plan_hash @@ -218,41 +209,28 @@ vectorized: true │ render plan: plan │ render index_recommendations: index_recommendations │ render max: max - │ render metadata: metadata - │ render statistics: statistics + │ render merge_stats_metadata: merge_stats_metadata + │ render merge_statement_stats: merge_statement_stats │ - └── • render - │ columns: (metadata, statistics, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max) - │ render metadata: crdb_internal.merge_stats_metadata(array_agg) - │ render statistics: crdb_internal.merge_statement_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render transaction_fingerprint_id: transaction_fingerprint_id - │ render plan_hash: plan_hash - │ render app_name: app_name - │ render agg_interval: agg_interval - │ render plan: plan - │ render index_recommendations: index_recommendations - │ render max: max + └── • group (hash) + │ columns: (fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) + │ estimated row count: 3 (missing stats) + │ aggregate 0: max(aggregated_ts) + │ aggregate 1: merge_stats_metadata(metadata) + │ aggregate 2: merge_statement_stats(statistics) + │ group by: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations │ - └── • group (hash) - │ columns: (fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, array_agg, array_agg) + └── • index join + │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) │ estimated row count: 3 (missing stats) - │ aggregate 0: max(aggregated_ts) - │ aggregate 1: array_agg(metadata) - │ aggregate 2: array_agg(statistics) - │ group by: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations + │ table: statement_statistics@primary + │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id │ - └── • index join - │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) - │ estimated row count: 3 (missing stats) - │ table: statement_statistics@primary - │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id - │ - └── • scan - columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8) - estimated row count: 3 (missing stats) - table: statement_statistics@execution_count_idx (partial index) - spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z + └── • scan + columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8) + estimated row count: 3 (missing stats) + table: statement_statistics@execution_count_idx (partial index) + spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z # Upsert top 500 statement_activity including all statements in the top 500 transactions query T retry @@ -288,8 +266,8 @@ EXPLAIN (VERBOSE) UPSERT ss.plan_hash, ss.app_name, ss.agg_interval, - crdb_internal.merge_stats_metadata(array_agg(ss.metadata)) AS metadata, - crdb_internal.merge_statement_stats(array_agg(ss.statistics)) AS statistics, + merge_stats_metadata(ss.metadata) AS metadata, + merge_statement_stats(ss.statistics) AS statistics, ss.plan, ss.index_recommendations FROM system.public.statement_statistics ss @@ -313,7 +291,7 @@ EXPLAIN (VERBOSE) UPSERT 0) desc) AS lPos FROM (SELECT fingerprint_id, app_name, - crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics + merge_statement_stats(statistics) AS statistics FROM system.public.statement_statistics WHERE aggregated_ts = '2023-04-10 16:00:00.000000 +00:00' and app_name not like '$ internal%' @@ -345,31 +323,31 @@ vectorized: true │ arbiter indexes: primary │ └── • project - │ columns: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, metadata, statistics, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) + │ columns: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, merge_stats_metadata, merge_statement_stats, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, merge_stats_metadata, merge_statement_stats, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) │ └── • lookup join (left outer) - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) │ estimated row count: 0 (missing stats) │ table: statement_activity@primary │ equality: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name) = (aggregated_ts,fingerprint_id,transaction_fingerprint_id,plan_hash,app_name) │ equality cols are key │ └── • distinct - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics) + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) │ estimated row count: 0 (missing stats) │ distinct on: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, max │ nulls are distinct │ error on duplicate │ └── • render - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics) - │ render int8: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render ?column?: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) + │ render int8: ((merge_statement_stats->'execution_statistics')->>'cnt')::INT8 + │ render ?column?: ((merge_statement_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 │ render execution_total_cluster_seconds: 100.0 - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render float8: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render coalesce: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_statement_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_statement_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render float8: (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render coalesce: COALESCE((((merge_statement_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) │ render fingerprint_id: fingerprint_id │ render transaction_fingerprint_id: transaction_fingerprint_id │ render plan_hash: plan_hash @@ -378,117 +356,98 @@ vectorized: true │ render plan: plan │ render index_recommendations: index_recommendations │ render max: max - │ render metadata: metadata - │ render statistics: statistics + │ render merge_stats_metadata: merge_stats_metadata + │ render merge_statement_stats: merge_statement_stats │ - └── • render - │ columns: (metadata, statistics, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max) - │ render metadata: crdb_internal.merge_stats_metadata(array_agg) - │ render statistics: crdb_internal.merge_statement_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render transaction_fingerprint_id: transaction_fingerprint_id - │ render plan_hash: plan_hash - │ render app_name: app_name - │ render agg_interval: agg_interval - │ render plan: plan - │ render index_recommendations: index_recommendations - │ render max: max + └── • group (hash) + │ columns: (fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) + │ estimated row count: 0 (missing stats) + │ aggregate 0: max(aggregated_ts) + │ aggregate 1: merge_stats_metadata(metadata) + │ aggregate 2: merge_statement_stats(statistics) + │ group by: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations │ - └── • group (hash) - │ columns: (fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, array_agg, array_agg) - │ estimated row count: 0 (missing stats) - │ aggregate 0: max(aggregated_ts) - │ aggregate 1: array_agg(metadata) - │ aggregate 2: array_agg(statistics) - │ group by: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations + └── • project + │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) │ - └── • project - │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) + └── • hash join (inner) + │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) + │ estimated row count: 0 (missing stats) + │ equality: (app_name, fingerprint_id) = (app_name, fingerprint_id) + │ right cols are key │ - └── • hash join (inner) - │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) - │ estimated row count: 0 (missing stats) - │ equality: (app_name, fingerprint_id) = (app_name, fingerprint_id) - │ right cols are key - │ - ├── • scan - │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) - │ estimated row count: 10 (missing stats) - │ table: statement_statistics@primary - │ spans: /0/2023-04-10T16:00:00Z-/0/2023-04-10T16:00:00.000000001Z /1/2023-04-10T16:00:00Z-/1/2023-04-10T16:00:00.000000001Z /2/2023-04-10T16:00:00Z-/2/2023-04-10T16:00:00.000000001Z /3/2023-04-10T16:00:00Z-/3/2023-04-10T16:00:00.000000001Z /4/2023-04-10T16:00:00Z-/4/2023-04-10T16:00:00.000000001Z /5/2023-04-10T16:00:00Z-/5/2023-04-10T16:00:00.000000001Z /6/2023-04-10T16:00:00Z-/6/2023-04-10T16:00:00.000000001Z /7/2023-04-10T16:00:00Z-/7/2023-04-10T16:00:00.000000001Z + ├── • scan + │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) + │ estimated row count: 10 (missing stats) + │ table: statement_statistics@primary + │ spans: /0/2023-04-10T16:00:00Z-/0/2023-04-10T16:00:00.000000001Z /1/2023-04-10T16:00:00Z-/1/2023-04-10T16:00:00.000000001Z /2/2023-04-10T16:00:00Z-/2/2023-04-10T16:00:00.000000001Z /3/2023-04-10T16:00:00Z-/3/2023-04-10T16:00:00.000000001Z /4/2023-04-10T16:00:00Z-/4/2023-04-10T16:00:00.000000001Z /5/2023-04-10T16:00:00Z-/5/2023-04-10T16:00:00.000000001Z /6/2023-04-10T16:00:00Z-/6/2023-04-10T16:00:00.000000001Z /7/2023-04-10T16:00:00Z-/7/2023-04-10T16:00:00.000000001Z + │ + └── • filter + │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) + │ estimated row count: 3 (missing stats) + │ filter: (((((row_number < 500) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500) │ - └── • filter + └── • window │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) │ estimated row count: 3 (missing stats) - │ filter: (((((row_number < 500) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500) + │ window 0: row_number() OVER (ORDER BY row_number_6_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) + │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_6_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_5_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_5_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_4_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_4_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_3_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_3_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_2_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_2_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_1_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ - └── • window - │ columns: (fingerprint_id, app_name, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) - │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_1_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + └── • render + │ columns: (fingerprint_id, app_name, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ render row_number_1_orderby_1_1: ((merge_statement_stats->'execution_statistics')->>'cnt')::INT8 + │ render row_number_2_orderby_1_1: (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render row_number_3_orderby_1_1: ((merge_statement_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render row_number_4_orderby_1_1: COALESCE((((merge_statement_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render row_number_5_orderby_1_1: COALESCE((((merge_statement_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render row_number_6_orderby_1_1: COALESCE((((merge_statement_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render fingerprint_id: fingerprint_id + │ render app_name: app_name │ - └── • render - │ columns: (fingerprint_id, app_name, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) - │ render row_number_1_orderby_1_1: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render row_number_2_orderby_1_1: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render row_number_3_orderby_1_1: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render row_number_4_orderby_1_1: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render row_number_5_orderby_1_1: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render row_number_6_orderby_1_1: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) - │ render fingerprint_id: fingerprint_id - │ render app_name: app_name + └── • group (hash) + │ columns: (fingerprint_id, app_name, merge_statement_stats) + │ estimated row count: 3 (missing stats) + │ aggregate 0: merge_statement_stats(statistics) + │ group by: fingerprint_id, app_name │ - └── • render - │ columns: (statistics, fingerprint_id, app_name) - │ render statistics: crdb_internal.merge_statement_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render app_name: app_name + └── • project + │ columns: (fingerprint_id, app_name, statistics) │ - └── • group (hash) - │ columns: (fingerprint_id, app_name, array_agg) + └── • index join + │ columns: (aggregated_ts, fingerprint_id, app_name, statistics) │ estimated row count: 3 (missing stats) - │ aggregate 0: array_agg(statistics) - │ group by: fingerprint_id, app_name + │ table: statement_statistics@primary + │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id │ - └── • project - │ columns: (fingerprint_id, app_name, statistics) - │ - └── • index join - │ columns: (aggregated_ts, fingerprint_id, app_name, statistics) - │ estimated row count: 3 (missing stats) - │ table: statement_statistics@primary - │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id - │ - └── • scan - columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8) - estimated row count: 3 (missing stats) - table: statement_statistics@execution_count_idx (partial index) - spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z + └── • scan + columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_plan_hash_transaction_fingerprint_id_shard_8) + estimated row count: 3 (missing stats) + table: statement_statistics@execution_count_idx (partial index) + spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z # Upsert top 500 transactions query T retry @@ -517,8 +476,8 @@ EXPLAIN (VERBOSE) UPSERT ts.app_name, ts.fingerprint_id, ts.agg_interval, - crdb_internal.merge_stats_metadata(array_agg(ts.metadata)) AS metadata, - crdb_internal.merge_transaction_stats(array_agg(statistics)) AS statistics + merge_stats_metadata(ts.metadata) AS metadata, + merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics ts inner join (SELECT fingerprint_id, app_name, agg_interval FROM (SELECT fingerprint_id, app_name, agg_interval, @@ -539,7 +498,7 @@ EXPLAIN (VERBOSE) UPSERT (statistics -> 'statistics' -> 'latencyInfo' ->> 'p99')::float, 0) desc) AS lPos FROM (SELECT fingerprint_id, app_name, agg_interval, - crdb_internal.merge_transaction_stats(array_agg(statistics)) AS statistics + merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics WHERE aggregated_ts = '2023-04-10 16:00:00.000000 +00:00' and app_name not like '$ internal%' @@ -569,151 +528,135 @@ vectorized: true │ arbiter indexes: primary │ └── • project - │ columns: (max, fingerprint_id, app_name, agg_interval, metadata, statistics, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, metadata, statistics, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) + │ columns: (max, fingerprint_id, app_name, agg_interval, merge_stats_metadata, merge_transaction_stats, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, merge_stats_metadata, merge_transaction_stats, query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) │ └── • lookup join (left outer) - │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, metadata, statistics, aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) + │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats, aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, query, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) │ estimated row count: 0 (missing stats) │ table: transaction_activity@primary │ equality: (max, fingerprint_id, app_name) = (aggregated_ts,fingerprint_id,app_name) │ equality cols are key │ └── • distinct - │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, metadata, statistics) + │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats) │ estimated row count: 0 (missing stats) │ distinct on: fingerprint_id, app_name, max │ nulls are distinct │ error on duplicate │ └── • render - │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, metadata, statistics) + │ columns: (query, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats) │ render query: '' - │ render int8: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render ?column?: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render int8: ((merge_transaction_stats->'execution_statistics')->>'cnt')::INT8 + │ render ?column?: ((merge_transaction_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_transaction_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 │ render execution_total_cluster_seconds: 100.0 - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render float8: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render coalesce: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_transaction_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_transaction_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render float8: (((merge_transaction_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render coalesce: COALESCE((((merge_transaction_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) │ render fingerprint_id: fingerprint_id │ render app_name: app_name │ render agg_interval: agg_interval │ render max: max - │ render metadata: metadata - │ render statistics: statistics + │ render merge_stats_metadata: merge_stats_metadata + │ render merge_transaction_stats: merge_transaction_stats │ - └── • render - │ columns: (metadata, statistics, fingerprint_id, app_name, agg_interval, max) - │ render metadata: crdb_internal.merge_stats_metadata(array_agg) - │ render statistics: crdb_internal.merge_transaction_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render app_name: app_name - │ render agg_interval: agg_interval - │ render max: max + └── • group (hash) + │ columns: (fingerprint_id, app_name, agg_interval, max, merge_stats_metadata, merge_transaction_stats) + │ estimated row count: 0 (missing stats) + │ aggregate 0: max(aggregated_ts) + │ aggregate 1: merge_stats_metadata(metadata) + │ aggregate 2: merge_transaction_stats(statistics) + │ group by: fingerprint_id, app_name, agg_interval │ - └── • group (hash) - │ columns: (fingerprint_id, app_name, agg_interval, max, array_agg, array_agg) - │ estimated row count: 0 (missing stats) - │ aggregate 0: max(aggregated_ts) - │ aggregate 1: array_agg(metadata) - │ aggregate 2: array_agg(statistics) - │ group by: fingerprint_id, app_name, agg_interval + └── • project + │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics) │ └── • project - │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics) + │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ - └── • project - │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, metadata, statistics, fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + └── • lookup join (inner) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number, aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, agg_interval, metadata, statistics) + │ estimated row count: 0 (missing stats) + │ table: transaction_statistics@primary + │ equality: (crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, aggregated_ts, fingerprint_id, app_name, node_id) = (crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8,aggregated_ts,fingerprint_id,app_name,node_id) + │ equality cols are key + │ pred: agg_interval = agg_interval │ └── • lookup join (inner) - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number, aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, agg_interval, metadata, statistics) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number, aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8) │ estimated row count: 0 (missing stats) - │ table: transaction_statistics@primary - │ equality: (crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, aggregated_ts, fingerprint_id, app_name, node_id) = (crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8,aggregated_ts,fingerprint_id,app_name,node_id) - │ equality cols are key - │ pred: agg_interval = agg_interval + │ table: transaction_statistics@fingerprint_stats_idx + │ lookup condition: (fingerprint_id = fingerprint_id) AND (crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8 IN (0, 1, 2, 3, 4, 5, 6, 7)) + │ pred: app_name = app_name │ - └── • lookup join (inner) - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number, aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8) - │ estimated row count: 0 (missing stats) - │ table: transaction_statistics@fingerprint_stats_idx - │ lookup condition: (fingerprint_id = fingerprint_id) AND (crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8 IN (0, 1, 2, 3, 4, 5, 6, 7)) - │ pred: app_name = app_name + └── • filter + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) + │ estimated row count: 3 (missing stats) + │ filter: (((((row_number < 500) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500) │ - └── • filter + └── • window │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) │ estimated row count: 3 (missing stats) - │ filter: (((((row_number < 500) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500)) OR (row_number < 500) + │ window 0: row_number() OVER (ORDER BY row_number_6_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1, row_number) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_6_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_5_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_5_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_4_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_4_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_3_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_3_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_2_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ └── • window - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_2_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + │ window 0: row_number() OVER (ORDER BY row_number_1_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) │ - └── • window - │ columns: (fingerprint_id, app_name, agg_interval, row_number, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) - │ estimated row count: 3 (missing stats) - │ window 0: row_number() OVER (ORDER BY row_number_1_orderby_1_1 DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + └── • render + │ columns: (fingerprint_id, app_name, agg_interval, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) + │ render row_number_1_orderby_1_1: ((merge_transaction_stats->'execution_statistics')->>'cnt')::INT8 + │ render row_number_2_orderby_1_1: (((merge_transaction_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render row_number_3_orderby_1_1: ((merge_transaction_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_transaction_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render row_number_4_orderby_1_1: COALESCE((((merge_transaction_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render row_number_5_orderby_1_1: COALESCE((((merge_transaction_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render row_number_6_orderby_1_1: COALESCE((((merge_transaction_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render fingerprint_id: fingerprint_id + │ render app_name: app_name + │ render agg_interval: agg_interval │ - └── • render - │ columns: (fingerprint_id, app_name, agg_interval, row_number_1_orderby_1_1, row_number_2_orderby_1_1, row_number_3_orderby_1_1, row_number_4_orderby_1_1, row_number_5_orderby_1_1, row_number_6_orderby_1_1) - │ render row_number_1_orderby_1_1: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render row_number_2_orderby_1_1: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render row_number_3_orderby_1_1: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render row_number_4_orderby_1_1: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render row_number_5_orderby_1_1: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render row_number_6_orderby_1_1: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) - │ render fingerprint_id: fingerprint_id - │ render app_name: app_name - │ render agg_interval: agg_interval + └── • group (hash) + │ columns: (fingerprint_id, app_name, agg_interval, merge_transaction_stats) + │ estimated row count: 3 (missing stats) + │ aggregate 0: merge_transaction_stats(statistics) + │ group by: fingerprint_id, app_name, agg_interval │ - └── • render - │ columns: (statistics, fingerprint_id, app_name, agg_interval) - │ render statistics: crdb_internal.merge_transaction_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render app_name: app_name - │ render agg_interval: agg_interval + └── • project + │ columns: (fingerprint_id, app_name, agg_interval, statistics) │ - └── • group (hash) - │ columns: (fingerprint_id, app_name, agg_interval, array_agg) + └── • index join + │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, statistics) │ estimated row count: 3 (missing stats) - │ aggregate 0: array_agg(statistics) - │ group by: fingerprint_id, app_name, agg_interval + │ table: transaction_statistics@primary + │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, aggregated_ts, fingerprint_id, app_name, node_id │ - └── • project - │ columns: (fingerprint_id, app_name, agg_interval, statistics) - │ - └── • index join - │ columns: (aggregated_ts, fingerprint_id, app_name, agg_interval, statistics) - │ estimated row count: 3 (missing stats) - │ table: transaction_statistics@primary - │ key columns: crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8, aggregated_ts, fingerprint_id, app_name, node_id - │ - └── • scan - columns: (aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8) - estimated row count: 3 (missing stats) - table: transaction_statistics@execution_count_idx (partial index) - spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z + └── • scan + columns: (aggregated_ts, fingerprint_id, app_name, node_id, crdb_internal_aggregated_ts_app_name_fingerprint_id_node_id_shard_8) + estimated row count: 3 (missing stats) + table: transaction_statistics@execution_count_idx (partial index) + spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z # Upsert top 500 transactions query T retry @@ -764,8 +707,8 @@ FROM (SELECT max(ss.aggregated_ts) AS aggregated_ts, ss.plan_hash, ss.app_name, ss.agg_interval, - crdb_internal.merge_stats_metadata(array_agg(ss.metadata)) AS metadata, - crdb_internal.merge_statement_stats(array_agg(ss.statistics)) AS statistics, + merge_stats_metadata(ss.metadata) AS metadata, + merge_statement_stats(ss.statistics) AS statistics, ss.plan, ss.index_recommendations FROM system.public.statement_statistics ss @@ -790,31 +733,31 @@ vectorized: true │ arbiter indexes: primary │ └── • project - │ columns: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, metadata, statistics, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) + │ columns: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, merge_stats_metadata, merge_statement_stats, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds, agg_interval, merge_stats_metadata, merge_statement_stats, plan, index_recommendations, int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", aggregated_ts) │ └── • lookup join (left outer) - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations, execution_count, execution_total_seconds, execution_total_cluster_seconds, contention_time_avg_seconds, cpu_sql_avg_nanos, service_latency_avg_seconds, service_latency_p99_seconds) │ estimated row count: 7 (missing stats) │ table: statement_activity@primary │ equality: (max, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name) = (aggregated_ts,fingerprint_id,transaction_fingerprint_id,plan_hash,app_name) │ equality cols are key │ └── • distinct - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics) + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) │ estimated row count: 7 (missing stats) │ distinct on: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, max │ nulls are distinct │ error on duplicate │ └── • render - │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, metadata, statistics) - │ render int8: ((statistics->'execution_statistics')->>'cnt')::INT8 - │ render ?column?: ((statistics->'execution_statistics')->>'cnt')::FLOAT8 * (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ columns: (int8, "?column?", execution_total_cluster_seconds, "coalesce", "coalesce", float8, "coalesce", fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) + │ render int8: ((merge_statement_stats->'execution_statistics')->>'cnt')::INT8 + │ render ?column?: ((merge_statement_stats->'execution_statistics')->>'cnt')::FLOAT8 * (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 │ render execution_total_cluster_seconds: 100.0 - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) - │ render coalesce: COALESCE((((statistics->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) - │ render float8: (((statistics->'statistics')->'svcLat')->>'mean')::FLOAT8 - │ render coalesce: COALESCE((((statistics->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_statement_stats->'execution_statistics')->'contentionTime')->>'mean')::FLOAT8, 0.0) + │ render coalesce: COALESCE((((merge_statement_stats->'execution_statistics')->'cpu_sql_nanos')->>'mean')::FLOAT8, 0.0) + │ render float8: (((merge_statement_stats->'statistics')->'svcLat')->>'mean')::FLOAT8 + │ render coalesce: COALESCE((((merge_statement_stats->'statistics')->'latencyInfo')->>'p99')::FLOAT8, 0.0) │ render fingerprint_id: fingerprint_id │ render transaction_fingerprint_id: transaction_fingerprint_id │ render plan_hash: plan_hash @@ -823,87 +766,74 @@ vectorized: true │ render plan: plan │ render index_recommendations: index_recommendations │ render max: max - │ render metadata: metadata - │ render statistics: statistics + │ render merge_stats_metadata: merge_stats_metadata + │ render merge_statement_stats: merge_statement_stats │ - └── • render - │ columns: (metadata, statistics, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max) - │ render metadata: crdb_internal.merge_stats_metadata(array_agg) - │ render statistics: crdb_internal.merge_statement_stats(array_agg) - │ render fingerprint_id: fingerprint_id - │ render transaction_fingerprint_id: transaction_fingerprint_id - │ render plan_hash: plan_hash - │ render app_name: app_name - │ render agg_interval: agg_interval - │ render plan: plan - │ render index_recommendations: index_recommendations - │ render max: max + └── • group (hash) + │ columns: (fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, merge_stats_metadata, merge_statement_stats) + │ estimated row count: 7 (missing stats) + │ aggregate 0: max(aggregated_ts) + │ aggregate 1: merge_stats_metadata(metadata) + │ aggregate 2: merge_statement_stats(statistics) + │ group by: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations │ - └── • group (hash) - │ columns: (fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations, max, array_agg, array_agg) - │ estimated row count: 7 (missing stats) - │ aggregate 0: max(aggregated_ts) - │ aggregate 1: array_agg(metadata) - │ aggregate 2: array_agg(statistics) - │ group by: fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, plan, index_recommendations + └── • project + │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) │ - └── • project - │ columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) + └── • hash join (inner) + │ columns: (fingerprint_id, app_name, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) + │ estimated row count: 7 (missing stats) + │ equality: (app_name, fingerprint_id) = (app_name, fingerprint_id) + │ left cols are key │ - └── • hash join (inner) - │ columns: (fingerprint_id, app_name, aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) - │ estimated row count: 7 (missing stats) - │ equality: (app_name, fingerprint_id) = (app_name, fingerprint_id) - │ left cols are key - │ - ├── • render - │ │ columns: (fingerprint_id, app_name) - │ │ render fingerprint_id: fingerprint_id - │ │ render app_name: app_name - │ │ - │ └── • distinct - │ │ columns: (app_name, fingerprint_id) - │ │ estimated row count: 64 (missing stats) - │ │ distinct on: app_name, fingerprint_id - │ │ - │ └── • project - │ │ columns: (app_name, fingerprint_id) - │ │ - │ └── • filter - │ │ columns: (fingerprint_id, aggregated_ts, app_name, aggregated_ts, fingerprint_id, app_name) - │ │ estimated row count: 89 (missing stats) - │ │ filter: fingerprint_id IS NULL - │ │ - │ └── • hash join (left outer) - │ │ columns: (fingerprint_id, aggregated_ts, app_name, aggregated_ts, fingerprint_id, app_name) - │ │ estimated row count: 100 (missing stats) - │ │ equality: (fingerprint_id, app_name, aggregated_ts) = (fingerprint_id, app_name, aggregated_ts) - │ │ - │ ├── • render - │ │ │ columns: (fingerprint_id, aggregated_ts, app_name) - │ │ │ render fingerprint_id: decode(jsonb_array_elements_text, 'hex') - │ │ │ render aggregated_ts: aggregated_ts - │ │ │ render app_name: app_name - │ │ │ - │ │ └── • project set - │ │ │ columns: (aggregated_ts, app_name, metadata, jsonb_array_elements_text) - │ │ │ estimated row count: 100 (missing stats) - │ │ │ render 0: jsonb_array_elements_text(metadata->'stmtFingerprintIDs') - │ │ │ - │ │ └── • scan - │ │ columns: (aggregated_ts, app_name, metadata) - │ │ estimated row count: 10 (missing stats) - │ │ table: transaction_activity@primary - │ │ spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z - │ │ - │ └── • scan - │ columns: (aggregated_ts, fingerprint_id, app_name) - │ estimated row count: 10 (missing stats) - │ table: statement_activity@execution_count_idx - │ spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z - │ - └── • scan - columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) - estimated row count: 10 (missing stats) - table: statement_statistics@primary - spans: /0/2023-04-10T16:00:00Z-/0/2023-04-10T16:00:00.000000001Z /1/2023-04-10T16:00:00Z-/1/2023-04-10T16:00:00.000000001Z /2/2023-04-10T16:00:00Z-/2/2023-04-10T16:00:00.000000001Z /3/2023-04-10T16:00:00Z-/3/2023-04-10T16:00:00.000000001Z /4/2023-04-10T16:00:00Z-/4/2023-04-10T16:00:00.000000001Z /5/2023-04-10T16:00:00Z-/5/2023-04-10T16:00:00.000000001Z /6/2023-04-10T16:00:00Z-/6/2023-04-10T16:00:00.000000001Z /7/2023-04-10T16:00:00Z-/7/2023-04-10T16:00:00.000000001Z + ├── • render + │ │ columns: (fingerprint_id, app_name) + │ │ render fingerprint_id: fingerprint_id + │ │ render app_name: app_name + │ │ + │ └── • distinct + │ │ columns: (app_name, fingerprint_id) + │ │ estimated row count: 64 (missing stats) + │ │ distinct on: app_name, fingerprint_id + │ │ + │ └── • project + │ │ columns: (app_name, fingerprint_id) + │ │ + │ └── • filter + │ │ columns: (fingerprint_id, aggregated_ts, app_name, aggregated_ts, fingerprint_id, app_name) + │ │ estimated row count: 89 (missing stats) + │ │ filter: fingerprint_id IS NULL + │ │ + │ └── • hash join (left outer) + │ │ columns: (fingerprint_id, aggregated_ts, app_name, aggregated_ts, fingerprint_id, app_name) + │ │ estimated row count: 100 (missing stats) + │ │ equality: (fingerprint_id, app_name, aggregated_ts) = (fingerprint_id, app_name, aggregated_ts) + │ │ + │ ├── • render + │ │ │ columns: (fingerprint_id, aggregated_ts, app_name) + │ │ │ render fingerprint_id: decode(jsonb_array_elements_text, 'hex') + │ │ │ render aggregated_ts: aggregated_ts + │ │ │ render app_name: app_name + │ │ │ + │ │ └── • project set + │ │ │ columns: (aggregated_ts, app_name, metadata, jsonb_array_elements_text) + │ │ │ estimated row count: 100 (missing stats) + │ │ │ render 0: jsonb_array_elements_text(metadata->'stmtFingerprintIDs') + │ │ │ + │ │ └── • scan + │ │ columns: (aggregated_ts, app_name, metadata) + │ │ estimated row count: 10 (missing stats) + │ │ table: transaction_activity@primary + │ │ spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z + │ │ + │ └── • scan + │ columns: (aggregated_ts, fingerprint_id, app_name) + │ estimated row count: 10 (missing stats) + │ table: statement_activity@execution_count_idx + │ spans: /2023-04-10T16:00:00Z-/2023-04-10T16:00:00.000000001Z + │ + └── • scan + columns: (aggregated_ts, fingerprint_id, transaction_fingerprint_id, plan_hash, app_name, agg_interval, metadata, statistics, plan, index_recommendations) + estimated row count: 10 (missing stats) + table: statement_statistics@primary + spans: /0/2023-04-10T16:00:00Z-/0/2023-04-10T16:00:00.000000001Z /1/2023-04-10T16:00:00Z-/1/2023-04-10T16:00:00.000000001Z /2/2023-04-10T16:00:00Z-/2/2023-04-10T16:00:00.000000001Z /3/2023-04-10T16:00:00Z-/3/2023-04-10T16:00:00.000000001Z /4/2023-04-10T16:00:00Z-/4/2023-04-10T16:00:00.000000001Z /5/2023-04-10T16:00:00Z-/5/2023-04-10T16:00:00.000000001Z /6/2023-04-10T16:00:00Z-/6/2023-04-10T16:00:00.000000001Z /7/2023-04-10T16:00:00Z-/7/2023-04-10T16:00:00.000000001Z diff --git a/pkg/sql/opt/memo/typing.go b/pkg/sql/opt/memo/typing.go index 0634952dc5de..e0a3cf500f8d 100644 --- a/pkg/sql/opt/memo/typing.go +++ b/pkg/sql/opt/memo/typing.go @@ -208,6 +208,10 @@ func init() { typingFuncMap[opt.LeadOp] = typeAsFirstArg typingFuncMap[opt.NthValueOp] = typeAsFirstArg + typingFuncMap[opt.MergeStatsMetadataOp] = typeAsFirstArg + typingFuncMap[opt.MergeStatementStatsOp] = typeAsFirstArg + typingFuncMap[opt.MergeTransactionStatsOp] = typeAsFirstArg + // Modifiers for aggregations pass through their argument. typingFuncMap[opt.AggDistinctOp] = typeAsFirstArg typingFuncMap[opt.AggFilterOp] = typeAsFirstArg diff --git a/pkg/sql/opt/operator.go b/pkg/sql/opt/operator.go index d5738345637d..bd69ff948005 100644 --- a/pkg/sql/opt/operator.go +++ b/pkg/sql/opt/operator.go @@ -183,52 +183,55 @@ var UnaryOpReverseMap = map[Operator]tree.UnaryOperatorSymbol{ // AggregateOpReverseMap maps from an optimizer operator type to the name of an // aggregation function. var AggregateOpReverseMap = map[Operator]string{ - ArrayAggOp: "array_agg", - ArrayCatAggOp: "array_cat_agg", - AvgOp: "avg", - BitAndAggOp: "bit_and", - BitOrAggOp: "bit_or", - BoolAndOp: "bool_and", - BoolOrOp: "bool_or", - ConcatAggOp: "concat_agg", - CountOp: "count", - CorrOp: "corr", - CountRowsOp: "count_rows", - CovarPopOp: "covar_pop", - CovarSampOp: "covar_samp", - RegressionAvgXOp: "regr_avgx", - RegressionAvgYOp: "regr_avgy", - RegressionInterceptOp: "regr_intercept", - RegressionR2Op: "regr_r2", - RegressionSlopeOp: "regr_slope", - RegressionSXXOp: "regr_sxx", - RegressionSXYOp: "regr_sxy", - RegressionSYYOp: "regr_syy", - RegressionCountOp: "regr_count", - MaxOp: "max", - MinOp: "min", - SumIntOp: "sum_int", - SumOp: "sum", - SqrDiffOp: "sqrdiff", - VarianceOp: "variance", - StdDevOp: "stddev", - XorAggOp: "xor_agg", - JsonAggOp: "json_agg", - JsonbAggOp: "jsonb_agg", - JsonObjectAggOp: "json_object_agg", - JsonbObjectAggOp: "jsonb_object_agg", - StringAggOp: "string_agg", - ConstAggOp: "any_not_null", - ConstNotNullAggOp: "any_not_null", - AnyNotNullAggOp: "any_not_null", - PercentileDiscOp: "percentile_disc_impl", - PercentileContOp: "percentile_cont_impl", - VarPopOp: "var_pop", - StdDevPopOp: "stddev_pop", - STMakeLineOp: "st_makeline", - STUnionOp: "st_union", - STCollectOp: "st_collect", - STExtentOp: "st_extent", + ArrayAggOp: "array_agg", + ArrayCatAggOp: "array_cat_agg", + AvgOp: "avg", + BitAndAggOp: "bit_and", + BitOrAggOp: "bit_or", + BoolAndOp: "bool_and", + BoolOrOp: "bool_or", + ConcatAggOp: "concat_agg", + CountOp: "count", + CorrOp: "corr", + CountRowsOp: "count_rows", + CovarPopOp: "covar_pop", + CovarSampOp: "covar_samp", + RegressionAvgXOp: "regr_avgx", + RegressionAvgYOp: "regr_avgy", + RegressionInterceptOp: "regr_intercept", + RegressionR2Op: "regr_r2", + RegressionSlopeOp: "regr_slope", + RegressionSXXOp: "regr_sxx", + RegressionSXYOp: "regr_sxy", + RegressionSYYOp: "regr_syy", + RegressionCountOp: "regr_count", + MaxOp: "max", + MinOp: "min", + SumIntOp: "sum_int", + SumOp: "sum", + SqrDiffOp: "sqrdiff", + VarianceOp: "variance", + StdDevOp: "stddev", + XorAggOp: "xor_agg", + JsonAggOp: "json_agg", + JsonbAggOp: "jsonb_agg", + JsonObjectAggOp: "json_object_agg", + JsonbObjectAggOp: "jsonb_object_agg", + StringAggOp: "string_agg", + ConstAggOp: "any_not_null", + ConstNotNullAggOp: "any_not_null", + AnyNotNullAggOp: "any_not_null", + PercentileDiscOp: "percentile_disc_impl", + PercentileContOp: "percentile_cont_impl", + VarPopOp: "var_pop", + StdDevPopOp: "stddev_pop", + STMakeLineOp: "st_makeline", + STUnionOp: "st_union", + STCollectOp: "st_collect", + STExtentOp: "st_extent", + MergeStatsMetadataOp: "merge_stats_metadata", + MergeStatementStatsOp: "merge_statement_stats", + MergeTransactionStatsOp: "merge_transaction_stats", } // WindowOpReverseMap maps from an optimizer operator type to the name of a @@ -327,7 +330,8 @@ func AggregateIgnoresNulls(op Operator) bool { PercentileContOp, STMakeLineOp, STCollectOp, STExtentOp, STUnionOp, StdDevPopOp, VarPopOp, CovarPopOp, CovarSampOp, RegressionAvgXOp, RegressionAvgYOp, RegressionInterceptOp, RegressionR2Op, RegressionSlopeOp, RegressionSXXOp, - RegressionSXYOp, RegressionSYYOp, RegressionCountOp: + RegressionSXYOp, RegressionSYYOp, RegressionCountOp, MergeStatsMetadataOp, + MergeStatementStatsOp, MergeTransactionStatsOp: return true case ArrayAggOp, ArrayCatAggOp, ConcatAggOp, ConstAggOp, CountRowsOp, @@ -353,7 +357,8 @@ func AggregateIsNullOnEmpty(op Operator) bool { JsonObjectAggOp, JsonbObjectAggOp, StdDevPopOp, STCollectOp, STExtentOp, STUnionOp, VarPopOp, CovarPopOp, CovarSampOp, RegressionAvgXOp, RegressionAvgYOp, RegressionInterceptOp, RegressionR2Op, RegressionSlopeOp, RegressionSXXOp, - RegressionSXYOp, RegressionSYYOp: + RegressionSXYOp, RegressionSYYOp, MergeStatsMetadataOp, MergeStatementStatsOp, + MergeTransactionStatsOp: return true case CountOp, CountRowsOp, RegressionCountOp: @@ -381,7 +386,8 @@ func AggregateIsNeverNullOnNonNullInput(op Operator) bool { StringAggOp, SumOp, SumIntOp, XorAggOp, PercentileDiscOp, PercentileContOp, JsonObjectAggOp, JsonbObjectAggOp, StdDevPopOp, STCollectOp, STUnionOp, VarPopOp, CovarPopOp, RegressionAvgXOp, RegressionAvgYOp, RegressionSXXOp, - RegressionSXYOp, RegressionSYYOp, RegressionCountOp: + RegressionSXYOp, RegressionSYYOp, RegressionCountOp, MergeStatsMetadataOp, + MergeStatementStatsOp, MergeTransactionStatsOp: return true case VarianceOp, StdDevOp, CorrOp, CovarSampOp, RegressionInterceptOp, @@ -436,7 +442,8 @@ func AggregatesCanMerge(inner, outer Operator) bool { SqrDiffOp, STCollectOp, StdDevOp, StringAggOp, VarianceOp, StdDevPopOp, VarPopOp, CovarPopOp, CovarSampOp, RegressionAvgXOp, RegressionAvgYOp, RegressionInterceptOp, RegressionR2Op, RegressionSlopeOp, RegressionSXXOp, - RegressionSXYOp, RegressionSYYOp, RegressionCountOp: + RegressionSXYOp, RegressionSYYOp, RegressionCountOp, MergeStatsMetadataOp, + MergeStatementStatsOp, MergeTransactionStatsOp: return false default: @@ -458,7 +465,8 @@ func AggregateIgnoresDuplicates(op Operator) bool { VarPopOp, JsonObjectAggOp, JsonbObjectAggOp, STCollectOp, CovarPopOp, CovarSampOp, RegressionAvgXOp, RegressionAvgYOp, RegressionInterceptOp, RegressionR2Op, RegressionSlopeOp, RegressionSXXOp, RegressionSXYOp, - RegressionSYYOp, RegressionCountOp: + RegressionSYYOp, RegressionCountOp, MergeStatsMetadataOp, MergeStatementStatsOp, + MergeTransactionStatsOp: return false default: diff --git a/pkg/sql/opt/ops/scalar.opt b/pkg/sql/opt/ops/scalar.opt index 827a68aec1bf..56b77d303f7d 100644 --- a/pkg/sql/opt/ops/scalar.opt +++ b/pkg/sql/opt/ops/scalar.opt @@ -1015,6 +1015,21 @@ define JsonbObjectAgg { Value ScalarExpr } +[Scalar, Aggregate] +define MergeStatsMetadata { + Input ScalarExpr +} + +[Scalar, Aggregate] +define MergeStatementStats { + Input ScalarExpr +} + +[Scalar, Aggregate] +define MergeTransactionStats { + Input ScalarExpr +} + [Scalar, Aggregate] define StringAgg { Input ScalarExpr diff --git a/pkg/sql/opt/optbuilder/groupby.go b/pkg/sql/opt/optbuilder/groupby.go index b3c6fe61de8c..399ec8aee90a 100644 --- a/pkg/sql/opt/optbuilder/groupby.go +++ b/pkg/sql/opt/optbuilder/groupby.go @@ -891,6 +891,12 @@ func (b *Builder) constructAggregate(name string, args []opt.ScalarExpr) opt.Sca return b.factory.ConstructJsonObjectAgg(args[0], args[1]) case "jsonb_object_agg": return b.factory.ConstructJsonbObjectAgg(args[0], args[1]) + case "merge_stats_metadata": + return b.factory.ConstructMergeStatsMetadata(args[0]) + case "merge_statement_stats": + return b.factory.ConstructMergeStatementStats(args[0]) + case "merge_transaction_stats": + return b.factory.ConstructMergeTransactionStats(args[0]) } panic(errors.AssertionFailedf("unhandled aggregate: %s", name)) diff --git a/pkg/sql/sem/builtins/aggregate_builtins.go b/pkg/sql/sem/builtins/aggregate_builtins.go index 5d0c31d84b19..24f3214e8d54 100644 --- a/pkg/sql/sem/builtins/aggregate_builtins.go +++ b/pkg/sql/sem/builtins/aggregate_builtins.go @@ -22,11 +22,13 @@ import ( "github.com/cockroachdb/cockroach/pkg/geo" "github.com/cockroachdb/cockroach/pkg/geo/geopb" "github.com/cockroachdb/cockroach/pkg/geo/geos" + "github.com/cockroachdb/cockroach/pkg/sql/appstatspb" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" + "github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/arith" "github.com/cockroachdb/cockroach/pkg/util/bitarray" @@ -259,6 +261,30 @@ var aggregates = map[string]builtinDefinition{ ), )), + "merge_stats_metadata": makeBuiltin(tree.FunctionProperties{ + Undocumented: true, + }, + makeAggOverload([]*types.T{types.Jsonb}, types.Jsonb, newAggStatementMetadata, + "Merges the meta data of the statistics.", volatility.Stable, true, /* calledOnNullInput */ + ), + ), + + "merge_statement_stats": makeBuiltin(tree.FunctionProperties{ + Undocumented: true, + }, + makeAggOverload([]*types.T{types.Jsonb}, types.Jsonb, newAggStatementStatistics, + "Merges the statistics data of the statement_statistics table.", volatility.Stable, true, /* calledOnNullInput */ + ), + ), + + "merge_transaction_stats": makeBuiltin(tree.FunctionProperties{ + Undocumented: true, + }, + makeAggOverload([]*types.T{types.Jsonb}, types.Jsonb, newAggTransactionStatistics, + "Merges the statistics data of the transaction_statistics table.", volatility.Stable, true, /* calledOnNullInput */ + ), + ), + "transition_regression_aggregate": makePrivate(makeTransitionRegressionAggregateBuiltin()), "covar_samp": makeRegressionAggregateBuiltin( @@ -1357,6 +1383,9 @@ const sizeOfSTMakeLineAggregate = int64(unsafe.Sizeof(stMakeLineAgg{})) const sizeOfSTUnionAggregate = int64(unsafe.Sizeof(stUnionAgg{})) const sizeOfSTCollectAggregate = int64(unsafe.Sizeof(stCollectAgg{})) const sizeOfSTExtentAggregate = int64(unsafe.Sizeof(stExtentAgg{})) +const sizeOfStatementStatistics = int64(unsafe.Sizeof(aggStatementStatistics{})) +const sizeOfAggregatedStatementMetadata = int64(unsafe.Sizeof(aggStatementMetadata{})) +const sizeOfTransactionStatistics = int64(unsafe.Sizeof(aggTransactionStatistics{})) // singleDatumAggregateBase is a utility struct that helps aggregate builtins // that store a single datum internally track their memory usage related to @@ -1562,6 +1591,246 @@ func (a *arrayAggregate) Size() int64 { return sizeOfArrayAggregate } +type aggStatementStatistics struct { + singleDatumAggregateBase + + stats appstatspb.StatementStatistics +} + +func newAggStatementStatistics( + params []*types.T, evalCtx *eval.Context, _ tree.Datums, +) eval.AggregateFunc { + return &aggStatementStatistics{ + singleDatumAggregateBase: makeSingleDatumAggregateBase(evalCtx), + } +} + +// Add the statistics information into a single object. +func (a *aggStatementStatistics) Add(ctx context.Context, datum tree.Datum, _ ...tree.Datum) error { + if datum == nil || datum == tree.DNull { + return nil + } + + // Rather than try to figure out how the size of a.stats object changes with + // each addition, we'll approximate its final memory usage as equal to the + // size of the last datum. + datumSize := int64(datum.Size()) + if err := a.updateMemoryUsage(ctx, datumSize); err != nil { + return err + } + + return mergeStatementStatsHelper(&a.stats, datum) +} + +// Result returns a copy of aggregated JSON object. +func (a *aggStatementStatistics) Result() (tree.Datum, error) { + aggregatedJSON, err := sqlstatsutil.BuildStmtStatisticsJSON(&a.stats) + if err != nil { + return nil, err + } + + return tree.NewDJSON(aggregatedJSON), nil +} + +// Reset implements eval.AggregateFunc interface. +func (a *aggStatementStatistics) Reset(ctx context.Context) { + a.reset(ctx) + a.stats.Reset() +} + +// Close allows the aggregate to release the memory it requested during +// operation. +func (a *aggStatementStatistics) Close(ctx context.Context) { + a.close(ctx) +} + +// Size is part of the eval.AggregateFunc interface. +func (a *aggStatementStatistics) Size() int64 { + return sizeOfStatementStatistics +} + +type aggStatementMetadata struct { + singleDatumAggregateBase + + stats appstatspb.AggregatedStatementMetadata +} + +func newAggStatementMetadata( + params []*types.T, evalCtx *eval.Context, _ tree.Datums, +) eval.AggregateFunc { + return &aggStatementMetadata{ + singleDatumAggregateBase: makeSingleDatumAggregateBase(evalCtx), + stats: appstatspb.AggregatedStatementMetadata{}, + } +} + +// Add the statistics and metadata to a single object. +func (a *aggStatementMetadata) Add(ctx context.Context, datum tree.Datum, _ ...tree.Datum) error { + if datum == nil || datum == tree.DNull { + return nil + } + + // Rather than try to figure out how the size of a.stats object changes with + // each addition, we'll approximate its final memory usage as equal to the + // size of the last datum. + datumSize := int64(datum.Size()) + if err := a.updateMemoryUsage(ctx, datumSize); err != nil { + return err + } + + return mergeStatsMetadataHelper(&a.stats, datum) +} + +// Result returns a copy of the aggregated json object. +func (a *aggStatementMetadata) Result() (tree.Datum, error) { + aggregatedJSON, err := sqlstatsutil.BuildStmtDetailsMetadataJSON(&a.stats) + if err != nil { + return nil, err + } + + return tree.NewDJSON(aggregatedJSON), nil +} + +// Reset implements eval.AggregateFunc interface. +func (a *aggStatementMetadata) Reset(ctx context.Context) { + a.stats.Reset() + a.reset(ctx) +} + +// Close allows the aggregate to release the memory it requested during +// operation. +func (a *aggStatementMetadata) Close(ctx context.Context) { + a.close(ctx) +} + +// Size is part of the eval.AggregateFunc interface. +func (a *aggStatementMetadata) Size() int64 { + return sizeOfAggregatedStatementMetadata +} + +type aggTransactionStatistics struct { + stats appstatspb.TransactionStatistics +} + +func newAggTransactionStatistics( + params []*types.T, evalCtx *eval.Context, _ tree.Datums, +) eval.AggregateFunc { + return &aggTransactionStatistics{} +} + +// Add the statistics to a single aggregated object. +func (a *aggTransactionStatistics) Add( + ctx context.Context, datum tree.Datum, _ ...tree.Datum, +) error { + if datum == nil || datum == tree.DNull { + return nil + } + + return mergeTransactionStatsHelper(&a.stats, datum) +} + +// Result returns a copy of aggregated JSON object. +func (a *aggTransactionStatistics) Result() (tree.Datum, error) { + aggregatedJSON, err := sqlstatsutil.BuildTxnStatisticsJSON( + &appstatspb.CollectedTransactionStatistics{ + Stats: a.stats, + }) + if err != nil { + return nil, err + } + + return tree.NewDJSON(aggregatedJSON), nil +} + +// Reset implements eval.AggregateFunc interface. +func (a *aggTransactionStatistics) Reset(ctx context.Context) { + a.stats.Reset() +} + +// Close allows the aggregate to release the memory it requested during +// operation. +func (a *aggTransactionStatistics) Close(ctx context.Context) { +} + +// Size is part of the eval.AggregateFunc interface. +func (a *aggTransactionStatistics) Size() int64 { + return sizeOfTransactionStatistics +} + +func mergeStatsMetadataHelper( + metadata *appstatspb.AggregatedStatementMetadata, metadataDatum tree.Datum, +) error { + if metadataDatum == tree.DNull { + return nil + } + + metadataJSON, ok := tree.AsDJSON(metadataDatum) + if !ok { + return nil + } + + var statistics appstatspb.CollectedStatementStatistics + + // Only decode and set the query info if it was not previously set. Avoid the + // overhead of parsing the query string which can be large. + if metadata.Query == "" || metadata.QuerySummary == "" { + err := sqlstatsutil.DecodeStmtStatsMetadataJSON(metadataJSON.JSON, &statistics) + if err != nil { + return err + } + } else { + err := sqlstatsutil.DecodeStmtStatsMetadataFlagsOnlyJSON(metadataJSON.JSON, &statistics) + if err != nil { + return err + } + } + + metadata.Add(&statistics) + return nil +} + +func mergeStatementStatsHelper( + aggregatedStats *appstatspb.StatementStatistics, statsDatum tree.Datum, +) error { + if statsDatum == tree.DNull { + return nil + } + + statsJSON, ok := tree.AsDJSON(statsDatum) + if !ok { + return nil + } + + var stats appstatspb.StatementStatistics + if err := sqlstatsutil.DecodeStmtStatsStatisticsJSON(statsJSON.JSON, &stats); err != nil { + return err + } + + aggregatedStats.Add(&stats) + return nil +} + +func mergeTransactionStatsHelper( + aggregatedStats *appstatspb.TransactionStatistics, statsDatum tree.Datum, +) error { + if statsDatum == tree.DNull { + return nil + } + + statsJSON, ok := tree.AsDJSON(statsDatum) + if !ok { + return nil + } + + var stats appstatspb.TransactionStatistics + if err := sqlstatsutil.DecodeTxnStatsStatisticsJSON(statsJSON.JSON, &stats); err != nil { + return err + } + + aggregatedStats.Add(&stats) + return nil +} + type arrayCatAggregate struct { arr *tree.DArray // Note that we do not embed singleDatumAggregateBase struct to help with diff --git a/pkg/sql/sem/builtins/builtins.go b/pkg/sql/sem/builtins/builtins.go index f217bb52794c..117070639e67 100644 --- a/pkg/sql/sem/builtins/builtins.go +++ b/pkg/sql/sem/builtins/builtins.go @@ -4279,16 +4279,10 @@ value if you rely on the HLC for accuracy.`, arr := tree.MustBeDArray(args[0]) var aggregatedStats appstatspb.StatementStatistics for _, statsDatum := range arr.Array { - if statsDatum == tree.DNull { - continue - } - var stats appstatspb.StatementStatistics - statsJSON := tree.MustBeDJSON(statsDatum).JSON - if err := sqlstatsutil.DecodeStmtStatsStatisticsJSON(statsJSON, &stats); err != nil { + err := mergeStatementStatsHelper(&aggregatedStats, statsDatum) + if err != nil { return nil, err } - - aggregatedStats.Add(&stats) } aggregatedJSON, err := sqlstatsutil.BuildStmtStatisticsJSON(&aggregatedStats) @@ -4313,13 +4307,10 @@ value if you rely on the HLC for accuracy.`, if statsDatum == tree.DNull { continue } - var stats appstatspb.TransactionStatistics - statsJSON := tree.MustBeDJSON(statsDatum).JSON - if err := sqlstatsutil.DecodeTxnStatsStatisticsJSON(statsJSON, &stats); err != nil { + err := mergeTransactionStatsHelper(&aggregatedStats, statsDatum) + if err != nil { return nil, err } - - aggregatedStats.Add(&stats) } aggregatedJSON, err := sqlstatsutil.BuildTxnStatisticsJSON( @@ -4349,31 +4340,10 @@ value if you rely on the HLC for accuracy.`, continue } - var statistics appstatspb.CollectedStatementStatistics - metadataJSON := tree.MustBeDJSON(metadataDatum).JSON - err := sqlstatsutil.DecodeStmtStatsMetadataJSON(metadataJSON, &statistics) + err := mergeStatsMetadataHelper(metadata, metadataDatum) if err != nil { return nil, err } - metadata.ImplicitTxn = statistics.Key.ImplicitTxn - metadata.Query = statistics.Key.Query - metadata.QuerySummary = statistics.Key.QuerySummary - metadata.StmtType = statistics.Stats.SQLType - metadata.Databases = util.CombineUniqueString(metadata.Databases, []string{statistics.Key.Database}) - - if statistics.Key.DistSQL { - metadata.DistSQLCount++ - } - if statistics.Key.Failed { - metadata.FailedCount++ - } - if statistics.Key.FullScan { - metadata.FullScanCount++ - } - if statistics.Key.Vec { - metadata.VecCount++ - } - metadata.TotalCount++ } aggregatedJSON, err := sqlstatsutil.BuildStmtDetailsMetadataJSON(metadata) if err != nil { diff --git a/pkg/sql/sem/builtins/fixed_oids.go b/pkg/sql/sem/builtins/fixed_oids.go index 8d054ec98dd3..da20701e46c3 100644 --- a/pkg/sql/sem/builtins/fixed_oids.go +++ b/pkg/sql/sem/builtins/fixed_oids.go @@ -2416,6 +2416,9 @@ var builtinOidsArray = []string{ 2487: `make_timestamptz(year: int, month: int, day: int, hour: int, min: int, sec: float, timezone: string) -> timestamptz`, 2488: `date_trunc(element: string, input: timestamptz, timezone: string) -> timestamptz`, 2489: `make_date(year: int, month: int, day: int) -> date`, + 2511: `merge_stats_metadata(arg1: jsonb) -> jsonb`, + 2512: `merge_statement_stats(arg1: jsonb) -> jsonb`, + 2513: `merge_transaction_stats(arg1: jsonb) -> jsonb`, } var builtinOidsBySignature map[string]oid.Oid diff --git a/pkg/sql/sql_activity_update_job_test.go b/pkg/sql/sql_activity_update_job_test.go index e6bf91f542a2..1a5b12468c8b 100644 --- a/pkg/sql/sql_activity_update_job_test.go +++ b/pkg/sql/sql_activity_update_job_test.go @@ -22,12 +22,16 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/appstatspb" "github.com/cockroachdb/cockroach/pkg/sql/sqlstats" "github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats" + "github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" + "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/upgrade/upgradebase" + jsonUtil "github.com/cockroachdb/cockroach/pkg/util/json" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" @@ -159,12 +163,128 @@ func TestSqlActivityUpdateJob(t *testing.T) { require.Zero(t, count, "crdb_internal.statement_activity after transfer: expect:0, actual:%d", count) } +// TestMergeFunctionLogic verifies the merge functions used in the +// SQL statements to verify the data. +func TestMergeFunctionLogic(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + ctx := context.Background() + + stubTime := timeutil.Now().Truncate(time.Hour) + sqlStatsKnobs := &sqlstats.TestingKnobs{ + StubTimeNow: func() time.Time { return stubTime }, + AOSTClause: "AS OF SYSTEM TIME '-1us'", + } + + // Start the cluster. + // Disable the job since it is called manually from a new instance to avoid + // any race conditions. + srv, sqlDB, _ := serverutils.StartServer(t, base.TestServerArgs{ + Insecure: true, + Knobs: base.TestingKnobs{ + SQLStatsKnobs: sqlStatsKnobs, + UpgradeManager: &upgradebase.TestingKnobs{ + DontUseJobs: true, + SkipUpdateSQLActivityJobBootstrap: true, + }}}) + defer srv.Stopper().Stop(context.Background()) + + db := sqlutils.MakeSQLRunner(sqlDB) + + appName := "TestMergeFunctionLogic" + db.Exec(t, "SET SESSION application_name=$1", appName) + db.Exec(t, "SELECT * FROM system.statement_statistics") + db.Exec(t, "SELECT * FROM system.statement_statistics") + db.Exec(t, "SELECT count_rows() FROM system.transaction_statistics") + + srv.SQLServer().(*Server).GetSQLStatsProvider().(*persistedsqlstats.PersistedSQLStats).Flush(ctx) + + db.Exec(t, "SET SESSION application_name=$1", "randomIgnore") + + var localAggTxnStats appstatspb.TransactionStatistics + rows := db.Query(t, "SELECT statistics FROM system.public.transaction_statistics WHERE app_name = $1", appName) + require.NoError(t, rows.Err()) + defer rows.Close() + for rows.Next() { + var tempStats appstatspb.TransactionStatistics + var jsonString string + require.NoError(t, rows.Scan(&jsonString)) + j, err := jsonUtil.ParseJSON(jsonString) + require.NoError(t, err) + require.NoError(t, sqlstatsutil.DecodeTxnStatsStatisticsJSON(j, &tempStats)) + require.Greater(t, tempStats.Count, int64(0), "empty object: json:%s\n obj:%v\n", jsonString, tempStats) + localAggTxnStats.Add(&tempStats) + } + + require.Equal(t, int64(3), localAggTxnStats.Count) + + row := db.QueryRow(t, `SELECT merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics WHERE app_name = $1 GROUP BY app_name`, appName) + var aggSqlTxnStats appstatspb.TransactionStatistics + var jsonString string + row.Scan(&jsonString) + j, err := jsonUtil.ParseJSON(jsonString) + require.NoError(t, err) + require.NoError(t, sqlstatsutil.DecodeTxnStatsStatisticsJSON(j, &aggSqlTxnStats)) + require.Equal(t, localAggTxnStats, aggSqlTxnStats) + + var localAggStmtStats appstatspb.StatementStatistics + rows = db.Query(t, "SELECT statistics FROM system.public.statement_statistics WHERE app_name = $1", appName) + require.NoError(t, rows.Err()) + defer rows.Close() + for rows.Next() { + var tempStats appstatspb.StatementStatistics + require.NoError(t, rows.Scan(&jsonString)) + j, err = jsonUtil.ParseJSON(jsonString) + require.NoError(t, err) + require.NoError(t, sqlstatsutil.DecodeStmtStatsStatisticsJSON(j, &tempStats)) + require.Greater(t, tempStats.Count, int64(0), "empty object: json:%s\n obj:%v\n", jsonString, tempStats) + localAggStmtStats.Add(&tempStats) + } + require.Equal(t, int64(3), localAggTxnStats.Count) + + row = db.QueryRow(t, `SELECT merge_statement_stats(statistics) AS statistics FROM system.public.statement_statistics WHERE app_name = $1 GROUP BY app_name`, appName) + var aggStmtStat appstatspb.StatementStatistics + row.Scan(&jsonString) + j, err = jsonUtil.ParseJSON(jsonString) + require.NoError(t, err) + require.NoError(t, sqlstatsutil.DecodeStmtStatsStatisticsJSON(j, &aggStmtStat)) + require.Equal(t, localAggStmtStats, aggStmtStat) + + // Verify metadata logic + var localAggStmtMeta appstatspb.AggregatedStatementMetadata + rows = db.Query(t, "SELECT metadata FROM system.public.statement_statistics WHERE app_name = $1", appName) + require.NoError(t, rows.Err()) + defer rows.Close() + for rows.Next() { + var tempStats appstatspb.CollectedStatementStatistics + require.NoError(t, rows.Scan(&jsonString)) + j, err = jsonUtil.ParseJSON(jsonString) + require.NoError(t, err) + require.NoError(t, sqlstatsutil.DecodeStmtStatsMetadataJSON(j, &tempStats)) + require.NotNil(t, tempStats.Key.Query, "empty object: json:%s\n obj:%v\n", jsonString, tempStats) + localAggStmtMeta.Add(&tempStats) + } + + require.Equal(t, int64(2), localAggStmtMeta.TotalCount) + + row = db.QueryRow(t, `SELECT merge_stats_metadata(metadata) AS metadata FROM system.public.statement_statistics WHERE app_name = $1 GROUP BY app_name`, appName) + var aggSqlStmtMeta appstatspb.AggregatedStatementMetadata + row.Scan(&jsonString) + j, err = jsonUtil.ParseJSON(jsonString) + require.NoError(t, err) + require.NoError(t, sqlstatsutil.DecodeAggregatedMetadataJSON(j, &aggSqlStmtMeta)) + require.Equal(t, localAggStmtMeta, aggSqlStmtMeta) +} + // TestSqlActivityUpdateTopLimitJob verifies that the // job is created. func TestSqlActivityUpdateTopLimitJob(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderStressRace(t, "test is too slow to run under race") + stubTime := timeutil.Now().Truncate(time.Hour) sqlStatsKnobs := &sqlstats.TestingKnobs{ StubTimeNow: func() time.Time { return stubTime }, @@ -239,7 +359,9 @@ func TestSqlActivityUpdateTopLimitJob(t *testing.T) { db.Exec(t, "SET SESSION application_name=$1", "randomIgnore") + db.Exec(t, "set cluster setting sql.stats.flush.enabled = true;") srv.SQLServer().(*Server).GetSQLStatsProvider().(*persistedsqlstats.PersistedSQLStats).Flush(ctx) + db.Exec(t, "set cluster setting sql.stats.flush.enabled = false;") // The max number of queries is number of top columns * max number of // queries per a column (6*3=18 for this test, 6*500=3000 default). Most of diff --git a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_decoding.go b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_decoding.go index 50b1035631c1..7f632cff2ef4 100644 --- a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_decoding.go +++ b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_decoding.go @@ -45,6 +45,16 @@ func DecodeStmtStatsMetadataJSON( return (*stmtStatsMetadata)(result).jsonFields().decodeJSON(metadata) } +// DecodeStmtStatsMetadataFlagsOnlyJSON decodes the 'metadata' flags only fields +// of the JSON representation of the statement statistics into +// appstatspb.CollectedStatementStatistics. This avoids the overhead of query +// string and query summary decoding. +func DecodeStmtStatsMetadataFlagsOnlyJSON( + metadata json.JSON, result *appstatspb.CollectedStatementStatistics, +) error { + return (*stmtStatsMetadata)(result).jsonFlagsOnlyFields().decodeJSON(metadata) +} + // DecodeAggregatedMetadataJSON decodes the 'aggregated metadata' represented by appstatspb.AggregatedStatementMetadata. func DecodeAggregatedMetadataJSON( metadata json.JSON, result *appstatspb.AggregatedStatementMetadata, diff --git a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go index 06b5c0326a87..549a150814f2 100644 --- a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go +++ b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go @@ -104,6 +104,17 @@ func (s *stmtStatsMetadata) jsonFields() jsonFields { } } +func (s *stmtStatsMetadata) jsonFlagsOnlyFields() jsonFields { + return jsonFields{ + {"db", (*jsonString)(&s.Key.Database)}, + {"distsql", (*jsonBool)(&s.Key.DistSQL)}, + {"failed", (*jsonBool)(&s.Key.Failed)}, + {"implicitTxn", (*jsonBool)(&s.Key.ImplicitTxn)}, + {"vec", (*jsonBool)(&s.Key.Vec)}, + {"fullScan", (*jsonBool)(&s.Key.FullScan)}, + } +} + type aggregatedMetadata appstatspb.AggregatedStatementMetadata func (s *aggregatedMetadata) jsonFields() jsonFields { From a9580456c5297effecf1a46989770dbf6e071243 Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Thu, 12 Oct 2023 15:09:06 -0700 Subject: [PATCH 2/2] sql: use newly added aggregates for sql activity jobs We introduced these in #111303, but in the last revision of that PR I think by mistake the changes to the actual job queries were reverted. Epic: None Release note: None --- pkg/sql/sql_activity_update_job.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/sql/sql_activity_update_job.go b/pkg/sql/sql_activity_update_job.go index fe4938be7e85..b8464c8ce5b5 100644 --- a/pkg/sql/sql_activity_update_job.go +++ b/pkg/sql/sql_activity_update_job.go @@ -257,7 +257,7 @@ func (u *sqlActivityUpdater) transferAllStats( fingerprint_id, agg_interval, max(metadata) as metadata, - crdb_internal.merge_transaction_stats(array_agg(statistics)) AS statistics + merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics WHERE aggregated_ts = $2 and app_name not like '$ internal%' @@ -310,8 +310,8 @@ INTO system.public.statement_activity (aggregated_ts, fingerprint_id, transactio plan_hash, app_name, agg_interval, - crdb_internal.merge_stats_metadata(array_agg(metadata)) AS metadata, - crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics, + merge_stats_metadata(metadata) AS metadata, + merge_statement_stats(statistics) AS statistics, plan, index_recommendations FROM system.public.statement_statistics @@ -401,7 +401,7 @@ INTO system.public.transaction_activity ts.fingerprint_id, ts.agg_interval, max(ts.metadata) AS metadata, - crdb_internal.merge_transaction_stats(array_agg(statistics)) AS statistics + merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics ts inner join (SELECT fingerprint_id, app_name, agg_interval FROM (SELECT fingerprint_id, app_name, agg_interval, @@ -422,7 +422,7 @@ INTO system.public.transaction_activity (statistics -> 'statistics' -> 'latencyInfo' ->> 'p99')::float, 0) desc) AS lPos FROM (SELECT fingerprint_id, app_name, agg_interval, - crdb_internal.merge_transaction_stats(array_agg(statistics)) AS statistics + merge_transaction_stats(statistics) AS statistics FROM system.public.transaction_statistics WHERE aggregated_ts = $2 and app_name not like '$ internal%' @@ -514,8 +514,8 @@ INTO system.public.statement_activity ss.plan_hash, ss.app_name, ss.agg_interval, - crdb_internal.merge_stats_metadata(array_agg(ss.metadata)) AS metadata, - crdb_internal.merge_statement_stats(array_agg(ss.statistics)) AS statistics, + merge_stats_metadata(ss.metadata) AS metadata, + merge_statement_stats(ss.statistics) AS statistics, ss.plan, ss.index_recommendations FROM system.public.statement_statistics ss @@ -539,7 +539,7 @@ INTO system.public.statement_activity 0) desc) AS lPos FROM (SELECT fingerprint_id, app_name, - crdb_internal.merge_statement_stats(array_agg(statistics)) AS statistics + merge_statement_stats(statistics) AS statistics FROM system.public.statement_statistics WHERE aggregated_ts = $2 and app_name not like '$ internal%' @@ -632,8 +632,8 @@ FROM (SELECT max(ss.aggregated_ts) AS aggregated_ts, ss.plan_hash, ss.app_name, ss.agg_interval, - crdb_internal.merge_stats_metadata(array_agg(ss.metadata)) AS metadata, - crdb_internal.merge_statement_stats(array_agg(ss.statistics)) AS statistics, + merge_stats_metadata(ss.metadata) AS metadata, + merge_statement_stats(ss.statistics) AS statistics, ss.plan, ss.index_recommendations FROM system.public.statement_statistics ss