Merge #62006

62006: sql: add execution stats to node_statement_statistics r=yuzefovich a=asubiotto Release justification: low-risk, high-benefit change to existing functionality since this commit exposes pre-existing execution stats through the SQL shell. Release note (sql change): sampled execution stats are now available through node_statement_statistics. Closes #61637 Putting @yuzefovich as primary reviewer but @maryliag and @Azhng, please take a look and let me know if you have any questions. Co-authored-by: Alfonso Subiotto Marques <[email protected]>
cockroachdb · Mar 16, 2021 · 98ae002 · 98ae002
2 parents db4f939 + 68d2e66
commit 98ae002
Show file tree

Hide file tree

Showing 4 changed files with 182 additions and 84 deletions.
diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go
@@ -810,6 +810,24 @@ func (t txnList) Less(i, j int) bool {
 	return t[i] < t[j]
 }
 
+// execStatAvg is a helper for execution stats shown in virtual tables. Returns
+// NULL when the count is 0, or the mean of the given NumericStat.
+func execStatAvg(count int64, n roachpb.NumericStat) tree.Datum {
+	if count == 0 {
+		return tree.DNull
+	}
+	return tree.NewDFloat(tree.DFloat(n.Mean))
+}
+
+// execStatVar is a helper for execution stats shown in virtual tables. Returns
+// NULL when the count is 0, or the variance of the given NumericStat.
+func execStatVar(count int64, n roachpb.NumericStat) tree.Datum {
+	if count == 0 {
+		return tree.DNull
+	}
+	return tree.NewDFloat(tree.DFloat(n.GetVariance(count)))
+}
+
 var crdbInternalStmtStatsTable = virtualSchemaTable{
 	comment: `statement statistics (in-memory, not durable; local node only). ` +
 		`This table is wiped periodically (by default, at least every two hours)`,
@@ -840,6 +858,16 @@ CREATE TABLE crdb_internal.node_statement_statistics (
   bytes_read_var      FLOAT NOT NULL,
   rows_read_avg       FLOAT NOT NULL,
   rows_read_var       FLOAT NOT NULL,
+  network_bytes_avg   FLOAT,
+  network_bytes_var   FLOAT,
+  network_msgs_avg    FLOAT,
+  network_msgs_var    FLOAT,
+  max_mem_usage_avg   FLOAT,
+  max_mem_usage_var   FLOAT,
+  max_disk_usage_avg  FLOAT,
+  max_disk_usage_var  FLOAT,
+  contention_time_avg FLOAT,
+  contention_time_var FLOAT,
   implicit_txn        BOOL NOT NULL
 )`,
 	populate: func(ctx context.Context, p *planner, _ *dbdesc.Immutable, addRow func(...tree.Datum) error) error {
@@ -908,32 +936,42 @@ CREATE TABLE crdb_internal.node_statement_statistics (
 					flags = "!" + flags
 				}
 				err := addRow(
-					tree.NewDInt(tree.DInt(nodeID)),
-					tree.NewDString(appName),
-					tree.NewDString(flags),
-					tree.NewDString(stmtKey.anonymizedStmt),
-					anonymized,
-					tree.NewDInt(tree.DInt(s.mu.data.Count)),
-					tree.NewDInt(tree.DInt(s.mu.data.FirstAttemptCount)),
-					tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),
-					errString,
-					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.GetVariance(s.mu.data.Count))),
-					tree.MakeDBool(tree.DBool(stmtKey.implicitTxn)),
+					tree.NewDInt(tree.DInt(nodeID)),                      // node_id
+					tree.NewDString(appName),                             // application_name
+					tree.NewDString(flags),                               // flags
+					tree.NewDString(stmtKey.anonymizedStmt),              // key
+					anonymized,                                           // anonymized
+					tree.NewDInt(tree.DInt(s.mu.data.Count)),             // count
+					tree.NewDInt(tree.DInt(s.mu.data.FirstAttemptCount)), // first_attempt_count
+					tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),        // max_retries
+					errString, // last_error
+					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),                             // rows_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),     // rows_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.Mean)),                            // parse_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.GetVariance(s.mu.data.Count))),    // parse_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.Mean)),                             // plan_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.GetVariance(s.mu.data.Count))),     // plan_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.Mean)),                              // run_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.GetVariance(s.mu.data.Count))),      // run_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),                          // service_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))),  // service_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.Mean)),                         // overhead_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.GetVariance(s.mu.data.Count))), // overhead_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.Mean)),                           // bytes_read_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.GetVariance(s.mu.data.Count))),   // bytes_read_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.Mean)),                            // rows_read_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.GetVariance(s.mu.data.Count))),    // rows_read_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes),        // network_bytes_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes),        // network_bytes_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages),     // network_msgs_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages),     // network_msgs_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage),         // max_mem_usage_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage),         // max_mem_usage_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage),        // max_disk_usage_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage),        // max_disk_usage_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime),      // contention_time_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime),      // contention_time_var
+					tree.MakeDBool(tree.DBool(stmtKey.implicitTxn)),                                 // implicit_txn
 				)
 				s.mu.Unlock()
 				if err != nil {
@@ -953,20 +991,30 @@ var crdbInternalTransactionStatisticsTable = virtualSchemaTable{
 		`This table is wiped periodically (by default, at least every two hours)`,
 	schema: `
 CREATE TABLE crdb_internal.node_transaction_statistics (
-  node_id           INT NOT NULL,
-  application_name  STRING NOT NULL,
-  key               STRING,
-  statement_ids     STRING[],
-  count             INT,
-  max_retries       INT,
-  service_lat_avg   FLOAT NOT NULL,
-  service_lat_var   FLOAT NOT NULL,
-  retry_lat_avg     FLOAT NOT NULL,
-  retry_lat_var     FLOAT NOT NULL,
-  commit_lat_avg    FLOAT NOT NULL,
-  commit_lat_var    FLOAT NOT NULL,
-  rows_read_avg     FLOAT NOT NULL,
-  rows_read_var     FLOAT NOT NULL
+  node_id             INT NOT NULL,
+  application_name    STRING NOT NULL,
+  key                 STRING,
+  statement_ids       STRING[],
+  count               INT,
+  max_retries         INT,
+  service_lat_avg     FLOAT NOT NULL,
+  service_lat_var     FLOAT NOT NULL,
+  retry_lat_avg       FLOAT NOT NULL,
+  retry_lat_var       FLOAT NOT NULL,
+  commit_lat_avg      FLOAT NOT NULL,
+  commit_lat_var      FLOAT NOT NULL,
+  rows_read_avg       FLOAT NOT NULL,
+  rows_read_var       FLOAT NOT NULL,
+  network_bytes_avg   FLOAT,
+  network_bytes_var   FLOAT,
+  network_msgs_avg    FLOAT,
+  network_msgs_var    FLOAT,
+  max_mem_usage_avg   FLOAT,
+  max_mem_usage_var   FLOAT,
+  max_disk_usage_avg  FLOAT,
+  max_disk_usage_var  FLOAT,
+  contention_time_avg FLOAT, 
+  contention_time_var FLOAT
 )
 `,
 	populate: func(ctx context.Context, p *planner, _ *dbdesc.Immutable, addRow func(...tree.Datum) error) error {
@@ -1032,20 +1080,30 @@ CREATE TABLE crdb_internal.node_transaction_statistics (
 				s.mu.Lock()
 
 				err := addRow(
-					tree.NewDInt(tree.DInt(nodeID)),
-					tree.NewDString(appName),
-					tree.NewDString(strconv.FormatUint(uint64(txnKey), 10)),
-					stmtIDsDatum,
-					tree.NewDInt(tree.DInt(s.mu.data.Count)),
-					tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.GetVariance(s.mu.data.Count))),
-					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),
-					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),
+					tree.NewDInt(tree.DInt(nodeID)),                         // node_id
+					tree.NewDString(appName),                                // application_name
+					tree.NewDString(strconv.FormatUint(uint64(txnKey), 10)), // key
+					stmtIDsDatum,                                                                   // statement_ids
+					tree.NewDInt(tree.DInt(s.mu.data.Count)),                                       // count
+					tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),                                  // max_retries
+					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),                         // service_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))), // service_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.Mean)),                           // retry_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.GetVariance(s.mu.data.Count))),   // retry_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.Mean)),                          // commit_lat_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.GetVariance(s.mu.data.Count))),  // commit_lat_var
+					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),                            // rows_read_avg
+					tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),    // rows_read_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes),       // network_bytes_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes),       // network_bytes_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages),    // network_msgs_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages),    // network_msgs_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage),        // max_mem_usage_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage),        // max_mem_usage_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage),       // max_disk_usage_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage),       // max_disk_usage_var
+					execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime),     // contention_time_avg
+					execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime),     // contention_time_var
 				)
 
 				s.mu.Unlock()

diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal b/pkg/sql/logictest/testdata/logic_test/crdb_internal
@@ -153,15 +153,15 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
 ----
 node_id  table_id  name  parent_id  expiration  deleted
 
-query ITTTTIIITRRRRRRRRRRRRRRRRR colnames
+query ITTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
 ----
-node_id  application_name  flags  key  anonymized  count  first_attempt_count  max_retries  last_error  rows_avg  rows_var  parse_lat_avg  parse_lat_var  plan_lat_avg  plan_lat_var  run_lat_avg  run_lat_var  service_lat_avg  service_lat_var  overhead_lat_avg  overhead_lat_var  bytes_read_avg  bytes_read_var  rows_read_avg  rows_read_var  implicit_txn
+node_id  application_name  flags  key  anonymized  count  first_attempt_count  max_retries  last_error  rows_avg  rows_var  parse_lat_avg  parse_lat_var  plan_lat_avg  plan_lat_var  run_lat_avg  run_lat_var  service_lat_avg  service_lat_var  overhead_lat_avg  overhead_lat_var  bytes_read_avg  bytes_read_var  rows_read_avg  rows_read_var  network_bytes_avg  network_bytes_var  network_msgs_avg  network_msgs_var  max_mem_usage_avg  max_mem_usage_var  max_disk_usage_avg  max_disk_usage_var  contention_time_avg  contention_time_var  implicit_txn
 
-query ITTTIIRRRRRRRR colnames
+query ITTTIIRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
 ----
-node_id  application_name  key  statement_ids  count  max_retries  service_lat_avg  service_lat_var  retry_lat_avg  retry_lat_var  commit_lat_avg  commit_lat_var  rows_read_avg  rows_read_var
+node_id  application_name  key  statement_ids  count  max_retries  service_lat_avg  service_lat_var  retry_lat_avg  retry_lat_var  commit_lat_avg  commit_lat_var  rows_read_avg  rows_read_var  network_bytes_avg  network_bytes_var  network_msgs_avg  network_msgs_var  max_mem_usage_avg  max_mem_usage_var  max_disk_usage_avg  max_disk_usage_var  contention_time_avg  contention_time_var
 
 query IITTTTTTT colnames
 SELECT * FROM crdb_internal.session_trace WHERE span_idx < 0

diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant b/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant
@@ -165,15 +165,15 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
 ----
 node_id  table_id  name  parent_id  expiration  deleted
 
-query ITTTTIIITRRRRRRRRRRRRRRRRR colnames
+query ITTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
 ----
-node_id  application_name  flags  key  anonymized  count  first_attempt_count  max_retries  last_error  rows_avg  rows_var  parse_lat_avg  parse_lat_var  plan_lat_avg  plan_lat_var  run_lat_avg  run_lat_var  service_lat_avg  service_lat_var  overhead_lat_avg  overhead_lat_var  bytes_read_avg  bytes_read_var  rows_read_avg  rows_read_var  implicit_txn
+node_id  application_name  flags  key  anonymized  count  first_attempt_count  max_retries  last_error  rows_avg  rows_var  parse_lat_avg  parse_lat_var  plan_lat_avg  plan_lat_var  run_lat_avg  run_lat_var  service_lat_avg  service_lat_var  overhead_lat_avg  overhead_lat_var  bytes_read_avg  bytes_read_var  rows_read_avg  rows_read_var  network_bytes_avg  network_bytes_var  network_msgs_avg  network_msgs_var  max_mem_usage_avg  max_mem_usage_var  max_disk_usage_avg  max_disk_usage_var  contention_time_avg  contention_time_var  implicit_txn
 
-query ITTTIIRRRRRRRR colnames
+query ITTTIIRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
 ----
-node_id  application_name  key  statement_ids  count  max_retries  service_lat_avg  service_lat_var  retry_lat_avg  retry_lat_var  commit_lat_avg  commit_lat_var  rows_read_avg  rows_read_var
+node_id  application_name  key  statement_ids  count  max_retries  service_lat_avg  service_lat_var  retry_lat_avg  retry_lat_var  commit_lat_avg  commit_lat_var  rows_read_avg  rows_read_var  network_bytes_avg  network_bytes_var  network_msgs_avg  network_msgs_var  max_mem_usage_avg  max_mem_usage_var  max_disk_usage_avg  max_disk_usage_var  contention_time_avg  contention_time_var
 
 query IITTTTTTT colnames
 SELECT * FROM crdb_internal.session_trace WHERE span_idx < 0