From 7bedcafe137cb7139aa6cbc425fd188409e6f216 Mon Sep 17 00:00:00 2001 From: Matt Jibson Date: Mon, 22 Jun 2020 13:14:24 -0600 Subject: [PATCH] sql: add telemetry for OOM during stats histogram generation With the addition of inverted stats, there's more memory usage during histogram collection. Record when an OOM occurs so we can decide whether to improve this situation. For example, we could split up histogram generation to be per column instead of all columns at the same time. Release note: None --- pkg/sql/rowexec/sample_aggregator.go | 3 +++ pkg/sql/rowexec/sampler.go | 3 +++ pkg/sql/sqltelemetry/planning.go | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/pkg/sql/rowexec/sample_aggregator.go b/pkg/sql/rowexec/sample_aggregator.go index 7bee6273ce41..c498c5e315a6 100644 --- a/pkg/sql/rowexec/sample_aggregator.go +++ b/pkg/sql/rowexec/sample_aggregator.go @@ -17,12 +17,14 @@ import ( "github.com/axiomhq/hyperloglog" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/kv" + "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/sql/sqltelemetry" "github.com/cockroachdb/cockroach/pkg/sql/stats" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util" @@ -371,6 +373,7 @@ func (s *sampleAggregator) sampleRow( // disable histogram sample collection. sr.Disable() log.Info(ctx, "disabling histogram collection due to excessive memory utilization") + telemetry.Inc(sqltelemetry.StatsHistogramOOMCounter) } return nil } diff --git a/pkg/sql/rowexec/sampler.go b/pkg/sql/rowexec/sampler.go index db50e9a63388..3342adb1f9a5 100644 --- a/pkg/sql/rowexec/sampler.go +++ b/pkg/sql/rowexec/sampler.go @@ -18,12 +18,14 @@ import ( "github.com/axiomhq/hyperloglog" "github.com/cockroachdb/cockroach/pkg/geo/geoindex" + "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/sql/sqltelemetry" "github.com/cockroachdb/cockroach/pkg/sql/stats" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util" @@ -459,6 +461,7 @@ func (s *samplerProcessor) sampleRow( // disable histogram sample collection. sr.Disable() log.Info(ctx, "disabling histogram collection due to excessive memory utilization") + telemetry.Inc(sqltelemetry.StatsHistogramOOMCounter) // Send a metadata record so the sample aggregator will also disable // histogram collection. diff --git a/pkg/sql/sqltelemetry/planning.go b/pkg/sql/sqltelemetry/planning.go index 91edc34f2bd2..c9ac1bc13731 100644 --- a/pkg/sql/sqltelemetry/planning.go +++ b/pkg/sql/sqltelemetry/planning.go @@ -113,6 +113,10 @@ var TurnAutoStatsOnUseCounter = telemetry.GetCounterOnce("sql.plan.automatic-sta // collection is explicitly disabled. var TurnAutoStatsOffUseCounter = telemetry.GetCounterOnce("sql.plan.automatic-stats.disabled") +// StatsHistogramOOMCounter is to be incremented whenever statistics histogram +// generation is disabled due to an out of memory error. +var StatsHistogramOOMCounter = telemetry.GetCounterOnce("sql.plan.stats.histogram-oom") + // JoinAlgoHashUseCounter is to be incremented whenever a hash join node is // planned. var JoinAlgoHashUseCounter = telemetry.GetCounterOnce("sql.plan.opt.node.join.algo.hash")