sentry: bytes_usage.go:397: flow: unexpected 286720 leftover bytes | string

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/1958837915/?referrer=webhooks_plugin

Panic message:

bytes_usage.go:397: flow: unexpected 286720 leftover bytes | string

Stacktrace (expand for inline code snippets):

cockroach/pkg/util/mon/bytes_usage.go

Lines 396 to 398 in be8c0a7

	if check && mm.mu.curAllocated != 0 {
	log.ReportOrPanic(
	ctx, &mm.settings.SV,

in pkg/util/mon.(*BytesMonitor).doStop

cockroach/pkg/util/mon/bytes_usage.go

Lines 383 to 385 in be8c0a7

	func (mm *BytesMonitor) Stop(ctx context.Context) {
	mm.doStop(ctx, true)
	}

in pkg/util/mon.(*BytesMonitor).Stop

cockroach/pkg/sql/sem/tree/eval.go

Lines 2931 to 2933 in be8c0a7

	func (ctx *EvalContext) Stop(c context.Context) {
	ctx.Mon.Stop(c)
	}

in pkg/sql/sem/tree.(*EvalContext).Stop

cockroach/pkg/sql/flowinfra/flow.go

Lines 422 to 424 in be8c0a7

	// This closes the monitor opened in ServerImpl.setupFlow.
	f.EvalCtx.Stop(ctx)
	for _, p := range f.processors {

in pkg/sql/flowinfra.(*FlowBase).Cleanup

cockroach/pkg/sql/rowflow/row_based_flow.go

Lines 413 to 415 in be8c0a7

	func (f *rowBasedFlow) Cleanup(ctx context.Context) {
	f.FlowBase.Cleanup(ctx)
	f.Release()

in pkg/sql/rowflow.(*rowBasedFlow).Cleanup

cockroach/pkg/sql/distsql_running.go

Lines 421 to 423 in be8c0a7

	curPlan.close(ctx)
	flow.Cleanup(ctx)
	}

in pkg/sql.(*DistSQLPlanner).Run.func8

cockroach/pkg/sql/conn_executor_exec.go

Lines 945 to 947 in be8c0a7

	if recv.commErr != nil || res.Err() != nil {
	return recv.bytesRead, recv.rowsRead, recv.commErr
	}

in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 831 to 833 in be8c0a7

	ex.sessionTracing.TraceExecStart(ctx, "distributed")
	bytesRead, rowsRead, err := ex.execWithDistSQLEngine(ctx, planner, stmt.AST.StatementType(), res, distributePlan, progAtomic)
	ex.sessionTracing.TraceExecEnd(ctx, res.Err(), res.RowsAffected())

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 536 to 538 in be8c0a7

	p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	return nil, nil, err

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 95 to 97 in be8c0a7

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1405 to 1407 in be8c0a7

	stmtCtx := withStatement(ctx, ex.curStmt)
	ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
	if err != nil {

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1334 to 1336 in be8c0a7

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if err == io.EOF || err == errDrainingComplete {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 478 to 480 in be8c0a7

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 581 to 583 in be8c0a7

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1

pkg/util/mon/bytes_usage.go in pkg/util/mon.(*BytesMonitor).doStop at line 397
pkg/util/mon/bytes_usage.go in pkg/util/mon.(*BytesMonitor).Stop at line 384
pkg/sql/sem/tree/eval.go in pkg/sql/sem/tree.(*EvalContext).Stop at line 2932
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Cleanup at line 423
pkg/sql/rowflow/row_based_flow.go in pkg/sql/rowflow.(*rowBasedFlow).Cleanup at line 414
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run.func8 at line 422
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 946
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 832
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 537
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 96
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1406
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1335
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 479
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 582

Tag	Value
Cockroach Release	v20.1.6
Cockroach SHA:	be8c0a7
Platform	linux amd64
Distribution	CCL
Environment	v20.1.6
Command	server
Go Version	go1.13.9
# of CPUs	2
# of Goroutines	463

[yuzefovich]

dup of #55408