Sentry: crash_reporting.go:390: flow 98ccc2bf: unexpected 51200 leftover bytes (1) attached stack trace -- stack trace: | github.com/cockroachdb/cockroach/pkg/util/log/logcrash.ReportOrPanic | ...

[cockroach-sentry]

This issue was auto filed by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry Link: https://cockroach-labs.sentry.io/issues/4472811959/?referrer=webhooks_plugin

Panic Message:

crash_reporting.go:390: flow 98ccc2bf: unexpected 51200 leftover bytes
(1) attached stack trace
  -- stack trace:
  | github.com/cockroachdb/cockroach/pkg/util/log/logcrash.ReportOrPanic
  | 	github.com/cockroachdb/cockroach/pkg/util/log/logcrash/crash_reporting.go:390
  | github.com/cockroachdb/cockroach/pkg/util/mon.(*BytesMonitor).doStop
  | 	github.com/cockroachdb/cockroach/pkg/util/mon/bytes_usage.go:592
  | github.com/cockroachdb/cockroach/pkg/util/mon.(*BytesMonitor).Stop
  | 	github.com/cockroachdb/cockroach/pkg/util/mon/bytes_usage.go:565
  | github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Cleanup
  | 	github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:699
  | github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Cleanup
  | 	github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:416
  | github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll.func4
  | 	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1662
  | github.com/cockroachdb/cockroach/pkg/sql.(*cleanupFuncStack).run
  | 	github.com/cockroachdb/cockroach/pkg/sql/prepared_stmt.go:251
  | github.com/cockroachdb/cockroach/pkg/sql.(*portalPauseInfo).cleanupAll
  | 	github.com/cockroachdb/cockroach/pkg/sql/prepared_stmt.go:373
  | github.com/cockroachdb/cockroach/pkg/sql.(*PreparedPortal).close
  | 	github.com/cockroachdb/cockroach/pkg/sql/prepared_stmt.go:224
  | github.com/cockroachdb/cockroach/pkg/sql.(*prepStmtNamespace).closeAllPortals
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1814
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1301
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1234
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:768
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:142
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2986
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:141
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2181
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2186
  | github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2103
  | github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn
  | 	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:900
  | github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1
  | 	github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:310
  | runtime.goexit
  | 	src/runtime/asm_arm64.s:1172
Wraps: (2) flow 98ccc2bf: unexpected 51200 leftover bytes
Error types: (1) *withstack.withStack (2) *errutil.leafError
-- report composition:
*errutil.leafError: flow 98ccc2bf: unexpected 51200 leftover bytes
crash_reporting.go:390: *withstack.withStack (top exception)

Stacktrace (expand for inline code snippets):

src/runtime/asm_arm64.s#L1171-L1173

cockroach/pkg/sql/pgwire/conn.go

Lines 309 to 311 in 3bacdbe

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(
	ctx,

cockroach/pkg/sql/conn_executor.go

Lines 899 to 901 in 3bacdbe

	}(ctx, h)
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

cockroach/pkg/sql/conn_executor.go

Lines 2102 to 2104 in 3bacdbe

	var err error
	if err = ex.execCmd(); err != nil {
	// Both of these errors are normal ways for the connExecutor to exit.

cockroach/pkg/sql/conn_executor.go

Lines 2185 to 2187 in 3bacdbe

	return err
	}()
	// Note: we write to ex.statsCollector.PhaseTimes, instead of ex.phaseTimes,

cockroach/pkg/sql/conn_executor.go

Lines 2180 to 2182 in 3bacdbe

	tcmd.LastInBatch || !implicitTxnForBatch)
	ev, payload, err = ex.execStmt(
	ctx, tcmd.Statement, nil /* portal */, nil /* pinfo */, stmtRes, canAutoCommit,

cockroach/pkg/sql/conn_executor_exec.go

Lines 140 to 142 in 3bacdbe

	}
	err = ex.execWithProfiling(ctx, ast, preparedStmt, func(ctx context.Context) error {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, portal, pinfo, res, canAutoCommit)

cockroach/pkg/sql/conn_executor_exec.go

Lines 2985 to 2987 in 3bacdbe

	} else {
	err = op(ctx)
	}

cockroach/pkg/sql/conn_executor_exec.go

Lines 141 to 143 in 3bacdbe

	err = ex.execWithProfiling(ctx, ast, preparedStmt, func(ctx context.Context) error {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, portal, pinfo, res, canAutoCommit)
	return err

cockroach/pkg/sql/conn_executor_exec.go

Lines 767 to 769 in 3bacdbe

	// CommitTransaction is executed fully here; there's no plan for it.
	ev, payload := ex.commitSQLTransaction(ctx, ast, ex.commitSQLTransactionInternal)
	return ev, payload, nil

cockroach/pkg/sql/conn_executor_exec.go

Lines 1233 to 1235 in 3bacdbe

	ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionStartTransactionCommit, timeutil.Now())
	if err := commitFn(ctx); err != nil {
	if descs.IsTwoVersionInvariantViolationError(err) {

cockroach/pkg/sql/conn_executor_exec.go

Lines 1300 to 1302 in 3bacdbe

	ex.extraTxnState.prepStmtsNamespace.closeAllPortals(ctx, &ex.extraTxnState.prepStmtsNamespaceMemAcc)

cockroach/pkg/sql/conn_executor.go

Lines 1813 to 1815 in 3bacdbe

	for name, p := range ns.portals {
	p.close(ctx, prepStmtsNamespaceMemAcc, name)
	delete(ns.portals, name)

cockroach/pkg/sql/prepared_stmt.go

Lines 223 to 225 in 3bacdbe

	if p.pauseInfo != nil {
	p.pauseInfo.cleanupAll()
	p.pauseInfo = nil

cockroach/pkg/sql/prepared_stmt.go

Lines 372 to 374 in 3bacdbe

	func (pm *portalPauseInfo) cleanupAll() {
	pm.resumableFlow.cleanup.run()
	pm.dispatchToExecutionEngine.cleanup.run()

cockroach/pkg/sql/prepared_stmt.go

Lines 250 to 252 in 3bacdbe

	for i := 0; i < len(n.stack); i++ {
	n.stack[i].f()
	}

cockroach/pkg/sql/distsql_running.go

Lines 1661 to 1663 in 3bacdbe

	fName: "cleanup flow", f: func() {
	p.resumableFlow.flow.Cleanup(ctx)
	},

cockroach/pkg/sql/colflow/vectorized_flow.go

Lines 415 to 417 in 3bacdbe

	}
	f.FlowBase.Cleanup(ctx)
	f.Release()

cockroach/pkg/sql/flowinfra/flow.go

Lines 698 to 700 in 3bacdbe

	f.DiskMonitor.Stop(ctx)
	f.Mon.Stop(ctx)
	}

cockroach/pkg/util/mon/bytes_usage.go

Lines 564 to 566 in 3bacdbe

	func (mm *BytesMonitor) Stop(ctx context.Context) {
	mm.doStop(ctx, true)
	}

cockroach/pkg/util/mon/bytes_usage.go

Lines 591 to 593 in 3bacdbe

	if check && mm.mu.curAllocated != 0 {
	logcrash.ReportOrPanic(
	ctx, &mm.settings.SV,

cockroach/pkg/util/log/logcrash/crash_reporting.go

Lines 389 to 391 in 3bacdbe

	) {
	err := errors.Newf(format, reportables...)
	if !build.IsRelease() || (sv != nil && PanicOnAssertions.Get(sv)) {

src/runtime/asm_arm64.s in runtime.goexit at line 1172
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 310
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 900
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 2103
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 2186
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func1 at line 2181
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 141
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithProfiling at line 2986
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt.func1 at line 142
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 768
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).commitSQLTransaction at line 1234
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).commitSQLTransactionInternal at line 1301
pkg/sql/conn_executor.go in pkg/sql.(*prepStmtNamespace).closeAllPortals at line 1814
pkg/sql/prepared_stmt.go in pkg/sql.(*PreparedPortal).close at line 224
pkg/sql/prepared_stmt.go in pkg/sql.(*portalPauseInfo).cleanupAll at line 373
pkg/sql/prepared_stmt.go in pkg/sql.(*cleanupFuncStack).run at line 251
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRunAll.func4 at line 1662
pkg/sql/colflow/vectorized_flow.go in pkg/sql/colflow.(*vectorizedFlow).Cleanup at line 416
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Cleanup at line 699
pkg/util/mon/bytes_usage.go in pkg/util/mon.(*BytesMonitor).Stop at line 565
pkg/util/mon/bytes_usage.go in pkg/util/mon.(*BytesMonitor).doStop at line 592
pkg/util/log/logcrash/crash_reporting.go in pkg/util/log/logcrash.ReportOrPanic at line 390

Tags

Tag	Value
Command	start-single-node
Environment	v23.1.6
Go Version	go1.19.10
Platform	darwin arm64
Distribution	CCL
Cockroach Release	v23.1.6
Cockroach SHA	3bacdbe
# of CPUs	10
# of Goroutines	447

Jira issue: CRDB-31452

[yuzefovich]

dup of #110486