sentry: txn_coord_sender.go:941: log.Fatal: transaction unexpectedly committed, ba: Put [×,×), Delete [×,×), ConditionalPut [×,×), EndTxn(commit) [×], [txn: 288a14e4]. txn: "sql txn" meta={id=288a14e4 pri=0.0...

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3131349124/?referrer=webhooks_plugin

Panic message:

txn_coord_sender.go:941: log.Fatal: transaction unexpectedly committed, ba: Put [×,×), Delete [×,×), ConditionalPut [×,×), EndTxn(commit) [×], [txn: 288a14e4]. txn: "sql txn" meta={id=288a14e4 pri=0.02771863 epo=0 ts=1648228555.549809776,2 min=1648228493.823148787,0 seq=4} lock=true stat=COMMITTED rts=1648228555.549809776,2 wto=false gul=1648228494.323148787,0 int=3 ifw=2: ×
--
*errutil.leafError: log.Fatal: transaction unexpectedly committed, ba: Put [×,×), Delete [×,×), ConditionalPut [×,×), EndTxn(commit) [×], [txn: 288a14e4]. txn: "sql txn" meta={id=288a14e4 pri=0.02771863 epo=0 ts=1648228555.549809776,2 min=1648228493.823148787,0 seq=4} lock=true stat=COMMITTED rts=1648228555.549809776,2 wto=false gul=1648228494.323148787,0 int=3 ifw=2: × (1)
*secondary.withSecondaryError: details for github.com/cockroachdb/errors/issuelink/*issuelink.withIssueLink::: (2)
txn_coord_sender.go:941: *withstack.withStack (top exception)
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go

Lines 940 to 942 in 37dee54

	if !DisableCommitSanityCheck {
	log.Fatalf(ctx, "%s", err)
	}

in pkg/kv/kvclient/kvcoord.sanityCheckCommittedErr

cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go

Lines 898 to 900 in 37dee54

	if errTxn.Status == roachpb.COMMITTED {
	pErr = sanityCheckCommittedErr(ctx, pErr, ba)
	}

in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).updateStateLocked

cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go

Lines 521 to 523 in 37dee54

	pErr = tc.updateStateLocked(ctx, ba, br, pErr)

in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send

cockroach/pkg/kv/db.go

Lines 871 to 873 in 37dee54

	br, pErr := sender.Send(ctx, ba)
	if pErr != nil {

in pkg/kv.(*DB).sendUsingSender

cockroach/pkg/kv/txn.go

Lines 1029 to 1031 in 37dee54

	txn.mu.Unlock()
	br, pErr := txn.db.sendUsingSender(ctx, ba, sender)
	if pErr == nil {

in pkg/kv.(*Txn).Send

cockroach/pkg/kv/db.go

Lines 764 to 766 in 37dee54

	ba.AdmissionHeader = b.AdmissionHeader
	b.response, b.pErr = send(ctx, ba)
	b.fillResults(ctx)

in pkg/kv.sendAndFill

cockroach/pkg/kv/txn.go

Lines 672 to 674 in 37dee54

	}
	return sendAndFill(ctx, txn.Send, b)
	}

in pkg/kv.(*Txn).Run

cockroach/pkg/kv/txn.go

Lines 738 to 740 in 37dee54

	b.initResult(1 /* calls */, 0, b.raw, nil)
	return txn.Run(ctx, b)
	}

in pkg/kv.(*Txn).CommitInBatch

cockroach/pkg/sql/tablewriter.go

Lines 221 to 223 in 37dee54

	// coordinator.
	err = tb.txn.CommitInBatch(ctx, tb.b)
	} else {

in pkg/sql.(*tableWriterBase).finalize

cockroach/pkg/sql/upsert.go

Lines 123 to 125 in 37dee54

	n.run.tw.setRowsWrittenLimit(params.extendedEvalCtx.SessionData())
	if err := n.run.tw.finalize(params.ctx); err != nil {
	return false, err

in pkg/sql.(*upsertNode).BatchedNext

cockroach/pkg/sql/plan_batch.go

Lines 172 to 174 in 37dee54

	for {
	if next, err := r.source.BatchedNext(params); !next {
	return err

in pkg/sql.(*rowCountNode).startExec

cockroach/pkg/sql/plan.go

Lines 508 to 510 in 37dee54

	}
	return n.startExec(params)
	},

in pkg/sql.startExec.func2

cockroach/pkg/sql/walk.go

Lines 111 to 113 in 37dee54

	}
	v.err = v.observer.leaveNode(name, plan)
	}()

in pkg/sql.(*planVisitor).visitInternal.func1

cockroach/pkg/sql/walk.go

Lines 288 to 290 in 37dee54

	}
	}

in pkg/sql.(*planVisitor).visitInternal

cockroach/pkg/sql/walk.go

Lines 78 to 80 in 37dee54

	}
	v.visitInternal(plan, name)
	return plan

in pkg/sql.(*planVisitor).visit

cockroach/pkg/sql/walk.go

Lines 42 to 44 in 37dee54

	v := makePlanVisitor(ctx, observer)
	v.visit(plan)
	return v.err

in pkg/sql.walkPlan

cockroach/pkg/sql/plan.go

Lines 511 to 513 in 37dee54

	}
	return walkPlan(params.ctx, plan, o)
	}

in pkg/sql.startExec

cockroach/pkg/sql/plan_node_to_row_source.go

Lines 141 to 143 in 37dee54

	// This starts all of the nodes below this node.
	if err := startExec(p.params, p.node); err != nil {
	p.MoveToDraining(err)

in pkg/sql.(*planNodeToRowSource).Start

cockroach/pkg/sql/colflow/flow_coordinator.go

Lines 118 to 120 in 37dee54

	if err := colexecerror.CatchVectorizedRuntimeError(func() {
	f.input.Start(ctx)
	}); err != nil {

in pkg/sql/colflow.(*FlowCoordinator).Start.func1

cockroach/pkg/sql/colexecerror/error.go

Lines 90 to 92 in 37dee54

	}()
	operation()
	return retErr

in pkg/sql/colexecerror.CatchVectorizedRuntimeError

cockroach/pkg/sql/colflow/flow_coordinator.go

Lines 117 to 119 in 37dee54

	ctx = f.StartInternalNoSpan(ctx)
	if err := colexecerror.CatchVectorizedRuntimeError(func() {
	f.input.Start(ctx)

in pkg/sql/colflow.(*FlowCoordinator).Start

cockroach/pkg/sql/execinfra/processorsbase.go

Lines 729 to 731 in 37dee54

	}
	pb.self.Start(ctx)
	Run(pb.Ctx, pb.self, pb.Output)

in pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run

cockroach/pkg/sql/flowinfra/flow.go

Lines 446 to 448 in 37dee54

	log.VEventf(ctx, 1, "running %T in the flow's goroutine", headProc)
	headProc.Run(ctx)
	}

in pkg/sql/flowinfra.(*FlowBase).Run

cockroach/pkg/sql/colflow/vectorized_flow.go

Lines 244 to 246 in 37dee54

	// as the root, so we run this flow with the default implementation.
	f.FlowBase.Run(ctx, doneFn)
	return

in pkg/sql/colflow.(*vectorizedFlow).Run

cockroach/pkg/sql/distsql_running.go

Lines 547 to 549 in 37dee54

	// TODO(radu): this should go through the flow scheduler.
	flow.Run(ctx, func() {})

in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 1288 to 1290 in 37dee54

	recv.expectedRowsRead = int64(physPlan.TotalEstimatedScannedRows)
	runCleanup := dsp.Run(planCtx, txn, physPlan, recv, evalCtx, nil /* finishedSetupFn */)
	return func() {

in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 1380 to 1382 in 37dee54

	// the planner whether or not to plan remote table readers.
	cleanup := ex.server.cfg.DistSQLPlanner.PlanAndRun(
	ctx, evalCtx, planCtx, planner.txn, planner.curPlan.main, recv,

in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 1071 to 1073 in 37dee54

	ex.sessionTracing.TraceExecStart(ctx, "distributed")
	stats, err := ex.execWithDistSQLEngine(
	ctx, planner, stmt.AST.StatementReturnType(), res, distributePlan.WillDistribute(), progAtomic,

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 708 to 710 in 37dee54

	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	stmtThresholdSpan.Finish()

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 128 to 130 in 37dee54

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, prepared, pinfo, res)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor_exec.go

Lines 215 to 217 in 37dee54

	}
	ev, payload, err = ex.execStmt(ctx, portal.Stmt.Statement, portal.Stmt, pinfo, stmtRes)
	// Portal suspension is supported via a "side" state machine

in pkg/sql.(*connExecutor).execPortal

cockroach/pkg/sql/conn_executor.go

Lines 1772 to 1774 in 37dee54

	res = stmtRes
	ev, payload, err = ex.execPortal(ctx, portal, portalName, stmtRes, pinfo)
	return err

in pkg/sql.(*connExecutor).execCmd.func2

pkg/kv/kvclient/kvcoord/txn_coord_sender.go in pkg/kv/kvclient/kvcoord.sanityCheckCommittedErr at line 941
pkg/kv/kvclient/kvcoord/txn_coord_sender.go in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).updateStateLocked at line 899
pkg/kv/kvclient/kvcoord/txn_coord_sender.go in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send at line 522
pkg/kv/db.go in pkg/kv.(*DB).sendUsingSender at line 872
pkg/kv/txn.go in pkg/kv.(*Txn).Send at line 1030
pkg/kv/db.go in pkg/kv.sendAndFill at line 765
pkg/kv/txn.go in pkg/kv.(*Txn).Run at line 673
pkg/kv/txn.go in pkg/kv.(*Txn).CommitInBatch at line 739
pkg/sql/tablewriter.go in pkg/sql.(*tableWriterBase).finalize at line 222
pkg/sql/upsert.go in pkg/sql.(*upsertNode).BatchedNext at line 124
pkg/sql/plan_batch.go in pkg/sql.(*rowCountNode).startExec at line 173
pkg/sql/plan.go in pkg/sql.startExec.func2 at line 509
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal.func1 at line 112
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal at line 289
pkg/sql/walk.go in pkg/sql.(*planVisitor).visit at line 79
pkg/sql/walk.go in pkg/sql.walkPlan at line 43
pkg/sql/plan.go in pkg/sql.startExec at line 512
pkg/sql/plan_node_to_row_source.go in pkg/sql.(*planNodeToRowSource).Start at line 142
pkg/sql/colflow/flow_coordinator.go in pkg/sql/colflow.(*FlowCoordinator).Start.func1 at line 119
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.CatchVectorizedRuntimeError at line 91
pkg/sql/colflow/flow_coordinator.go in pkg/sql/colflow.(*FlowCoordinator).Start at line 118
pkg/sql/execinfra/processorsbase.go in pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run at line 730
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Run at line 447
pkg/sql/colflow/vectorized_flow.go in pkg/sql/colflow.(*vectorizedFlow).Run at line 245
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 548
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 1289
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 1381
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 1072
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 709
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 129
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execPortal at line 216
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func2 at line 1773

Tag	Value
Cockroach Release	v21.2.7
Cockroach SHA:	37dee54
Platform	linux amd64
Distribution	CCL
Environment	v21.2.7
Command	server
Go Version	``
# of CPUs
# of Goroutines

Jira issue: CRDB-14163

[yuzefovich]

dup of #67765