sentry: txn_coord_sender.go:1023: cannot set fixed timestamp, txn "sql txn" meta={id=8518a1b2 pri=0.00664745 epo=0 ts=1651620688.801542000,0 min=1651620688.801542000,0 seq=0} lock=false stat=PENDING rts=16516...

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3246740925/?referrer=webhooks_plugin

Panic message:

txn_coord_sender.go:1023: cannot set fixed timestamp, txn "sql txn" meta={id=8518a1b2 pri=0.00664745 epo=0 ts=1651620688.801542000,0 min=1651620688.801542000,0 seq=0} lock=false stat=PENDING rts=1651620688.801542000,0 wto=false gul=1651620688.801542000,0 already performed reads
--
*errutil.leafError: cannot set fixed timestamp, txn "sql txn" meta={id=8518a1b2 pri=0.00664745 epo=0 ts=1651620688.801542000,0 min=1651620688.801542000,0 seq=0} lock=false stat=PENDING rts=1651620688.801542000,0 wto=false gul=1651620688.801542000,0 already performed reads (1)
txn_coord_sender.go:1023: *withstack.withStack (top exception)
*assert.withAssertionFailure
*contexttags.withContext: n1 (2)
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go

Lines 1022 to 1024 in af7f257

	if !tc.interceptorAlloc.txnSpanRefresher.refreshFootprint.empty() {
	return errors.WithContextTags(errors.AssertionFailedf(
	"cannot set fixed timestamp, txn %s already performed reads", tc.mu.txn), ctx)

in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).SetFixedTimestamp

cockroach/pkg/kv/txn.go

Lines 1372 to 1374 in af7f257

	defer txn.mu.Unlock()
	return txn.mu.sender.SetFixedTimestamp(ctx, ts)
	}

in pkg/kv.(*Txn).SetFixedTimestamp

cockroach/pkg/sql/txn_state.go

Lines 271 to 273 in af7f257

	defer ts.mu.Unlock()
	if err := ts.mu.txn.SetFixedTimestamp(ctx, historicalTimestamp); err != nil {
	return err

in pkg/sql.(*txnState).setHistoricalTimestamp

cockroach/pkg/sql/conn_executor.go

Lines 2334 to 2336 in af7f257

	if !asOfTs.IsEmpty() {
	if err := ex.state.setHistoricalTimestamp(ex.Ctx(), asOfTs); err != nil {
	return err

in pkg/sql.(*connExecutor).setTransactionModes

cockroach/pkg/sql/set_transaction.go

Lines 36 to 38 in af7f257

	if err := p.extendedEvalCtx.TxnModesSetter.setTransactionModes(n.Modes, asOfTs); err != nil {
	return nil, err

in pkg/sql.(*planner).SetTransaction

cockroach/pkg/sql/opaque.go

Lines 191 to 193 in af7f257

	case *tree.SetTransaction:
	return p.SetTransaction(ctx, n)
	case *tree.SetSessionAuthorizationDefault:

in pkg/sql.planOpaque

cockroach/pkg/sql/opaque.go

Lines 60 to 62 in af7f257

	var err error
	plan, err = planOpaque(ctx, p, stmt)
	if err != nil {

in pkg/sql.buildOpaque

cockroach/pkg/sql/opt/optbuilder/opaque.go

Lines 68 to 70 in af7f257

	}
	obj, err := info.buildFn(b.ctx, b.semaCtx, b.evalCtx, stmt)
	if err != nil {

in pkg/sql/opt/optbuilder.(*Builder).tryBuildOpaque

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 367 to 369 in af7f257

	// See if we have an opaque handler registered for this statement type.
	if outScope := b.tryBuildOpaque(stmt, inScope); outScope != nil {
	// The opaque handler may resolve objects; we don't care about caching

in pkg/sql/opt/optbuilder.(*Builder).buildStmt

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 234 to 236 in af7f257

	b.ctes = nil
	outScope = b.buildStmt(stmt, desiredTypes, inScope)
	// Build With operators for any CTEs hoisted to the top level.

in pkg/sql/opt/optbuilder.(*Builder).buildStmtAtRoot

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 208 to 210 in af7f257

	// and physical properties.
	outScope := b.buildStmtAtRoot(b.stmt, nil /* desiredTypes */)

in pkg/sql/opt/optbuilder.(*Builder).Build

cockroach/pkg/sql/plan_opt.go

Lines 520 to 522 in af7f257

	bld := optbuilder.New(ctx, &p.semaCtx, p.EvalContext(), &opc.catalog, f, opc.p.stmt.AST)
	if err := bld.Build(); err != nil {
	return nil, err

in pkg/sql.(*optPlanningCtx).buildExecMemo

cockroach/pkg/sql/plan_opt.go

Lines 193 to 195 in af7f257

	execMemo, err := opc.buildExecMemo(ctx)
	if err != nil {

in pkg/sql.(*planner).makeOptimizerPlan

cockroach/pkg/sql/conn_executor_exec.go

Lines 1257 to 1259 in af7f257

	func (ex *connExecutor) makeExecPlan(ctx context.Context, planner *planner) error {
	if err := planner.makeOptimizerPlan(ctx); err != nil {
	log.VEventf(ctx, 1, "optimizer plan failed: %v", err)

in pkg/sql.(*connExecutor).makeExecPlan

cockroach/pkg/sql/conn_executor_exec.go

Lines 969 to 971 in af7f257

	// between here and there needs to happen even if there's an error.
	err := ex.makeExecPlan(ctx, planner)
	// We'll be closing the plan manually below after execution; this

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 707 to 709 in af7f257

	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	stmtThresholdSpan.Finish()

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 128 to 130 in af7f257

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, prepared, pinfo, res)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1706 to 1708 in af7f257

	ev, payload, err = ex.execStmt(ctx, tcmd.Statement, nil /* prepared */, nil /* pinfo */, stmtRes)
	return err

in pkg/sql.(*connExecutor).execCmd.func1

cockroach/pkg/sql/conn_executor.go

Lines 1708 to 1710 in af7f257

	return err
	}()
	// Note: we write to ex.statsCollector.PhaseTimes, instead of ex.phaseTimes,

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1630 to 1632 in af7f257

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if errors.IsAny(err, io.EOF, errDrainingComplete) {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 669 to 671 in af7f257

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 647 to 649 in af7f257

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1
/usr/local/go/src/runtime/asm_amd64.s#L1370-L1372 in runtime.goexit

pkg/kv/kvclient/kvcoord/txn_coord_sender.go in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).SetFixedTimestamp at line 1023
pkg/kv/txn.go in pkg/kv.(*Txn).SetFixedTimestamp at line 1373
pkg/sql/txn_state.go in pkg/sql.(*txnState).setHistoricalTimestamp at line 272
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).setTransactionModes at line 2335
pkg/sql/set_transaction.go in pkg/sql.(*planner).SetTransaction at line 37
pkg/sql/opaque.go in pkg/sql.planOpaque at line 192
pkg/sql/opaque.go in pkg/sql.buildOpaque at line 61
pkg/sql/opt/optbuilder/opaque.go in pkg/sql/opt/optbuilder.(*Builder).tryBuildOpaque at line 69
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).buildStmt at line 368
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).buildStmtAtRoot at line 235
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).Build at line 209
pkg/sql/plan_opt.go in pkg/sql.(*optPlanningCtx).buildExecMemo at line 521
pkg/sql/plan_opt.go in pkg/sql.(*planner).makeOptimizerPlan at line 194
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).makeExecPlan at line 1258
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 970
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 708
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 129
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func1 at line 1707
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1709
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1631
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 670
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 648
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1371

Tag	Value
Cockroach Release	v21.2.2
Cockroach SHA:	af7f257
Platform	darwin amd64
Distribution	CCL
Environment	development
Command	start-single-node
Go Version	``
# of CPUs
# of Goroutines

Jira issue: CRDB-15437

[yuzefovich]

dup of #77265