sentry: txn_coord_sender.go:1058: cannot set fixed timestamp, txn "sql txn" meta={id=7afcaaf3 key=/Min pri=0.00000391 epo=0 ts=1674692850.180401352,0 min=1674692850.180401352,0 seq=0} lock=false stat=PENDING ...

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3897866027/?referrer=webhooks_plugin

Panic message:

txn_coord_sender.go:1058: cannot set fixed timestamp, txn "sql txn" meta={id=7afcaaf3 key=/Min pri=0.00000391 epo=0 ts=1674692850.180401352,0 min=1674692850.180401352,0 seq=0} lock=false stat=PENDING rts=1674692850.180401352,0 wto=false gul=1674692850.680401352,0 already performed reads
(1) tags: [n6,client=10.102.2.131:33172,user=root]
Wraps: (2) assertion failure
Wraps: (3) attached stack trace
-- stack trace:
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).SetFixedTimestamp
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:1058
| github.com/cockroachdb/cockroach/pkg/kv.(*Txn).SetFixedTimestamp
| github.com/cockroachdb/cockroach/pkg/kv/txn.go:1446
| github.com/cockroachdb/cockroach/pkg/sql.(*txnState).setHistoricalTimestamp
| github.com/cockroachdb/cockroach/pkg/sql/txn_state.go:310
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAOST
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:768
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:611
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:131
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2428
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1933
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1937
| github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1859
| github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn
| github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:832
| github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1
| github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728
| runtime.goexit
| GOROOT/src/runtime/asm_amd64.s:1594
Wraps: (4) cannot set fixed timestamp, txn "sql txn" meta={id=7afcaaf3 key=/Min pri=0.00000391 epo=0 ts=1674692850.180401352,0 min=1674692850.180401352,0 seq=0} lock=false stat=PENDING rts=1674692850.180401352,0 wto=false gul=1674692850.680401352,0 already performed reads
Error types: (1) *contexttags.withContext (2) *assert.withAssertionFailure (3) *withstack.withStack (4) *errutil.leafError
-- report composition:
*errutil.leafError: cannot set fixed timestamp, txn "sql txn" meta={id=7afcaaf3 key=/Min pri=0.00000391 epo=0 ts=1674692850.180401352,0 min=1674692850.180401352,0 seq=0} lock=false stat=PENDING rts=1674692850.180401352,0 wto=false gul=1674692850.680401352,0 already performed reads
txn_coord_sender.go:1058: *withstack.withStack (top exception)
*assert.withAssertionFailure
*contexttags.withContext: n6

Stacktrace (expand for inline code snippets):

cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go

Lines 1057 to 1059 in 81a114c

	if tc.hasPerformedReadsLocked() {
	return errors.WithContextTags(errors.AssertionFailedf(
	"cannot set fixed timestamp, txn %s already performed reads", tc.mu.txn), ctx)

in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).SetFixedTimestamp

cockroach/pkg/kv/txn.go

Lines 1445 to 1447 in 81a114c

	defer txn.mu.Unlock()
	return txn.mu.sender.SetFixedTimestamp(ctx, ts)
	}

in pkg/kv.(*Txn).SetFixedTimestamp

cockroach/pkg/sql/txn_state.go

Lines 309 to 311 in 81a114c

	if err := ts.mu.txn.SetFixedTimestamp(ctx, historicalTimestamp); err != nil {
	return err

in pkg/sql.(*txnState).setHistoricalTimestamp

cockroach/pkg/sql/conn_executor_exec.go

Lines 767 to 769 in 81a114c

	p.extendedEvalCtx.SetTxnTimestamp(asOf.Timestamp.GoTime())
	if err := ex.state.setHistoricalTimestamp(ctx, asOf.Timestamp); err != nil {
	return err

in pkg/sql.(*connExecutor).handleAOST

cockroach/pkg/sql/conn_executor_exec.go

Lines 610 to 612 in 81a114c

	if err := ex.handleAOST(ctx, ast); err != nil {
	return makeErrEvent(err)

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 130 to 132 in 81a114c

	err = ex.execWithProfiling(ctx, ast, prepared, func(ctx context.Context) error {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, prepared, pinfo, res, canAutoCommit)
	return err

in pkg/sql.(*connExecutor).execStmt.func1

cockroach/pkg/sql/conn_executor_exec.go

Lines 2427 to 2429 in 81a114c

	} else {
	err = op(ctx)
	}

in pkg/sql.(*connExecutor).execWithProfiling

cockroach/pkg/sql/conn_executor_exec.go

Lines 129 to 131 in 81a114c

	case stateOpen:
	err = ex.execWithProfiling(ctx, ast, prepared, func(ctx context.Context) error {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, prepared, pinfo, res, canAutoCommit)

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1932 to 1934 in 81a114c

	canAutoCommit := ex.implicitTxn() && (tcmd.LastInBatch || !implicitTxnForBatch)
	ev, payload, err = ex.execStmt(
	ctx, tcmd.Statement, nil /* prepared */, nil /* pinfo */, stmtRes, canAutoCommit,

in pkg/sql.(*connExecutor).execCmd.func1

cockroach/pkg/sql/conn_executor.go

Lines 1936 to 1938 in 81a114c

	return err
	}()
	// Note: we write to ex.statsCollector.PhaseTimes, instead of ex.phaseTimes,

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1858 to 1860 in 81a114c

	var err error
	if err = ex.execCmd(); err != nil {
	if errors.IsAny(err, io.EOF, errDrainingComplete) {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 831 to 833 in 81a114c

	}(ctx, h)
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 727 to 729 in 81a114c

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1
GOROOT/src/runtime/asm_amd64.s#L1593-L1595 in runtime.goexit

pkg/kv/kvclient/kvcoord/txn_coord_sender.go in pkg/kv/kvclient/kvcoord.(*TxnCoordSender).SetFixedTimestamp at line 1058
pkg/kv/txn.go in pkg/kv.(*Txn).SetFixedTimestamp at line 1446
pkg/sql/txn_state.go in pkg/sql.(*txnState).setHistoricalTimestamp at line 310
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).handleAOST at line 768
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 611
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt.func1 at line 131
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithProfiling at line 2428
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 130
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func1 at line 1933
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1937
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1859
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 832
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 728
GOROOT/src/runtime/asm_amd64.s in runtime.goexit at line 1594

Tag	Value
Cockroach Release	v22.2.3
Cockroach SHA:	81a114c
Platform	linux amd64
Distribution	CCL
Environment	v22.2.3
Command	server
Go Version	``
# of CPUs
# of Goroutines

Jira issue: CRDB-23796

[yuzefovich]

dup of #85066