sentry: error.go:88: unexpected error from the vectorized engine: context canceled (1) attached stack trace -- stack trace: | github.com/cockroachdb/cockroach/pkg/sql/colflow/colrpc.(*Inbox).Next.func1 ...

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3415266891/?referrer=webhooks_plugin

Panic message:

error.go:88: unexpected error from the vectorized engine: context canceled
(1) attached stack trace
-- stack trace:
| github.com/cockroachdb/cockroach/pkg/sql/colflow/colrpc.(*Inbox).Next.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/colrpc/inbox.go:338
| runtime.gopanic
| /usr/local/go/src/runtime/panic.go:965
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.ExpectedError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:200
| github.com/cockroachdb/cockroach/pkg/sql/colflow/colrpc.(*Inbox).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/colrpc/inbox.go:378
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*ParallelUnorderedSynchronizer).Init.func1.1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/parallel_unordered_synchronizer.go:185
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*ParallelUnorderedSynchronizer).init.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/parallel_unordered_synchronizer.go:249
| runtime.goexit
| /usr/local/go/src/runtime/asm_amd64.s:1371
Wraps: (2)
Wraps: (3) tags: [n1,client=10.254.32.62:36548,hostssl,user=×,streamID=×,received-error]
Wraps: (4) tags: [n×,streamID=×,sent-error=×]
Wraps: (5)
Wraps: (6)
Wraps: (7) assertion failure
Wraps: (8)
| (opaque error wrapper)
| type name: github.com/cockroachdb/errors/withstack/*withstack.withStack
| reportable 0:
|
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:88
| runtime.gopanic
| /usr/local/go/src/runtime/panic.go:965
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.InternalError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:192
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*ParallelUnorderedSynchronizer).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/parallel_unordered_synchronizer.go:353
| github.com/cockroachdb/cockroach/pkg/sql/colexec/colexechash.(*HashTable).FullBuild
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexechash/hashtable.go:361
| github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecjoin.(*hashJoiner).build
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecjoin/hashjoiner.go:318
| github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecjoin.(*hashJoiner).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecjoin/hashjoiner.go:280
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:261
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:286
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:292
| github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:664
| github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:603
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:222
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*unorderedDistinct).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/unordered_distinct.go:95
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).processNextBatch
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:639
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).Run.func1.1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:563
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).Run.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:601
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).Run
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:558
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlowCreator).setupRouter.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:793
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlowCreatorHelper).accumulateAsyncComponent.func1.1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:1320
| runtime.goexit
| /usr/local/go/src/runtime/asm_amd64.s:1371
Wraps: (9) unexpected error from the vectorized engine
Wraps: (10) context canceled
| -- cause hidden behind barrier
| context canceled
| (1) context canceled
| Error types: (1) *errors.errorString
Error types: (1) *withstack.withStack (2) *colexecerror.notInternalError (3) *contexttags.withContext (4) *contexttags.withContext (5) *colexecerror.notInternalError (6) *colexecerror.notInternalError (7) *assert.withAssertionFailure (8) *errbase.opaqueWrapper (9) *errutil.withPrefix (10) *barriers.barrierErr
-- report composition:
*barriers.barrierErr: masked error: context canceled
*errutil.withPrefix: unexpected error from the vectorized engine
error.go:88: *withstack.withStack (top exception)
*assert.withAssertionFailure
*colexecerror.notInternalError
*colexecerror.notInternalError
*contexttags.withContext: n2
*contexttags.withContext: n1
*colexecerror.notInternalError
inbox.go:338: *withstack.withStack (1)
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/colflow/colrpc/inbox.go

Lines 337 to 339 in 11787ed

	i.close()
	err := logcrash.PanicAsError(0, panicObj)
	log.VEventf(i.Ctx, 1, "Inbox encountered an error in Next: %v", err)

in pkg/sql/colflow/colrpc.(*Inbox).Next.func1
/usr/local/go/src/runtime/panic.go#L964-L966 in runtime.gopanic

cockroach/pkg/sql/colexecerror/error.go

Lines 199 to 201 in 11787ed

	func ExpectedError(err error) {
	panic(newNotInternalError(err))
	}

in pkg/sql/colexecerror.ExpectedError

cockroach/pkg/sql/colflow/colrpc/inbox.go

Lines 377 to 379 in 11787ed

	// and returned in DrainMeta.
	colexecerror.ExpectedError(meta.Err)
	}

in pkg/sql/colflow/colrpc.(*Inbox).Next

cockroach/pkg/sql/colexec/parallel_unordered_synchronizer.go

Lines 184 to 186 in 11787ed

	return func() {
	s.batches[inputIdx] = inputOp.Next()
	}

in pkg/sql/colexec.(*ParallelUnorderedSynchronizer).Init.func1.1

cockroach/pkg/sql/colexecerror/error.go

Lines 90 to 92 in 11787ed

	}()
	operation()
	return retErr

in pkg/sql/colexecerror.CatchVectorizedRuntimeError

cockroach/pkg/sql/colexec/parallel_unordered_synchronizer.go

Lines 248 to 250 in 11787ed

	case parallelUnorderedSynchronizerStateRunning:
	if err := colexecerror.CatchVectorizedRuntimeError(s.nextBatch[inputIdx]); err != nil {
	if s.getState() == parallelUnorderedSynchronizerStateDraining && s.Ctx.Err() == nil && s.cancelLocalInput[inputIdx] != nil {

in pkg/sql/colexec.(*ParallelUnorderedSynchronizer).init.func1
/usr/local/go/src/runtime/asm_amd64.s#L1370-L1372 in runtime.goexit

pkg/sql/colflow/colrpc/inbox.go in pkg/sql/colflow/colrpc.(*Inbox).Next.func1 at line 338
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 965
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.ExpectedError at line 200
pkg/sql/colflow/colrpc/inbox.go in pkg/sql/colflow/colrpc.(*Inbox).Next at line 378
pkg/sql/colexec/parallel_unordered_synchronizer.go in pkg/sql/colexec.(*ParallelUnorderedSynchronizer).Init.func1.1 at line 185
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.CatchVectorizedRuntimeError at line 91
pkg/sql/colexec/parallel_unordered_synchronizer.go in pkg/sql/colexec.(*ParallelUnorderedSynchronizer).init.func1 at line 249
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1371

Tag	Value
Cockroach Release	v21.2.9
Cockroach SHA:	11787ed
Platform	linux amd64
Distribution	CCL
Environment	v21.2.9
Command	server
Go Version	``
# of CPUs
# of Goroutines

Jira issue: CRDB-17498

[yuzefovich]

dup of #82947