sentry: error.go:88: unexpected error from the vectorized engine: context canceled (1) Wraps: (2) Wraps: (3) assertion failure Wraps: (4) attached stack trace -- stack trace: | github.com/cockroachdb/cock...

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3382772524/?referrer=webhooks_plugin

Panic message:

error.go:88: unexpected error from the vectorized engine: context canceled
(1)
Wraps: (2)
Wraps: (3) assertion failure
Wraps: (4) attached stack trace
-- stack trace:
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:88
| runtime.gopanic
| /usr/local/go/src/runtime/panic.go:965
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.InternalError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:192
| github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.HandleErrorFromDiskQueue
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/utils.go:287
| github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*SpillingQueue).Enqueue
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/spilling_queue.go:185
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*routerOutputOp).addBatch
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:342
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).processNextBatch
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:657
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).Run.func1.1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:563
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).Run.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:601
| github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:91
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*HashRouter).Run
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/routers.go:558
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlowCreator).setupRouter.func1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:799
| github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlowCreatorHelper).accumulateAsyncComponent.func1.1
| /go/src/github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:1326
| runtime.goexit
| /usr/local/go/src/runtime/asm_amd64.s:1371
Wraps: (5) unexpected error from the vectorized engine
Wraps: (6) context canceled
| -- cause hidden behind barrier
| context canceled
| (1) context canceled
| Error types: (1) *errors.errorString
Error types: (1) *colexecerror.notInternalError (2) *colexecerror.notInternalError (3) *assert.withAssertionFailure (4) *withstack.withStack (5) *errutil.withPrefix (6) *barriers.barrierErr
-- report composition:
*barriers.barrierErr: masked error: context canceled
*errutil.withPrefix: unexpected error from the vectorized engine
error.go:88: *withstack.withStack (top exception)
*assert.withAssertionFailure
*colexecerror.notInternalError
*colexecerror.notInternalError

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/colexecerror/error.go

Lines 87 to 89 in b05bab3

	// unexpected.
	retErr = errors.NewAssertionErrorWithWrappedErrf(err, "unexpected error from the vectorized engine")
	}

in pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1
/usr/local/go/src/runtime/panic.go#L964-L966 in runtime.gopanic

cockroach/pkg/sql/colexecerror/error.go

Lines 191 to 193 in b05bab3

	func InternalError(err error) {
	panic(err)
	}

in pkg/sql/colexecerror.InternalError

cockroach/pkg/sql/colexec/colexecutils/utils.go

Lines 286 to 288 in b05bab3

	} else {
	colexecerror.InternalError(err)
	}

in pkg/sql/colexec/colexecutils.HandleErrorFromDiskQueue

cockroach/pkg/sql/colexec/colexecutils/spilling_queue.go

Lines 184 to 186 in b05bab3

	if err := q.maybeSpillToDisk(ctx); err != nil {
	HandleErrorFromDiskQueue(err)
	}

in pkg/sql/colexec/colexecutils.(*SpillingQueue).Enqueue

cockroach/pkg/sql/colflow/routers.go

Lines 341 to 343 in b05bab3

	o.mu.numUnread += batch.Length()
	o.mu.data.Enqueue(ctx, batch)
	if o.testingKnobs.addBatchTestInducedErrorCb != nil {

in pkg/sql/colflow.(*routerOutputOp).addBatch

cockroach/pkg/sql/colflow/routers.go

Lines 656 to 658 in b05bab3

	colexecutils.UpdateBatchState(b, len(selections[i]), true /* usesSel */, selections[i])
	if o.addBatch(ctx, b) {
	// This batch blocked the output.

in pkg/sql/colflow.(*HashRouter).processNextBatch

cockroach/pkg/sql/colflow/routers.go

Lines 562 to 564 in b05bab3

	processNextBatch := func() {
	done = r.processNextBatch(ctx)
	}

in pkg/sql/colflow.(*HashRouter).Run.func1.1

cockroach/pkg/sql/colexecerror/error.go

Lines 90 to 92 in b05bab3

	}()
	operation()
	return retErr

in pkg/sql/colexecerror.CatchVectorizedRuntimeError

cockroach/pkg/sql/colflow/routers.go

Lines 600 to 602 in b05bab3

	if err := colexecerror.CatchVectorizedRuntimeError(processNextBatch); err != nil {
	r.cancelOutputs(ctx, err)

in pkg/sql/colflow.(*HashRouter).Run.func1

cockroach/pkg/sql/colexecerror/error.go

Lines 90 to 92 in b05bab3

	}()
	operation()
	return retErr

in pkg/sql/colexecerror.CatchVectorizedRuntimeError

cockroach/pkg/sql/colflow/routers.go

Lines 557 to 559 in b05bab3

	// well for more fine-grained control of error propagation.
	if err := colexecerror.CatchVectorizedRuntimeError(func() {
	r.Input.Init(ctx)

in pkg/sql/colflow.(*HashRouter).Run

cockroach/pkg/sql/colflow/vectorized_flow.go

Lines 798 to 800 in b05bab3

	runRouter := func(ctx context.Context, _ context.CancelFunc) {
	router.Run(logtags.AddTag(ctx, "hashRouterID", streamIDs))
	}

in pkg/sql/colflow.(*vectorizedFlowCreator).setupRouter.func1

cockroach/pkg/sql/colflow/vectorized_flow.go

Lines 1325 to 1327 in b05bab3

	go func() {
	run(ctx, flowCtxCancel)
	if wg != nil {

in pkg/sql/colflow.(*vectorizedFlowCreatorHelper).accumulateAsyncComponent.func1.1
/usr/local/go/src/runtime/asm_amd64.s#L1370-L1372 in runtime.goexit

pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.CatchVectorizedRuntimeError.func1 at line 88
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 965
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.InternalError at line 192
pkg/sql/colexec/colexecutils/utils.go in pkg/sql/colexec/colexecutils.HandleErrorFromDiskQueue at line 287
pkg/sql/colexec/colexecutils/spilling_queue.go in pkg/sql/colexec/colexecutils.(*SpillingQueue).Enqueue at line 185
pkg/sql/colflow/routers.go in pkg/sql/colflow.(*routerOutputOp).addBatch at line 342
pkg/sql/colflow/routers.go in pkg/sql/colflow.(*HashRouter).processNextBatch at line 657
pkg/sql/colflow/routers.go in pkg/sql/colflow.(*HashRouter).Run.func1.1 at line 563
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.CatchVectorizedRuntimeError at line 91
pkg/sql/colflow/routers.go in pkg/sql/colflow.(*HashRouter).Run.func1 at line 601
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.CatchVectorizedRuntimeError at line 91
pkg/sql/colflow/routers.go in pkg/sql/colflow.(*HashRouter).Run at line 558
pkg/sql/colflow/vectorized_flow.go in pkg/sql/colflow.(*vectorizedFlowCreator).setupRouter.func1 at line 799
pkg/sql/colflow/vectorized_flow.go in pkg/sql/colflow.(*vectorizedFlowCreatorHelper).accumulateAsyncComponent.func1.1 at line 1326
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1371

Tag	Value
Cockroach Release	v21.2.11
Cockroach SHA:	b05bab3
Platform	linux amd64
Distribution	CCL
Environment	v21.2.11
Command	server
Go Version	``
# of CPUs
# of Goroutines

Jira issue: CRDB-17079

[yuzefovich]

dup of #82947