sentry: panic due to decoding unset EncDatum

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/986216120/?project=164528&referrer=webhooks_plugin

Panic message:

conn_executor.go:645: panic while executing 1 statements: SELECT , _ FROM .. WHERE ((_ IS ) AND ( = $1)) AND (_ = $2) ORDER BY _ DESC LIMIT $3: caused by

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/conn_executor.go

Lines 386 to 388 in 1634c6b

	r := recover()
	h.ex.closeWrapper(ctx, r)
	}()

in pkg/sql.(*Server).ServeConn.func1
/usr/local/go/src/runtime/asm_amd64.s#L572-L574 in runtime.call32
/usr/local/go/src/runtime/panic.go#L501-L503 in runtime.gopanic

cockroach/pkg/sql/sqlbase/encoded_datum.go

Lines 209 to 211 in 1634c6b

	if ed.encoded == nil {
	panic("decoding unset EncDatum")
	}

in pkg/sql/sqlbase.(*EncDatum).EnsureDecoded

cockroach/pkg/sql/distsqlrun/expr.go

Lines 131 to 133 in 1634c6b

	func (eh *exprHelper) IndexedVarEval(idx int, ctx *tree.EvalContext) (tree.Datum, error) {
	err := eh.row[idx].EnsureDecoded(&eh.types[idx], &eh.datumAlloc)
	if err != nil {

in pkg/sql/distsqlrun.(*exprHelper).IndexedVarEval

cockroach/pkg/sql/sem/tree/indexed_vars.go

Lines 79 to 81 in 1634c6b

	}
	return ctx.IVarContainer.IndexedVarEval(v.Idx, ctx)
	}

in pkg/sql/sem/tree.(*IndexedVar).Eval

cockroach/pkg/sql/sem/tree/eval.go

Lines 3341 to 3343 in 1634c6b

	func (expr *ComparisonExpr) Eval(ctx *EvalContext) (Datum, error) {
	left, err := expr.Left.(TypedExpr).Eval(ctx)
	if err != nil {

in pkg/sql/sem/tree.(*ComparisonExpr).Eval

cockroach/pkg/sql/sqlbase/expr_filter.go

Lines 25 to 27 in 1634c6b

	d, err := filter.Eval(evalCtx)
	if err != nil {

in pkg/sql/sqlbase.RunFilter

cockroach/pkg/sql/distsqlrun/expr.go

Lines 171 to 173 in 1634c6b

	eh.evalCtx.PushIVarContainer(eh)
	pass, err := sqlbase.RunFilter(eh.expr, eh.evalCtx)
	eh.evalCtx.PopIVarContainer()

in pkg/sql/distsqlrun.(*exprHelper).evalFilter

cockroach/pkg/sql/distsqlrun/processors.go

Lines 330 to 332 in 1634c6b

	// Filtering.
	passes, err := h.filter.evalFilter(row)
	if err != nil {

in pkg/sql/distsqlrun.(*ProcOutputHelper).ProcessRow

cockroach/pkg/sql/distsqlrun/processors.go

Lines 707 to 709 in 1634c6b

	func (pb *ProcessorBase) ProcessRowHelper(row sqlbase.EncDatumRow) sqlbase.EncDatumRow {
	outRow, ok, err := pb.out.ProcessRow(pb.Ctx, row)
	if err != nil {

in pkg/sql/distsqlrun.(*ProcessorBase).ProcessRowHelper

cockroach/pkg/sql/distsqlrun/tablereader.go

Lines 259 to 261 in 1634c6b

	if outRow := tr.ProcessRowHelper(row); outRow != nil {
	return outRow, nil

in pkg/sql/distsqlrun.(*tableReader).Next

cockroach/pkg/sql/distsqlrun/base.go

Lines 171 to 173 in 1634c6b

	for {
	row, meta := src.Next()
	// Emit the row; stop if no more rows are needed.

in pkg/sql/distsqlrun.Run

cockroach/pkg/sql/distsqlrun/processors.go

Lines 730 to 732 in 1634c6b

	ctx = pb.self.Start(ctx)
	Run(ctx, pb.self, pb.out.output)
	if wg != nil {

in pkg/sql/distsqlrun.(*ProcessorBase).Run

cockroach/pkg/sql/distsqlrun/flow.go

Lines 606 to 608 in 1634c6b

	if len(f.processors) > 0 {
	f.processors[len(f.processors)-1].Run(ctx, nil)
	}

in pkg/sql/distsqlrun.(*Flow).StartSync

cockroach/pkg/sql/distsql_running.go

Lines 252 to 254 in 1634c6b

	// TODO(radu): this should go through the flow scheduler.
	if err := flow.StartSync(ctx, func() {}); err != nil {
	log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %s "+

in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 757 to 759 in 1634c6b

	dsp.FinalizePlan(planCtx, &physPlan)
	dsp.Run(planCtx, txn, &physPlan, recv, evalCtx, nil /* finishedSetupFn */)
	}

in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 981 to 983 in 1634c6b

	// the planner whether or not to plan remote table readers.
	ex.server.cfg.DistSQLPlanner.PlanAndRun(
	ctx, evalCtx, &planCtx, planner.txn, planner.curPlan.plan, recv)

in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 823 to 825 in 1634c6b

	ex.sessionTracing.TraceExecStart(ctx, "distributed")
	err = ex.execWithDistSQLEngine(ctx, planner, stmt.AST.StatementType(), res, distributePlan)
	} else {

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 401 to 403 in 1634c6b

	p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
	if err := ex.dispatchToExecutionEngine(ctx, stmt, p, res); err != nil {
	return nil, nil, err

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 95 to 97 in 1634c6b

	case stateOpen:
	ev, payload, err = ex.execStmtInOpenState(ctx, stmt, pinfo, res)
	switch ev.(type) {

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1174 to 1176 in 1634c6b

	ctx := withStatement(ex.Ctx(), ex.curStmt)
	ev, payload, err = ex.execStmt(ctx, curStmt, stmtRes, pinfo, pos)
	if err != nil {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 388 to 390 in 1634c6b

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 312 to 314 in 1634c6b

	go func() {
	writerErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	// TODO(andrei): Should we sometimes transmit the writerErr's to the

in pkg/sql/pgwire.(*conn).serveImpl.func4

pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn.func1 at line 387
/usr/local/go/src/runtime/asm_amd64.s in runtime.call32 at line 573
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 502
pkg/sql/sqlbase/encoded_datum.go in pkg/sql/sqlbase.(*EncDatum).EnsureDecoded at line 210
pkg/sql/distsqlrun/expr.go in pkg/sql/distsqlrun.(*exprHelper).IndexedVarEval at line 132
pkg/sql/sem/tree/indexed_vars.go in pkg/sql/sem/tree.(*IndexedVar).Eval at line 80
pkg/sql/sem/tree/eval.go in pkg/sql/sem/tree.(*ComparisonExpr).Eval at line 3342
pkg/sql/sqlbase/expr_filter.go in pkg/sql/sqlbase.RunFilter at line 26
pkg/sql/distsqlrun/expr.go in pkg/sql/distsqlrun.(*exprHelper).evalFilter at line 172
pkg/sql/distsqlrun/processors.go in pkg/sql/distsqlrun.(*ProcOutputHelper).ProcessRow at line 331
pkg/sql/distsqlrun/processors.go in pkg/sql/distsqlrun.(*ProcessorBase).ProcessRowHelper at line 708
pkg/sql/distsqlrun/tablereader.go in pkg/sql/distsqlrun.(*tableReader).Next at line 260
pkg/sql/distsqlrun/base.go in pkg/sql/distsqlrun.Run at line 172
pkg/sql/distsqlrun/processors.go in pkg/sql/distsqlrun.(*ProcessorBase).Run at line 731
pkg/sql/distsqlrun/flow.go in pkg/sql/distsqlrun.(*Flow).StartSync at line 607
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 253
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 758
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 982
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 824
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 402
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 96
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1175
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 389
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).serveImpl.func4 at line 313

Tag	Value
Cockroach Release	v2.1.5
Cockroach SHA:	1634c6b
Platform	linux amd64
Distribution	CCL
Environment	v2.1.5
Command	server
Go Version	go1.10.7
# of CPUs	8
# of Goroutines	389

[asubiotto]

Related to #36356. Keeping it open as the stacktraces are different. From a cursory look it looks like both use an ORDER BY _ DESC LIMIT _. Might be hard to do anything else if we don't have the schema but let's attempt to repro and close if we can't.

[irfansharif]

I realize this was left open for the SQL team to address, but this is almost certainly fixed by #42833. So I'm going to go ahead and close it (it'll still be searchable should anyone run into the same).