sql: v20.2.6: failed assertion when renaming database

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2317253918/?referrer=webhooks_plugin

Panic message:

rename_database.go:312: expected to find column ids × in table id 266
--
*errutil.leafError: expected to find column ids × in table id 266 (1)
rename_database.go:312: *withstack.withStack (top exception)
*assert.withAssertionFailure
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/rename_database.go

Lines 311 to 313 in 4da6dc3

	if colIDs.Len() > 0 {
	return false, "", errors.AssertionFailedf(
	"expected to find column ids %s in table id %d",

in pkg/sql.isAllowedDependentDescInRenameDatabase

cockroach/pkg/sql/rename_database.go

Lines 168 to 170 in 4da6dc3

	isAllowed, referencedCol, err := isAllowedDependentDescInRenameDatabase(
	ctx,

in pkg/sql.(*renameDatabaseNode).startExec.func1

cockroach/pkg/sql/catalog/tabledesc/structured.go

Lines 550 to 552 in 4da6dc3

	for i := range desc.DependedOnBy {
	if err := f(&desc.DependedOnBy[i]); err != nil {
	return err

in pkg/sql/catalog/tabledesc.(*Immutable).ForeachDependedOnBy

cockroach/pkg/sql/rename_database.go

Lines 162 to 164 in 4da6dc3

	if err := tbDesc.ForeachDependedOnBy(func(dependedOn *descpb.TableDescriptor_Reference) error {
	dependentDesc, err := catalogkv.MustGetTableDescByID(ctx, p.txn, p.ExecCfg().Codec, dependedOn.ID)

in pkg/sql.(*renameDatabaseNode).startExec

cockroach/pkg/sql/plan.go

Lines 505 to 507 in 4da6dc3

	}
	return n.startExec(params)
	},

in pkg/sql.startExec.func2

cockroach/pkg/sql/walk.go

Lines 118 to 120 in 4da6dc3

	}
	v.err = v.observer.leaveNode(name, plan)
	}()

in pkg/sql.(*planVisitor).visitInternal.func1

cockroach/pkg/sql/walk.go

Lines 297 to 299 in 4da6dc3

	}
	}

in pkg/sql.(*planVisitor).visitInternal

cockroach/pkg/sql/walk.go

Lines 85 to 87 in 4da6dc3

	}
	v.visitInternal(plan, name)
	return plan

in pkg/sql.(*planVisitor).visit

cockroach/pkg/sql/walk.go

Lines 49 to 51 in 4da6dc3

	v := makePlanVisitor(ctx, observer)
	v.visit(plan)
	return v.err

in pkg/sql.walkPlan

cockroach/pkg/sql/plan.go

Lines 508 to 510 in 4da6dc3

	}
	return walkPlan(params.ctx, plan, o)
	}

in pkg/sql.startExec

cockroach/pkg/sql/plan_node_to_row_source.go

Lines 124 to 126 in 4da6dc3

	// This starts all of the nodes below this node.
	if err := startExec(p.params, p.node); err != nil {
	p.MoveToDraining(err)

in pkg/sql.(*planNodeToRowSource).Start

cockroach/pkg/sql/execinfra/processorsbase.go

Lines 773 to 775 in 4da6dc3

	}
	ctx = pb.self.Start(ctx)
	Run(ctx, pb.self, pb.Out.output)

in pkg/sql/execinfra.(*ProcessorBase).Run

cockroach/pkg/sql/flowinfra/flow.go

Lines 392 to 394 in 4da6dc3

	log.VEventf(ctx, 1, "running %T in the flow's goroutine", headProc)
	headProc.Run(ctx)
	return nil

in pkg/sql/flowinfra.(*FlowBase).Run

cockroach/pkg/sql/distsql_running.go

Lines 416 to 418 in 4da6dc3

	// TODO(radu): this should go through the flow scheduler.
	if err := flow.Run(ctx, func() {}); err != nil {
	log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %v\n"+

in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 996 to 998 in 4da6dc3

	recv.expectedRowsRead = int64(physPlan.TotalEstimatedScannedRows)
	return dsp.Run(planCtx, txn, physPlan, recv, evalCtx, nil /* finishedSetupFn */)
	}

in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 1000 to 1002 in 4da6dc3

	// the planner whether or not to plan remote table readers.
	cleanup := ex.server.cfg.DistSQLPlanner.PlanAndRun(
	ctx, evalCtx, planCtx, planner.txn, planner.curPlan.main, recv,

in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 871 to 873 in 4da6dc3

	ex.sessionTracing.TraceExecStart(ctx, "distributed")
	stats, err := ex.execWithDistSQLEngine(
	ctx, planner, stmt.AST.StatementType(), res, distributePlan.WillDistribute(), progAtomic,

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 638 to 640 in 4da6dc3

	p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	return nil, nil, err

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 113 to 115 in 4da6dc3

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1464 to 1466 in 4da6dc3

	stmtCtx := withStatement(ctx, ex.curStmt)
	ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
	return err

in pkg/sql.(*connExecutor).execCmd.func1

cockroach/pkg/sql/conn_executor.go

Lines 1466 to 1468 in 4da6dc3

	return err
	}()
	// Note: we write to ex.statsCollector.phaseTimes, instead of ex.phaseTimes,

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1390 to 1392 in 4da6dc3

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if errors.IsAny(err, io.EOF, errDrainingComplete) {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 507 to 509 in 4da6dc3

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 625 to 627 in 4da6dc3

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1
/usr/local/go/src/runtime/asm_amd64.s#L1356-L1358 in runtime.goexit

pkg/sql/rename_database.go in pkg/sql.isAllowedDependentDescInRenameDatabase at line 312
pkg/sql/rename_database.go in pkg/sql.(*renameDatabaseNode).startExec.func1 at line 169
pkg/sql/catalog/tabledesc/structured.go in pkg/sql/catalog/tabledesc.(*Immutable).ForeachDependedOnBy at line 551
pkg/sql/rename_database.go in pkg/sql.(*renameDatabaseNode).startExec at line 163
pkg/sql/plan.go in pkg/sql.startExec.func2 at line 506
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal.func1 at line 119
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal at line 298
pkg/sql/walk.go in pkg/sql.(*planVisitor).visit at line 86
pkg/sql/walk.go in pkg/sql.walkPlan at line 50
pkg/sql/plan.go in pkg/sql.startExec at line 509
pkg/sql/plan_node_to_row_source.go in pkg/sql.(*planNodeToRowSource).Start at line 125
pkg/sql/execinfra/processorsbase.go in pkg/sql/execinfra.(*ProcessorBase).Run at line 774
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Run at line 393
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 417
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 997
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 1001
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 872
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 639
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 114
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func1 at line 1465
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1467
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1391
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 508
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 626
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1357

Tag	Value
Cockroach Release	v20.2.6
Cockroach SHA:	4da6dc3
Platform	linux amd64
Distribution	CCL
Environment	v20.2.6
Command	start-single-node
Go Version	``
# of CPUs
# of Goroutines

[jordanlewis]

This indicates a situation in which a view or type that depends on a table somehow lost its column reference to that table, but the view or type is there. Possibly a DROP COLUMN related thing.

@ajwerner, are we sure that Validate would catch this issue going forward?

[ajwerner]

@ajwerner, are we sure that Validate would catch this issue going forward?

No 😞

cockroach/pkg/sql/catalog/tabledesc/validate_test.go

Lines 93 to 98 in bd62e2c

	"DependsOn": {
	status: todoIAmKnowinglyAddingTechDebt,
	reason: "initial import: TODO(features): add validation"},
	"DependedOnBy": {
	status: todoIAmKnowinglyAddingTechDebt,
	reason: "initial import: TODO(features): add validation"},

#63147

[ajwerner]

Closing as unactionable because we have no clue how to get into this state but we have #63147 to detect it in a more in your face way in the future.