sql: v21.2.4: show_create_all_tables_builtin failed. len(ids): 62 not equal to len(topologicallySortedIDs): 63

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2988809625/?referrer=webhooks_plugin

Panic message:

show_create_all_tables_builtin.go:142: show_create_all_tables_builtin failed. len(ids): 62 not equal to len(topologicallySortedIDs): 63
--
*errutil.leafError: show_create_all_tables_builtin failed. len(ids): 62 not equal to len(topologicallySortedIDs): 63 (1)
show_create_all_tables_builtin.go:142: *withstack.withStack (top exception)
*assert.withAssertionFailure
*colexecerror.notInternalError
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/sem/builtins/show_create_all_tables_builtin.go

Lines 141 to 143 in 0c8df44

	if len(ids) != len(topologicallyOrderedIDs) {
	return nil, errors.AssertionFailedf("show_create_all_tables_builtin failed. "+
	"len(ids):% d not equal to len(topologicallySortedIDs): %d",

in pkg/sql/sem/builtins.getTopologicallySortedTableIDs

cockroach/pkg/sql/sem/builtins/generator_builtins.go

Lines 1730 to 1732 in 0c8df44

	// showCreateAllTablesGenerator.
	ids, err := getTopologicallySortedTableIDs(
	ctx, s.ie, txn, s.dbName, &s.acc,

in pkg/sql/sem/builtins.(*showCreateAllTablesGenerator).Start

cockroach/pkg/sql/rowexec/project_set.go

Lines 157 to 159 in 0c8df44

	}
	if err := gen.Start(ps.Ctx, ps.FlowCtx.Txn); err != nil {
	return nil, nil, err

in pkg/sql/rowexec.(*projectSetProcessor).nextInputRow

cockroach/pkg/sql/rowexec/project_set.go

Lines 240 to 242 in 0c8df44

	// Read the row from the source.
	row, meta, err := ps.nextInputRow()
	if meta != nil {

in pkg/sql/rowexec.(*projectSetProcessor).Next

cockroach/pkg/sql/colexec/columnarizer.go

Lines 221 to 223 in 0c8df44

	for ; nRows < c.batch.Capacity(); nRows++ {
	row, meta := c.input.Next()
	if meta != nil {

in pkg/sql/colexec.(*Columnarizer).Next

cockroach/pkg/sql/colflow/stats.go

Lines 97 to 99 in 0c8df44

	bic.stopwatch.Start()
	batch = bic.Operator.Next()
	bic.stopwatch.Stop()

in pkg/sql/colflow.(*batchInfoCollector).Next

cockroach/pkg/sql/colflow/flow_coordinator.go

Lines 238 to 240 in 0c8df44

	func (f *BatchFlowCoordinator) nextAdapter() {
	f.batch = f.input.Root.Next()
	}

in pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter

cockroach/pkg/sql/colexecerror/error.go

Lines 90 to 92 in 0c8df44

	}()
	operation()
	return retErr

in pkg/sql/colexecerror.CatchVectorizedRuntimeError

cockroach/pkg/sql/colflow/flow_coordinator.go

Lines 242 to 244 in 0c8df44

	func (f *BatchFlowCoordinator) next() error {
	return colexecerror.CatchVectorizedRuntimeError(f.nextAdapter)
	}

in pkg/sql/colflow.(*BatchFlowCoordinator).next

cockroach/pkg/sql/colflow/flow_coordinator.go

Lines 281 to 283 in 0c8df44

	for status == execinfra.NeedMoreRows {
	err := f.next()
	if err != nil {

in pkg/sql/colflow.(*BatchFlowCoordinator).Run

cockroach/pkg/sql/colflow/vectorized_flow.go

Lines 257 to 259 in 0c8df44

	log.VEvent(ctx, 1, "running the batch flow coordinator in the flow's goroutine")
	f.batchFlowCoordinator.Run(ctx)
	}

in pkg/sql/colflow.(*vectorizedFlow).Run

cockroach/pkg/sql/distsql_running.go

Lines 547 to 549 in 0c8df44

	// TODO(radu): this should go through the flow scheduler.
	flow.Run(ctx, func() {})

in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 1288 to 1290 in 0c8df44

	recv.expectedRowsRead = int64(physPlan.TotalEstimatedScannedRows)
	runCleanup := dsp.Run(planCtx, txn, physPlan, recv, evalCtx, nil /* finishedSetupFn */)
	return func() {

in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 1379 to 1381 in 0c8df44

	// the planner whether or not to plan remote table readers.
	cleanup := ex.server.cfg.DistSQLPlanner.PlanAndRun(
	ctx, evalCtx, planCtx, planner.txn, planner.curPlan.main, recv,

in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 1070 to 1072 in 0c8df44

	ex.sessionTracing.TraceExecStart(ctx, "distributed")
	stats, err := ex.execWithDistSQLEngine(
	ctx, planner, stmt.AST.StatementReturnType(), res, distributePlan.WillDistribute(), progAtomic,

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 708 to 710 in 0c8df44

	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	stmtThresholdSpan.Finish()

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 128 to 130 in 0c8df44

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, parserStmt, prepared, pinfo, res)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1706 to 1708 in 0c8df44

	ev, payload, err = ex.execStmt(ctx, tcmd.Statement, nil /* prepared */, nil /* pinfo */, stmtRes)
	return err

in pkg/sql.(*connExecutor).execCmd.func1

cockroach/pkg/sql/conn_executor.go

Lines 1708 to 1710 in 0c8df44

	return err
	}()
	// Note: we write to ex.statsCollector.PhaseTimes, instead of ex.phaseTimes,

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1630 to 1632 in 0c8df44

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if errors.IsAny(err, io.EOF, errDrainingComplete) {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 669 to 671 in 0c8df44

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 647 to 649 in 0c8df44

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1
/usr/local/go/src/runtime/asm_amd64.s#L1370-L1372 in runtime.goexit

pkg/sql/sem/builtins/show_create_all_tables_builtin.go in pkg/sql/sem/builtins.getTopologicallySortedTableIDs at line 142
pkg/sql/sem/builtins/generator_builtins.go in pkg/sql/sem/builtins.(*showCreateAllTablesGenerator).Start at line 1731
pkg/sql/rowexec/project_set.go in pkg/sql/rowexec.(*projectSetProcessor).nextInputRow at line 158
pkg/sql/rowexec/project_set.go in pkg/sql/rowexec.(*projectSetProcessor).Next at line 241
pkg/sql/colexec/columnarizer.go in pkg/sql/colexec.(*Columnarizer).Next at line 222
pkg/sql/colflow/stats.go in pkg/sql/colflow.(*batchInfoCollector).Next at line 98
pkg/sql/colflow/flow_coordinator.go in pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter at line 239
pkg/sql/colexecerror/error.go in pkg/sql/colexecerror.CatchVectorizedRuntimeError at line 91
pkg/sql/colflow/flow_coordinator.go in pkg/sql/colflow.(*BatchFlowCoordinator).next at line 243
pkg/sql/colflow/flow_coordinator.go in pkg/sql/colflow.(*BatchFlowCoordinator).Run at line 282
pkg/sql/colflow/vectorized_flow.go in pkg/sql/colflow.(*vectorizedFlow).Run at line 258
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 548
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 1289
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 1380
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 1071
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 709
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 129
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func1 at line 1707
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1709
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1631
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 670
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 648
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1371

Tag	Value
Cockroach Release	v21.2.4
Cockroach SHA:	0c8df44
Platform	linux amd64
Distribution	CCL
Environment	development
Command	start-single-node
Go Version	``
# of CPUs
# of Goroutines

[rafiss]

This can happen when the dependsOnIDs map has values which contain IDs that are not in the original ids slice. I think that can happen if a new constraint is added while the show create all command is running -- but I haven't reproduced that. I'm not sure if that's expected or not given the current state of transactional schema changes.