sql: v20.2.3: foreign key does not exist during backfill

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2324059169/?referrer=webhooks_plugin

Panic message:

backfill.go:1885: foreign key × does not exist
--
*errutil.leafError: foreign key × does not exist (1)
backfill.go:1885: *withstack.withStack (top exception)
*assert.withAssertionFailure
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/backfill.go

Lines 1884 to 1886 in 89dd2d1

	if fk == nil {
	return errors.AssertionFailedf("foreign key %s does not exist", fkName)
	}

in pkg/sql.validateFkInTxn

cockroach/pkg/sql/backfill.go

Lines 661 to 663 in 89dd2d1

	case descpb.ConstraintToUpdate_FOREIGN_KEY:
	if err := validateFkInTxn(ctx, sc.leaseMgr, &evalCtx.EvalContext, desc, txn, c.Name); err != nil {
	return err

in pkg/sql.(*SchemaChanger).validateConstraints.func2.1

cockroach/pkg/sql/backfill.go

Lines 132 to 134 in 89dd2d1

	evalCtx := createSchemaChangeEvalCtx(ctx, sc.execCfg, readAsOf, sc.ieFactory)
	return retryable(ctx, txn, &evalCtx)
	})

in pkg/sql.(*SchemaChanger).makeFixedTimestampRunner.func1.1

cockroach/pkg/sql/backfill.go

Lines 145 to 147 in 89dd2d1

	txn.SetFixedTimestamp(ctx, readAsOf)
	return retryable(ctx, txn)
	})

in pkg/sql.(*SchemaChanger).fixedTimestampTxn.func1

cockroach/pkg/kv/db.go

Lines 706 to 708 in 89dd2d1

	err := txn.exec(ctx, func(ctx context.Context, txn *Txn) error {
	return retryable(ctx, txn)
	})

in pkg/kv.(*DB).Txn.func1

cockroach/pkg/kv/txn.go

Lines 810 to 812 in 89dd2d1

	}
	err = fn(ctx, txn)

in pkg/kv.(*Txn).exec

cockroach/pkg/kv/db.go

Lines 705 to 707 in 89dd2d1

	txn.SetDebugName("unnamed")
	err := txn.exec(ctx, func(ctx context.Context, txn *Txn) error {
	return retryable(ctx, txn)

in pkg/kv.(*DB).Txn

cockroach/pkg/sql/backfill.go

Lines 143 to 145 in 89dd2d1

	) error {
	return sc.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
	txn.SetFixedTimestamp(ctx, readAsOf)

in pkg/sql.(*SchemaChanger).fixedTimestampTxn

cockroach/pkg/sql/backfill.go

Lines 129 to 131 in 89dd2d1

	runner := func(ctx context.Context, retryable scTxnFn) error {
	return sc.fixedTimestampTxn(ctx, readAsOf, func(ctx context.Context, txn *kv.Txn) error {
	// We need to re-create the evalCtx since the txn may retry.

in pkg/sql.(*SchemaChanger).makeFixedTimestampRunner.func1

cockroach/pkg/sql/backfill.go

Lines 643 to 645 in 89dd2d1

	// Each check operates at the historical timestamp.
	return runHistoricalTxn(ctx, func(ctx context.Context, txn *kv.Txn, evalCtx *extendedEvalContext) error {
	// If the constraint is a check constraint that fails validation, we

in pkg/sql.(*SchemaChanger).validateConstraints.func2

cockroach/pkg/util/ctxgroup/ctxgroup.go

Lines 165 to 167 in 89dd2d1

	g.wrapped.Go(func() error {
	return f(g.ctx)
	})

in pkg/util/ctxgroup.Group.GoCtx.func1
https://github.com/cockroachdb/cockroach/blob/89dd2d14c29787378c434ed54937757ef5d9877c/vendor/golang.org/x/sync/errgroup/errgroup.go#L56-L58 in golang.org/x/sync/errgroup.(*Group).Go.func1
/usr/local/go/src/runtime/asm_amd64.s#L1356-L1358 in runtime.goexit

pkg/sql/backfill.go in pkg/sql.validateFkInTxn at line 1885
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).validateConstraints.func2.1 at line 662
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).makeFixedTimestampRunner.func1.1 at line 133
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).fixedTimestampTxn.func1 at line 146
pkg/kv/db.go in pkg/kv.(*DB).Txn.func1 at line 707
pkg/kv/txn.go in pkg/kv.(*Txn).exec at line 811
pkg/kv/db.go in pkg/kv.(*DB).Txn at line 706
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).fixedTimestampTxn at line 144
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).makeFixedTimestampRunner.func1 at line 130
pkg/sql/backfill.go in pkg/sql.(*SchemaChanger).validateConstraints.func2 at line 644
pkg/util/ctxgroup/ctxgroup.go in pkg/util/ctxgroup.Group.GoCtx.func1 at line 166
vendor/golang.org/x/sync/errgroup/errgroup.go in golang.org/x/sync/errgroup.(*Group).Go.func1 at line 57
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1357

Tag	Value
Cockroach Release	v20.2.3
Cockroach SHA:	89dd2d1
Platform	linux amd64
Distribution	CCL
Environment	v20.2.3
Command	start-single-node
Go Version	``
# of CPUs
# of Goroutines

[yuzefovich]

Same as #57376

[ajwerner]

Mystery. Don't know how to reproduce. Maybe something to do with dropping the referenced column or something. Closing as unactionable but there's a bug here somewhere.