Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sentry: datum.go:391: cannot convert × to type × -- *errutil.leafError: cannot convert × to type × (1) datum.go:391: *withstack.withStack (top exception) *assert.withAssertionFailure (check the extra data payloads) #57084

Closed
cockroach-teamcity opened this issue Nov 24, 2020 · 1 comment · Fixed by #57100
Assignees
Labels
A-partial-indexes Relating to partial indexes. A-sql-optimizer SQL logical planning and optimizations. C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. O-sentry Originated from an in-the-wild panic report.

Comments

@cockroach-teamcity
Copy link
Member

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2053283846/?referrer=webhooks_plugin

Panic message:

datum.go:391: cannot convert × to type ×
--
*errutil.leafError: cannot convert × to type × (1)
datum.go:391: *withstack.withStack (top exception)
*assert.withAssertionFailure
(check the extra data payloads)

Stacktrace (expand for inline code snippets):

}
return false, errors.AssertionFailedf("cannot convert %s to type %s", d.ResolvedType(), types.Bool)
}
in pkg/sql/sem/tree.GetBool
if colIdx < len(partialIndexDelVals) {
val, err := tree.GetBool(partialIndexDelVals[colIdx])
if err != nil {
in pkg/sql/row.(*PartialIndexUpdateHelper).Init

cockroach/pkg/sql/update.go

Lines 304 to 306 in fe9afec

err := pm.Init(partialIndexPutVals, partialIndexDelVals, u.run.tu.tableDesc())
if err != nil {
in pkg/sql.(*updateNode).processSourceRow

cockroach/pkg/sql/update.go

Lines 173 to 175 in fe9afec

// accumulating the result row for later.
if err := u.processSourceRow(params, u.source.Values()); err != nil {
return false, err
in pkg/sql.(*updateNode).BatchedNext
for {
if next, err := r.source.BatchedNext(params); !next {
return err
in pkg/sql.(*rowCountNode).startExec

cockroach/pkg/sql/plan.go

Lines 513 to 515 in fe9afec

}
return n.startExec(params)
},
in pkg/sql.startExec.func2

cockroach/pkg/sql/walk.go

Lines 118 to 120 in fe9afec

}
v.err = v.observer.leaveNode(name, plan)
}()
in pkg/sql.(*planVisitor).visitInternal.func1

cockroach/pkg/sql/walk.go

Lines 297 to 299 in fe9afec

}
}
in pkg/sql.(*planVisitor).visitInternal
}
v.visitInternal(plan, name)
return plan
in pkg/sql.(*planVisitor).visit
v := makePlanVisitor(ctx, observer)
v.visit(plan)
return v.err
in pkg/sql.walkPlan

cockroach/pkg/sql/plan.go

Lines 516 to 518 in fe9afec

}
return walkPlan(params.ctx, plan, o)
}
in pkg/sql.startExec
// This starts all of the nodes below this node.
if err := startExec(p.params, p.node); err != nil {
p.MoveToDraining(err)
in pkg/sql.(*planNodeToRowSource).Start
}
ctx = pb.self.Start(ctx)
Run(ctx, pb.self, pb.Out.output)
in pkg/sql/execinfra.(*ProcessorBase).Run
}
headProc.Run(ctx)
return nil
in pkg/sql/flowinfra.(*FlowBase).Run
// TODO(radu): this should go through the flow scheduler.
if err := flow.Run(ctx, func() {}); err != nil {
log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %v\n"+
in pkg/sql.(*DistSQLPlanner).Run
postqueryRecv.resultWriter = &errOnlyResultWriter{}
dsp.Run(postqueryPlanCtx, planner.txn, postqueryPhysPlan, postqueryRecv, evalCtx, nil /* finishedSetupFn */)()
if postqueryRecv.commErr != nil {
in pkg/sql.(*DistSQLPlanner).planAndRunPostquery
if err := dsp.planAndRunPostquery(
ctx,
in pkg/sql.(*DistSQLPlanner).PlanAndRunCascadesAndChecks
ex.server.cfg.DistSQLPlanner.PlanAndRunCascadesAndChecks(
ctx, planner, evalCtxFactory, &planner.curPlan.planComponents, recv, distribute,
in pkg/sql.(*connExecutor).execWithDistSQLEngine
ex.sessionTracing.TraceExecStart(ctx, "distributed")
stats, err := ex.execWithDistSQLEngine(
ctx, planner, stmt.AST.StatementType(), res, distributePlan.WillDistribute(), progAtomic,
in pkg/sql.(*connExecutor).dispatchToExecutionEngine
p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
return nil, nil, err
in pkg/sql.(*connExecutor).execStmtInOpenState
} else {
ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
}
in pkg/sql.(*connExecutor).execStmt
stmtCtx := withStatement(ctx, ex.curStmt)
ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
return err
in pkg/sql.(*connExecutor).execCmd.func1
return err
}()
// Note: we write to ex.statsCollector.phaseTimes, instead of ex.phaseTimes,
in pkg/sql.(*connExecutor).execCmd
var err error
if err = ex.execCmd(ex.Ctx()); err != nil {
if errors.IsAny(err, io.EOF, errDrainingComplete) {
in pkg/sql.(*connExecutor).run
}()
return h.ex.run(ctx, s.pool, reserved, cancel)
}
in pkg/sql.(*Server).ServeConn
reservedOwned = false // We're about to pass ownership away.
retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
}()
in pkg/sql/pgwire.(*conn).processCommandsAsync.func1
/usr/local/go/src/runtime/asm_amd64.s#L1356-L1358 in runtime.goexit

pkg/sql/sem/tree/datum.go in pkg/sql/sem/tree.GetBool at line 391
pkg/sql/row/partial_index.go in pkg/sql/row.(*PartialIndexUpdateHelper).Init at line 70
pkg/sql/update.go in pkg/sql.(*updateNode).processSourceRow at line 305
pkg/sql/update.go in pkg/sql.(*updateNode).BatchedNext at line 174
pkg/sql/plan_batch.go in pkg/sql.(*rowCountNode).startExec at line 159
pkg/sql/plan.go in pkg/sql.startExec.func2 at line 514
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal.func1 at line 119
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal at line 298
pkg/sql/walk.go in pkg/sql.(*planVisitor).visit at line 86
pkg/sql/walk.go in pkg/sql.walkPlan at line 50
pkg/sql/plan.go in pkg/sql.startExec at line 517
pkg/sql/plan_node_to_row_source.go in pkg/sql.(*planNodeToRowSource).Start at line 125
pkg/sql/execinfra/processorsbase.go in pkg/sql/execinfra.(*ProcessorBase).Run at line 774
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Run at line 392
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 422
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).planAndRunPostquery at line 1182
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRunCascadesAndChecks at line 1087
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 1011
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 872
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 639
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 114
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd.func1 at line 1465
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1467
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1391
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 508
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 626
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1357
Tag Value
Cockroach Release v20.2.1
Cockroach SHA: fe9afec
Platform darwin amd64
Distribution CCL
Environment v20.2.1
Command demo
Go Version ``
# of CPUs
# of Goroutines
@cockroach-teamcity cockroach-teamcity added C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. O-sentry Originated from an in-the-wild panic report. labels Nov 24, 2020
@jordanlewis jordanlewis added A-sql-optimizer SQL logical planning and optimizations. A-partial-indexes Relating to partial indexes. labels Nov 24, 2020
@mgartner
Copy link
Collaborator

See #57085

mgartner added a commit to mgartner/cockroach that referenced this issue Nov 25, 2020
This commit fixes several bugs that are present when using foreign key
cascading updates and partial indexes.

Previously, the optimizer was not synthesizing partial index DEL columns
for FK cascading updates. As a result, a partial index on a child table
could become inconsistent with the rows in the primary index,
ultimately resulting in incorrect query results. The optbuilder has
been refactored to project these columns and to reduce the complexity
of doing so. As a result, partial index PUT and DEL columns are now
projected in the same expression, rather than the DEL columns being
projected as far down in the expression tree as possible.

Additionally, the execution engine was unable to handle extraneous
columns that can be added as input to FK cascading updates. These
extraneous columns would be incorrectly interpreted as synthesized
partial index columns. This commit works around this issue by passing a
set of partial indexes that columns have been synthesized for from the
optimizer to the execution engine. The longer term fix is to not produce
these columns (see issue cockroachdb#57097).

Fixes cockroachdb#57085
Fixes cockroachdb#57084

Release justification: This is a critical bug fix to a new feature,
partial indexes.

Release note (bug fix): Previously, updating parent table of a foreign
key relationship with cascading updates could cause errors or
inconsistencies in partial indexes for child tables with partial
indexes. The inconsistency of partial indexes could result in incorrect
query results. This has been fixed.
mgartner added a commit to mgartner/cockroach that referenced this issue Nov 26, 2020
Previously, the optimizer was not synthesizing partial index DEL columns
for FK cascading updates and deletes. As a result, a cascading `UPDATE`
could corrupt a child table's partial index, ultimately resulting in
incorrect query results. A cascading `DELETE` would not corrupt
partial indexes, but unnecessary `DEL` operations would be issued on
the partial index.

The optbuilder has been refactored so that these columns are correctly
projected. Both PUT and DEL columns are now projected in the same
function, `mutationBuilder.projectPartialIndexCols`. This function is
called from principal functions in the optbuilder where CHECK constraint
columns are also projected, like `mutationBuilder.buildUpdate`. In
theory this should make it harder in the future to omit these necessary
projections.

Additionally, the execution engine was unable to handle extraneous
columns that can be added as input to FK cascading updates. These
extraneous columns would be incorrectly interpreted as synthesized
partial index columns. This commit works around this issue by slicing
the source values with an upper bound in `upateNode.processSourceRow`.
The longer term fix is to not produce these columns (see issue cockroachdb#57097).

Fixes cockroachdb#57085
Fixes cockroachdb#57084

Release justification: This is a critical bug fix to a new feature,
partial indexes.

Release note (bug fix): A bug has been fixed that caused errors or
corrupted partial indexes of child tables in foreign key relationships
with cascading `UPDATE`s and `DELETE`s. The corrupt partial indexes
could result in incorrect query results. Any partial indexes on child
tables of foreign key relationships with `ON DELETE CASCADE` or `ON
UPDATE CASCADE` actions may be corrupt and should be dropped and
re-created. This bug was introduce in version 20.2.
mgartner added a commit to mgartner/cockroach that referenced this issue Dec 1, 2020
Previously, the optimizer was not synthesizing partial index DEL columns
for FK cascading updates and deletes. As a result, a cascading `UPDATE`
could corrupt a child table's partial index, ultimately resulting in
incorrect query results. A cascading `DELETE` would not corrupt
partial indexes, but unnecessary `DEL` operations would be issued on
the partial index.

The optbuilder has been refactored so that these columns are correctly
projected. Both PUT and DEL columns are now projected in the same
function, `mutationBuilder.projectPartialIndexCols`. This function is
called from principal functions in the optbuilder where CHECK constraint
columns are also projected, like `mutationBuilder.buildUpdate`. In
theory this should make it harder in the future to omit these necessary
projections.

Additionally, the execution engine was unable to handle extraneous
columns that can be added as input to FK cascading updates. These
extraneous columns would be incorrectly interpreted as synthesized
partial index columns. This commit works around this issue by slicing
the source values with an upper bound in `upateNode.processSourceRow`.
The longer term fix is to not produce these columns (see issue cockroachdb#57097).

Fixes cockroachdb#57085
Fixes cockroachdb#57084

Release justification: This is a critical bug fix to a new feature,
partial indexes.

Release note (bug fix): A bug has been fixed that caused errors or
corrupted partial indexes of child tables in foreign key relationships
with cascading `UPDATE`s and `DELETE`s. The corrupt partial indexes
could result in incorrect query results. Any partial indexes on child
tables of foreign key relationships with `ON DELETE CASCADE` or `ON
UPDATE CASCADE` actions may be corrupt and should be dropped and
re-created. This bug was introduce in version 20.2.
mgartner added a commit to mgartner/cockroach that referenced this issue Dec 1, 2020
Previously, the optimizer was not synthesizing partial index DEL columns
for FK cascading updates and deletes. As a result, a cascading `UPDATE`
could corrupt a child table's partial index, ultimately resulting in
incorrect query results. A cascading `DELETE` would not corrupt
partial indexes, but unnecessary `DEL` operations would be issued on
the partial index.

The optbuilder has been refactored so that these columns are correctly
projected. There are now three functions for projecting PUT columns, DEL
columns, and both PUT and DEL columns, each ensuring that the input
scopes are non-nil. These three functions are called from principal
functions in the optbuilder where CHECK constraint columns are also
projected, like `mutationBuilder.buildUpdate`. In theory this should
make it harder in the future to omit these necessary projections.

Additionally, the execution engine was unable to handle extraneous
columns that can be added as input to FK cascading updates. These
extraneous columns would be incorrectly interpreted as synthesized
partial index columns. This commit works around this issue by slicing
the source values with an upper bound in `updateNode.processSourceRow`.
The longer term fix is to not produce these columns (see issue cockroachdb#57097).

Fixes cockroachdb#57085
Fixes cockroachdb#57084

Release justification: This is a critical bug fix to a new feature,
partial indexes.

Release note (bug fix): A bug has been fixed that caused errors or
corrupted partial indexes of child tables in foreign key relationships
with cascading `UPDATE`s and `DELETE`s. The corrupt partial indexes
could result in incorrect query results. Any partial indexes on child
tables of foreign key relationships with `ON DELETE CASCADE` or `ON
UPDATE CASCADE` actions may be corrupt and should be dropped and
re-created. This bug was introduce in version 20.2.
craig bot pushed a commit that referenced this issue Dec 1, 2020
57100: opt: fix FK cascades to child tables with partial indexes r=mgartner a=mgartner

Previously, the optimizer was not synthesizing partial index DEL columns
for FK cascading updates and deletes. As a result, a cascading `UPDATE`
could corrupt a child table's partial index, ultimately resulting in
incorrect query results. A cascading `DELETE` would not corrupt
partial indexes, but unnecessary `DEL` operations would be issued on
the partial index.

The optbuilder has been refactored so that these columns are correctly
projected. There are now three functions for projecting PUT columns, DEL
columns, and both PUT and DEL columns, each ensuring that the input
scopes are non-nil. These three functions are called from principal
functions in the optbuilder where CHECK constraint columns are also
projected, like `mutationBuilder.buildUpdate`. In theory this should
make it harder in the future to omit these necessary projections.

Additionally, the execution engine was unable to handle extraneous
columns that can be added as input to FK cascading updates. These
extraneous columns would be incorrectly interpreted as synthesized
partial index columns. This commit works around this issue by slicing
the source values with an upper bound in `updateNode.processSourceRow`.
The longer term fix is to not produce these columns (see issue #57097).

Fixes #57085
Fixes #57084

Release justification: This is a critical bug fix to a new feature,
partial indexes.

Release note (bug fix): A bug has been fixed that caused errors or
corrupted partial indexes of child tables in foreign key relationships
with cascading `UPDATE`s and `DELETE`s. The corrupt partial indexes
could result in incorrect query results. Any partial indexes on child
tables of foreign key relationships with `ON DELETE CASCADE` or `ON
UPDATE CASCADE` actions may be corrupt and should be dropped and
re-created. This bug was introduce in version 20.2.


57323: authors: add Paul Kernfeld to authors r=kernfeld-cockroach a=kernfeld-cockroach

Release note: None

57324: authors: add ricky to authors r=rickystewart a=rickystewart

Release note: None

Co-authored-by: Marcus Gartner <[email protected]>
Co-authored-by: Paul Kernfeld <[email protected]>
Co-authored-by: Ricky Stewart <[email protected]>
@craig craig bot closed this as completed in fb857a1 Dec 1, 2020
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
A-partial-indexes Relating to partial indexes. A-sql-optimizer SQL logical planning and optimizations. C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. O-sentry Originated from an in-the-wild panic report.
Projects
None yet
Development

Successfully merging a pull request may close this issue.

3 participants