roachtest: schemachange/during/tpcc failed

[cockroach-teamcity]

roachtest.schemachange/during/tpcc failed with artifacts on master @ 8a5cb5109d6b7e122e996a44f940bfd61698cd81:

test artifacts and logs in: /artifacts/schemachange/during/tpcc/run_1
(test_impl.go:291).Fatal: output in run_140628.274165790_n5_cockroach_workload_check_tpcc: ./cockroach workload check tpcc --warehouses=1000 {pgurl:1} returned: COMMAND_PROBLEM: exit status 1

Parameters: ROACHTEST_cloud=gce , ROACHTEST_cpu=16 , ROACHTEST_encrypted=false , ROACHTEST_ssd=0

Help

See: roachtest README

See: How To Investigate (internal)

/cc @cockroachdb/sql-schema

This test on roachdash | Improve this report!

Jira issue: CRDB-22125

[ajwerner]

panic: can't call AddRow after having set error: grpc: context canceled [code 1/Canceled] [recovered]
	panic: can't call AddRow after having set error: grpc: context canceled [code 1/Canceled] [recovered]
	panic: can't call AddRow after having set error: grpc: context canceled [code 1/Canceled]

goroutine 161971595 [running]:
panic({0x4f76a60, 0xc03c611f20})
	GOROOT/src/runtime/panic.go:987 +0x3ba fp=0xc0320c3090 sp=0xc0320c2fd0 pc=0x48d7da
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).closeWrapper(0xc012428000, {0x6979bc0, 0xc008259b80}, {0x4f76520?, 0xc0220d6f60?})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1077 +0x1b4 fp=0xc0320c3120 sp=0xc0320c3090 pc=0x33c4a14
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn.func1({0x6979bc0, 0xc008259b80}, {0x4351e3aaf7?})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:829 +0x4a fp=0xc0320c3160 sp=0xc0320c3120 pc=0x33c2a0a
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn.func3()
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:830 +0x32 fp=0xc0320c3188 sp=0xc0320c3160 pc=0x33c2c12
runtime.deferCallSave(0xc0320c3258, 0xc0320c5df0?)
	GOROOT/src/runtime/panic.go:796 +0x88 fp=0xc0320c3198 sp=0xc0320c3188 pc=0x48d3c8
runtime.runOpenDeferFrame(0xc0128536d0?, 0xc02eafeaf0)
	GOROOT/src/runtime/panic.go:769 +0x1a5 fp=0xc0320c31e0 sp=0xc0320c3198 pc=0x48d1e5
panic({0x4f76520, 0xc0220d6f60})
	GOROOT/src/runtime/panic.go:884 +0x212 fp=0xc0320c32a0 sp=0xc0320c31e0 pc=0x48d632
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Wait(0xc01cad5c00)
	github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:512 +0x1c5 fp=0xc0320c3330 sp=0xc0320c32a0 pc=0x270b6a5
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run.func1()
	github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:312 +0x26 fp=0xc0320c3348 sp=0xc0320c3330 pc=0x31ade26
runtime.deferCallSave(0xc0320c3418, 0xc0320c3758?)
	GOROOT/src/runtime/panic.go:796 +0x88 fp=0xc0320c3358 sp=0xc0320c3348 pc=0x48d3c8
runtime.runOpenDeferFrame(0xc0128536d0?, 0xc02eafeaf0)
	GOROOT/src/runtime/panic.go:769 +0x1a5 fp=0xc0320c33a0 sp=0xc0320c3358 pc=0x48d1e5
panic({0x4f76520, 0xc0220d6f60})
	GOROOT/src/runtime/panic.go:884 +0x212 fp=0xc0320c3460 sp=0xc0320c33a0 pc=0x48d632
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*commandResult).beforeAdd(0xc01ee6eb40?)
	github.com/cockroachdb/cockroach/pkg/sql/pgwire/command_result.go:203 +0x133 fp=0xc0320c34c8 sp=0xc0320c3460 pc=0x37d6253
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*commandResult).AddBatch(0xc00cee4b08, {0x6979c68, 0xc007c39a10}, {0x69cd160?, 0xc01c273e60?})
	github.com/cockroachdb/cockroach/pkg/sql/pgwire/command_result.go:226 +0x45 fp=0xc0320c3518 sp=0xc0320c34c8 pc=0x37d63a5
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLReceiver).PushBatch(0xc0122cca80, {0x69cd160, 0xc01c273e60}, 0x16?)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1467 +0x1ae fp=0xc0320c3550 sp=0xc0320c3518 pc=0x348dfae
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc026867a20, {0x6979bc0?, 0xc02ede1f80?})
	github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:311 +0x35e fp=0xc0320c36e0 sp=0xc0320c3550 pc=0x31a8e5e
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0269c5ef0, {0x6979bc0, 0xc02ede1f80})
	github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:321 +0x1d8 fp=0xc0320c3780 sp=0xc0320c36e0 pc=0x31add78
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00477cb40, {0x6979c68, 0xc007c39a10}, 0xc020429260, 0xc026f9e210, 0xc0252c8b80, 0xc0122cca80, 0xc0124288b0, 0x0)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:813 +0x955 fp=0xc0320c3ec8 sp=0xc0320c3780 pc=0x348b715
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc007c39b00?, {0x6979c68, 0xc007c39a10}, 0x0?, 0xc020429260, 0x0?, {{0x697bc20?, 0xc02daa37a0?}, 0x0?}, 0xc0122cca80)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1766 +0x1f9 fp=0xc0320c3f80 sp=0xc0320c3ec8 pc=0x348ff39
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc020429260?, {0x6979c68, 0xc007c39a10}, 0xc012428540?, 0x1?, 0xc012428540, 0xc0122cca80, 0xc001197aa0?)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1518 +0x206 fp=0xc0320c4098 sp=0xc0320c3f80 pc=0x348e346
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc012428000, {0x6979c68, 0xc007c39a10}, 0xc012428540, 0xc007c39a10?, {0x7fda9d72b570?, 0xc00cee4b08}, 0x0?, 0xc02b6965d8)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1615 +0x4c8 fp=0xc0320c41a0 sp=0xc0320c4098 pc=0x33df308
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc012428000, {0x6979c68, 0xc007c392f0}, 0xc012428540, {0x7fda9d72b570, 0xc00cee4b08})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1236 +0xbe5 fp=0xc0320c45d0 sp=0xc0320c41a0 pc=0x33dc785
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc012428000, {0x6979c68?, 0xc031f623c0?}, {{0x69a2d60, 0xc02b32cb40}, {0x0, 0x0, 0x0}, {0xc015ff4217, 0x5c}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:730 +0x2614 fp=0xc0320c5020 sp=0xc0320c45d0 pc=0x33d7d94
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x6979c68?, 0xc031f623c0?})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:130 +0xbc fp=0xc0320c50d0 sp=0xc0320c5020 pc=0x33d51bc
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x6979c68?, {0x6979c68?, 0xc031f623c0?}, {0x69a2d60?, 0xc02b32cb40?}, 0xab54d2e01f1?, 0x6979c68?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2475 +0x330 fp=0xc0320c5220 sp=0xc0320c50d0 pc=0x33e5830
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc012428000, {0x6979c68, 0xc031f623c0}, {{0x69a2d60, 0xc02b32cb40}, {0x0, 0x0, 0x0}, {0xc015ff4217, 0x5c}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:129 +0x4fa fp=0xc0320c53b0 sp=0xc0320c5220 pc=0x33d4bba
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc012428000, {0x6979c68, 0xc031f623c0}, {0xc01752c5a0, {0xc007c39260, 0x3, 0x3}, {0xc005be52ae, 0x1, 0x1}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:219 +0x18b fp=0xc0320c5518 sp=0xc0320c53b0 pc=0x33d540b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0371882d3, 0x0}, 0x0, {0xc0db9c990dae07b0, 0xab54d2dda30, 0x0}, 0x1}, 0xc012428000, 0xc0320c5890, 0xc0320c5880, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2030 +0x565 fp=0xc0320c56d0 sp=0xc0320c5518 pc=0x33c9305
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc012428000)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2032 +0x705 fp=0xc0320c5cd0 sp=0xc0320c56d0 pc=0x33c7865
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc012428000, {0x6979bc0, 0xc008259b80}, 0x41088f9000000000?, 0xc020bdfdd8?, 0xc0067959a0?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1874 +0x20b fp=0xc0320c5d88 sp=0xc0320c5cd0 pc=0x33c6dcb
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc00475b860?, {0x6979bc0?, 0xc008259b80?}, {0xc001c59800?}, 0x3?, 0xc00798a498?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:831 +0xe6 fp=0xc0320c5e38 sp=0xc0320c5d88 pc=0x33c2b26
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1()
	github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x40d fp=0xc0320c5fe0 sp=0xc0320c5e38 pc=0x37db50d
runtime.goexit()
	GOROOT/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0320c5fe8 sp=0xc0320c5fe0 pc=0x4c24a1
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
	github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x22e

[rytaft]

@yuzefovich do you have any idea what might be causing this? Do you think this should be a release blocker?

[yuzefovich]

I think it's a fallout from #89649 - we're setting the error from another goroutine and the main goroutine hasn't realized that there is an error yet, and we also didn't update the status of the DistSQLReceiver.