Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

github.com/cockroachdb/cockroach/pkg/sql/tests: TestMonotonicInserts/distsql=off failed under stress #26125

Closed
cockroach-teamcity opened this issue May 26, 2018 · 6 comments
Closed

github.com/cockroachdb/cockroach/pkg/sql/tests: TestMonotonicInserts/distsql=off failed under stress #26125

cockroach-teamcity opened this issue May 26, 2018 · 6 comments
Assignees
@windchan7
Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Milestone
2.1

Comments

@cockroach-teamcity
Copy link
Member

SHA: https://github.com/cockroachdb/cockroach/commits/cbbc13cd30b14bc16cd8ebe56be093f264465ebe

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=681689&tab=buildLog


goroutine 4166 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x4be6101, 0xc4204da960, 0x0, 0x12)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:956 +0xdd
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x4221dc0, 0xc400000004, 0x3d137b6, 0x12, 0xf21, 0xc420b2aa80, 0xd9)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:834 +0x7f4
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x316e020, 0xc4215a30b0, 0xc400000004, 0x2, 0x2c294e2, 0x6, 0xc423ae3b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:154 +0x31e
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x316e020, 0xc4215a30b0, 0x1, 0x4, 0x2c294e2, 0x6, 0xc423ae3b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:55 +0x9a
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x316e020, 0xc4215a30b0, 0x2c294e2, 0x6, 0xc423ae3b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:149 +0x90
github.com/cockroachdb/cockroach/pkg/storage.fatalOnRaftReadyErr(0x316e020, 0xc4215a30b0, 0x2c45f1b, 0x16, 0x314c7c0, 0xc421dd3fe0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3873 +0x219
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x316e020, 0xc4215a30b0, 0xc42148d880, 0x316e020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3576 +0x215
github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc421f3a000, 0x316e020, 0xc4215a30b0, 0xc4216b1d40, 0xc423ae3e98, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3178 +0x169
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc421f3a000, 0x316e020, 0xc420bd74d0, 0x4)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3563 +0x302
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42160ed80, 0x316e020, 0xc420bd74d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:226 +0x2f2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420bd74d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204142e0, 0xc421042240, 0xc4204142d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1 [chan receive]:
testing.(*T).Run(0xc42023a960, 0x2c413dc, 0x14, 0x2d49f28, 0xc42068dc01)
	/usr/local/go/src/testing/testing.go:825 +0x597
testing.runTests.func1(0xc42023a960)
	/usr/local/go/src/testing/testing.go:1063 +0xa5
testing.tRunner(0xc42023a960, 0xc42068dd68)
	/usr/local/go/src/testing/testing.go:777 +0x16e
testing.runTests(0xc42000da40, 0x3fdd040, 0xb, 0xb, 0xbeba71e487b06d7b)
	/usr/local/go/src/testing/testing.go:1061 +0x4e2
testing.(*M).Run(0xc4204c8500, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x2ce
github.com/cockroachdb/cockroach/pkg/sql/tests_test.TestMain(0xc4204c8500)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/tests/main_test.go:36 +0x15e
main.main()
	_testmain.go:66 +0x22b

goroutine 20 [syscall, 2 minutes]:
os/signal.signal_recv(0x866ec1)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x4f

goroutine 22 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0x132
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x1b5

goroutine 23 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x1cd

goroutine 34 [select, 2 minutes, locked to thread]:
runtime.gopark(0x2d4c870, 0x0, 0x2c2aa6c, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0xf9
runtime.selectgo(0xc42058ff50, 0xc420028060)
	/usr/local/go/src/runtime/select.go:392 +0x11d4
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x19f
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 3997 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x316e020, 0xc4213d0360)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420370050, 0xc421042240, 0xc423654080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5121 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc42277c000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x31d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0x1651

goroutine 4729 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x316e020, 0xc4211103f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1124 +0x632
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4206970c0, 0xc4221cfb90, 0xc423655340)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4801 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0x9fc46d84000003c4)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc420f42630)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420f42630)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0b10, 0xc4221cfb90, 0xc4202c0b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4818 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0xa77e162b000003c3)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc420f42600)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420f42600)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0b30, 0xc4221cfb90, 0xc4202c0b20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5046 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x5955e0b400000319)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42131d020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42131d020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a3710, 0xc4219e2870, 0xc4202a3580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4820 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0x3d6e3d09000003ca)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc420f42660)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420f42660)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0b70, 0xc4221cfb90, 0xc4202c0b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4821 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0x9fc46d84000003b7)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc420f426c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420f426c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0ba0, 0xc4221cfb90, 0xc4202c0b90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4819 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0xa77e162b000003c0)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc420f42690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420f42690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0b50, 0xc4221cfb90, 0xc4202c0b40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4823 [semacquire]:
sync.runtime_Semacquire(0xc421042474)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc421042468)
	/usr/local/go/src/sync/waitgroup.go:129 +0xb3
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc4210423f0, 0x316e020, 0xc420f426f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x3f
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x316e020, 0xc420f426f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3676 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0c00, 0xc4221cfb90, 0xc4202c0be0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4824 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop(0xc4201c8000, 0x316e020, 0xc420f42750)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3692 +0x3a7
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm(0x316e020, 0xc420f42750)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3678 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0c30, 0xc4221cfb90, 0xc4202c0c10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4822 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0x9fc46d84000003ba)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc420f42720)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420f42720)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0bd0, 0xc4221cfb90, 0xc4202c0bb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4826 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x316e020, 0xc420f42780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1420 +0x149
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0c80, 0xc4221cfb90, 0xc420450140)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4825 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc4201c8000, 0x316e020, 0xc420f42810)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3724 +0x1a6
github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x316e020, 0xc420f42810)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3679 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0c60, 0xc4221cfb90, 0xc4202c0c40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4827 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x316e020, 0xc420f425a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1558 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0ca0, 0xc4221cfb90, 0xc421e301e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4829 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x316e020, 0xc420f42870)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1558 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0cd0, 0xc4221cfb90, 0xc421e30280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5070 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc421836c00, 0x818652, 0xc42118dbc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:240 +0x155
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc421836c00, 0xc42118dbc0, 0xc421377de0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:251 +0x3d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc420ed32b0, 0x2af83e0, 0xc42068b830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:525 +0xadb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc421780380, 0x2af83e0, 0xc42068b830, 0xc4219f4850, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x67
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4202d9700, 0xc42057a000, 0xc421377ea8, 0x9a9463)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:366 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x3182a80, 0xc4202d9700, 0xc42038e640, 0xc42297c690, 0x316e020, 0xc42068b800, 0x8325fd, 0xc420d95180)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x316e020, 0xc42068b800)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d97a0, 0xc421042240, 0xc42068b7d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4831 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x316e020, 0xc420f42900)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0da0, 0xc4221cfb90, 0xc420450460)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4828 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x316e020, 0xc420f42840)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1558 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c0cb0, 0xc4221cfb90, 0xc421e30230)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 54 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420bb0fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 55 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420bb10e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 5037 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x86ac0ce500000327)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42131d350)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42131d350)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a33d0, 0xc4219e2870, 0xc4202a33c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4888 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420794580)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1373 +0x2ac
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420794580)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:949 +0x247
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:940 +0x173

goroutine 4235 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x316e020, 0xc420bd7b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1124 +0x632
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420415190, 0xc421042240, 0xc4215842a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4642 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x316e020, 0xc420423320)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42080e6e0, 0xc4221cfb90, 0xc42080e6d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4233 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x316e020, 0xc42123a5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:126 +0x18d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420415160, 0xc421042240, 0xc420415140)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4238 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x316e020, 0xc420bd7b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x1a2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420415210, 0xc421042240, 0xc420415200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4981 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x316e020, 0xc4210aa210)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c1860, 0xc4219e2870, 0xc42194b040)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4229 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x316e020, 0xc4220baf30)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420415040, 0xc421042240, 0xc4215841e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4049 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203aaee0, 0xc4200ca050, 0x29829c0, 0x3fc7810, 0x2bed540)
	<autogenerated>:1 +0x80
net/http.(*Server).Serve(0xc421414270, 0x316bb20, 0xc4203aaee0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2770 +0x1f6
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4(0x316e020, 0xc4213d01e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1101 +0x146
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203475c0, 0xc421042240, 0xc422f48720)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4994 [select]:
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func1(0x316e020, 0xc4210aa360)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:259 +0x16a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c1b50, 0xc4219e2870, 0xc4211957a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5006 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4206c6000, 0xc421898e10, 0xc4202dabc0, 0xf, 0xc42089ecc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:575 +0x5a1
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x316e020, 0xc4206c8fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:507 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42326b3a0, 0xc421042240, 0xc4206c8e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5016 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x86ac0ce500000312)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42122e330)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42122e330)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a2ec0, 0xc4219e2870, 0xc4202a2eb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5017 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x86ac0ce50000030e)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc420eee270)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc420eee270)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a2ee0, 0xc4219e2870, 0xc4202a2ed0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5015 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x86ac0ce500000316)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42122e3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42122e3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a2ea0, 0xc4219e2870, 0xc4202a2e90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4557 [semacquire]:
sync.runtime_notifyListWait(0xc420ff21d0, 0xc4138563000003be)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420ff21c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4210423f0, 0x316e020, 0xc4207265a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc4207265a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c01b0, 0xc4221cfb90, 0xc4202c01a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4576 [select]:
database/sql.(*DB).connectionResetter(0xc421eaba40, 0x316df60, 0xc4215f3ec0)
	/usr/local/go/src/database/sql/sql.go:948 +0x155
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:635 +0x2f3

goroutine 4620 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4215f38c0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc420f443c0, 0xc4208aa7e0, 0xc4215f38c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc422a60900)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 4993 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x316e020, 0xc4210aa3c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:473 +0x295
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202c1b40, 0xc4219e2870, 0xc420bda000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5035 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0xad2f4c1d00000331)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42131d2f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42131d2f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a3350, 0xc4219e2870, 0xc4202a3340)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5043 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0xad2f4c1d00000332)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42131d440)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42131d440)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a34b0, 0xc4219e2870, 0xc4202a34a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5023 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x1bb503c300000321)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42122e4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42122e4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a2fd0, 0xc4219e2870, 0xc4202a2fb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 6480 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc42297c5a0, 0x316e020, 0xc4210ab5c0, 0xc42297c5f0, 0x18, 0xc42148b810, 0x3, 0x3, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1298 +0x60c
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc42297c5a0, 0x316e020, 0xc4210ab5c0, 0x18, 0xc42148b810, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:393 +0x2bf
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc42297c5a0, 0x316e020, 0xc4210ab5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:460 +0x266
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc42297c5a0, 0x316e020, 0xc4210ab5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1067 +0x3de
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc42297c5a0, 0x316e020, 0xc4210ab5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:739 +0x1bdc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc42297c5a0, 0x316e020, 0xc4210ab5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:655 +0x566
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42167d880, 0x316e020, 0xc4210ab590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x2c5
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc423388ed0, 0x316e020, 0xc4210aae10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x107
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc423388ed0, 0x316e020, 0xc4210aae10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xb6
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x316e020, 0xc4210aae10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x97
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x316e020, 0xc4210aae10, 0xc42195d868, 0xc422208000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0x13e
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc423388ed0, 0x316e020, 0xc4210aae10, 0xc422208000, 0x3, 0xc420fe2ec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0xee
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc42297e690, 0x316e020, 0xc4210aae10, 0xc422a12480, 0x4, 0x4, 0x15323102a6018e5a, 0x0, 0x100000001, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0x105c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processWriteIntentError(0xc42297e690, 0x316e020, 0xc4210aae10, 0xc42148b630, 0x318ef40, 0xc420fe2d80, 0x15323102a6018e5a, 0x0, 0x100000001, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:109 +0x133
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc421f3a000, 0x316e020, 0xc4210aae10, 0x15323102a55edccb, 0x0, 0x100000001, 0x1, 0x18, 0x3ff0000000000000, 0xc4208baa00, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2969 +0xc2d
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4203be790, 0x316e020, 0xc4210aacf0, 0x0, 0x0, 0x100000001, 0x1, 0x18, 0x3ff0000000000000, 0xc4208baa00, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:185 +0x127
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x316e020, 0xc4210aacf0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:921 +0x1e0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421042240, 0x316e020, 0xc4210aacf0, 0x2c3ae8f, 0x10, 0xc42195f938, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:243 +0xfb
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4221bb800, 0x316e020, 0xc4210aacf0, 0xc4206eb870, 0xc4210aacf0, 0x18b2bb9, 0x316df60)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:908 +0x363
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4221bb800, 0x316e020, 0xc4210aacf0, 0xc4206eb870, 0x1f, 0x18d4e26, 0x2ad8600)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:949 +0xb8
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421926000, 0xc4206eb860, 0x316df60, 0xc420fe2e40, 0xc4210eac70, 0xc4223cd6b0, 0x8325fd)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:242 +0xa8f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421926000, 0x316df60, 0xc420fe2e40, 0xc420454da0, 0xf, 0x0, 0x0, 0x100000001, 0x1, 0x18, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:273 +0xa1
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421926000, 0x316df60, 0xc420fe2e40, 0xc4206eb7c0, 0xc423641bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:224 +0xd8
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:222 +0x3a6

goroutine 5598 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc420b77d40, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc422a59740, 0xc42044c4e0, 0xc420b77d40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc422e9c400)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:276 +0x1ab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:273 +0x162c

goroutine 6478 [chan receive]:
database/sql.(*Tx).awaitDone(0xc421470680)
	/usr/local/go/src/database/sql/sql.go:1839 +0x6b
created by database/sql.(*DB).beginDC
	/usr/local/go/src/database/sql/sql.go:1595 +0x32a

goroutine 5040 [semacquire]:
sync.runtime_notifyListWait(0xc42171b9d0, 0x86ac0ce500000325)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc42171b9c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4221ce6c0, 0x316e020, 0xc42131d3e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc42131d3e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202a3430, 0xc4219e2870, 0xc4202a3420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4886 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x316e020, 0xc4229e5c50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4211aa830, 0xc4219e2870, 0xc42035fd00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4181 [semacquire]:
sync.runtime_notifyListWait(0xc4211d2450, 0x78da00d6000007d1)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4211d2440)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42160ed80, 0x316e020, 0xc421194000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc421194000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204145f0, 0xc421042240, 0xc4204145e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 5071 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42297c690, 0x3182ae0, 0xc420696080, 0x31544a0, 0xc42297c690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x2db
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x2be7c60, 0xc42297c690, 0x317d620, 0xc4208d6160, 0x316e020, 0xc423972570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:415 +0xd3
github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x2be7c60, 0xc42297c690, 0x317d620, 0xc4208d6160, 0xc4203f2100, 0x2d4a180, 0x316e2a0, 0xc4206c60c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x16e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc421f10000, 0x3185540, 0xc420dfc400, 0xc4210cc200, 0xc4233891a0, 0x3fd56a0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1167 +0x995
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc421f10000, 0x3185540, 0xc420dfc400, 0xc4210cc200, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1253 +0x12f9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42036aa30, 0xc421f10000, 0x3185540, 0xc420dfc400, 0xc4210cc200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:680 +0xad
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:678 +0xb9

goroutine 6283 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc42088c3c0, 0x316e020, 0xc4208979e0, 0xc42088c410, 0x18, 0xc42359cd20, 0x3, 0x3, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1298 +0x60c
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc42088c3c0, 0x316e020, 0xc4208979e0, 0x18, 0xc42359cd20, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:393 +0x2bf
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc42088c3c0, 0x316e020, 0xc4208979e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42230e900, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:460 +0x266
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc42088c3c0, 0x316e020, 0xc4208979e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42230e900, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1067 +0x3de
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc42088c3c0, 0x316e020, 0xc4208979e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42230e900, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:739 +0x1bdc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc42088c3c0, 0x316e020, 0xc4208979e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42230e600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:655 +0x566
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42003cc40, 0x316e020, 0xc420897980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42230e600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x2c5
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4207981e0, 0x316e020, 0xc420896c90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42230e600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x107
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4235ef1e0, 0x316e020, 0xc420896c90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:954 +0x66f
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*txnKVFetcher).fetch(0xc420f1b110, 0x316e020, 0xc420896c90, 0xc420d22078, 0x214f25e)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/kvfetcher.go:299 +0x6c1
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*txnKVFetcher).nextBatch(0xc420f1b110, 0x316e020, 0xc420896c90, 0x8, 0x2af6b60, 0x0, 0x0, 0x9, 0xc4215d4300)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/kvfetcher.go:363 +0xb9
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).nextKV(0xc4215d44a0, 0x316e020, 0xc420896c90, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:442 +0x211
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).NextKey(0xc4215d44a0, 0x316e020, 0xc420896c90, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:476 +0x9e
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).StartScanFrom(0xc4215d44a0, 0x316e020, 0xc420896c90, 0x3154420, 0xc420f1b110, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:432 +0xf7
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).StartScan(0xc4215d44a0, 0x316e020, 0xc420896c90, 0xc4235ef1e0, 0xc420897620, 0x1, 0x1, 0xc4215d4301, 0x0, 0x80b600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:422 +0x298
github.com/cockroachdb/cockroach/pkg/sql.(*scanNode).initScan(0xc4215d4300, 0x316e020, 0xc420896c90, 0xc420229098, 0xc420229028, 0xc421592150, 0xc420d22498)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/scan.go:242 +0x207
github.com/cockroachdb/cockroach/pkg/sql.(*scanNode).Next(0xc4215d4300, 0x316e020, 0xc420896c90, 0xc420229098, 0xc420229028, 0xc4211d8360, 0x0, 0xc4211d8300)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/scan.go:202 +0x452
github.com/cockroachdb/cockroach/pkg/sql.(*renderNode).Next(0xc4206f1600, 0x316e020, 0xc420896c90, 0xc420229098, 0xc420229028, 0x816e01, 0xc421503780, 0xc420d22698)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/render.go:335 +0xab
github.com/cockroachdb/cockroach/pkg/sql.(*groupNode).Next(0xc4206f1760, 0x316e020, 0xc420896c90, 0xc420229098, 0xc420229028, 0x31632a0, 0xc4206f1760, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/group.go:434 +0xd7c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).forEachRow(0xc420228c00, 0x316e020, 0xc420896c90, 0xc420229098, 0xc420229028, 0x31632a0, 0xc4206f1760, 0xc420d22870, 0x886eb5, 0x88a4bc)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:764 +0x7d
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc420228c00, 0x316e020, 0xc420896c90, 0xc420229028, 0x3, 0x2ba4231353a8, 0xc421ea8700, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:709 +0x2ff
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc420228c00, 0x316e020, 0xc420896c90, 0x31704a0, 0xc421503440, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:657 +0xbaf
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc420228c00, 0x316e020, 0xc420896c90, 0x31704a0, 0xc421503440, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:398 +0x1090
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc420228c00, 0x316e020, 0xc420896c90, 0x31704a0, 0xc421503440, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:92 +0x46f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc420228c00, 0x316df60, 0xc421545d80, 0xc42032abd0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1055 +0x31a4
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4210fba00, 0x316df60, 0xc421545d80, 0x0, 0x0, 0xc424508022, 0x4, 0x0, 0x0, 0x3155d20, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:401 +0x1e1
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4210fba00, 0x316df60, 0xc421545d80, 0xc42003c700, 0x5400, 0x15000, 0xc421ccccc0, 0xc42032abd0, 0xc42032abb0, 0xc4218aea60, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:266 +0x1aa
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:265 +0x147b

goroutine 4690 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc420dfc200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x31d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0x1651

goroutine 4912 [IO wait]:
internal/poll.runtime_pollWait(0x2ba4230d3d60, 0x72, 0xc420b58cb8)
	/usr/local/go/src/runtime/netpoll.go:173 +0x5e
internal/poll.(*pollDesc).wait(0xc421734d18, 0x72, 0xc4210be300, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0xe5
internal/poll.(*pollDesc).waitRead(0xc421734d18, 0xffffffffffffff00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x4b
internal/poll.(*FD).Accept(0xc421734d00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:372 +0x2e2
net.(*netFD).accept(0xc421734d00, 0xc4204f83c0, 0x2ba42311f8e8, 0xc420497e00)
	/usr/local/go/src/net/fd_unix.go:238 +0x53
net.(*TCPListener).accept(0xc420db2a10, 0xc4210be2f0, 0xc4220dcec8, 0x8325fd)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x4e
net.(*TCPListener).Accept(0xc420db2a10, 0x2d4a630, 0xc421545680, 0xc4204f83c0, 0xc4210be240)
	/usr/local/go/src/net/tcpsock.go:259 +0x50
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc421545680, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xc1
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3(0x316e020, 0xc4207044e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1089 +0x47
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42326a4c0, 0xc4219e2870, 0xc420f3de20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4263 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4206c6000, 0xc42364eb90, 0xc420454da0, 0xf, 0xc42115e480, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:575 +0x5a1
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x316e020, 0xc4212e7680)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:507 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42107b8a0, 0xc421042240, 0xc4212e7650)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4191 [semacquire]:
sync.runtime_notifyListWait(0xc4211d2450, 0x78da00d6000007d8)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4211d2440)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42160ed80, 0x316e020, 0xc4211941b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x316e020, 0xc4211941b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420414770, 0xc421042240, 0xc420414760)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4567 [select]:
database/sql.(*DB).connectionOpener(0xc4212b7ea0, 0x316df60, 0xc4212f72c0)
	/usr/local/go/src/database/sql/sql.go:935 +0x13f
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:634 +0x2bd

goroutine 6607 [select]:
github.com/cockroachdb/cockroach/pkg/storage/txnwait.(*Queue).MaybeWaitForQuery(0xc4219cb8c0, 0x316e020, 0xc42068a180, 0x314bee0, 0xc42081f500, 0xc420b70d10, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/txnwait/queue.go:633 +0x521
github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeWaitForPushee(0xc421f3a000, 0x316e020, 0xc42068a150, 0xc4221c3630, 0xc42081f500, 0x0, 0xc4221c36a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3028 +0x726
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc421f3a000, 0x316e020, 0xc42068a150, 0x15323102b2817525, 0x0, 0x100000001, 0x1, 0x18, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2902 +0x6d0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4203be790, 0x316e020, 0xc42068a030, 0x0, 0x0, 0x100000001, 0x1, 0x18, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:185 +0x127
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x316e020, 0xc42068a030, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:921 +0x1e0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421042240, 0x316e020, 0xc42068a030, 0x2c3ae8f, 0x10, 0xc4221c3938, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:243 +0xfb
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4221bb800, 0x316e020, 0xc42068a030, 0xc42169de10, 0xc42068a030, 0x18b2bb9, 0x316df60)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:908 +0x363
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4221bb800, 0x316e020, 0xc42068a030, 0xc42169de10, 0x1f, 0x3b9aca00, 0x4000000000000000)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:949 +0xb8
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4208fb730, 0xc42169de00, 0x316df60, 0xc4228f52c0, 0xc420d4b230, 0xc42047e6b0, 0x8325fd)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:242 +0xa8f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4208fb730, 0x316df60, 0xc4228f52c0, 0xc420454da0, 0xf, 0x0, 0x0, 0x100000001, 0x1, 0x18, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:273 +0xa1
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4208fb730, 0x316df60, 0xc4228f52c0, 0xc42169dd60, 0xc422a58a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:224 +0xd8
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:222 +0x3a6

goroutine 4591 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4239a0180, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc42341aa20, 0xc420696950, 0xc4239a0180)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc42342c000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 4614 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc4221cfb90, 0xc4216d0070)
	/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x11d
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
	/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x57

goroutine 5718 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc42249c480)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1176 +0x1ad
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0x1558

goroutine 4646 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x316e020, 0xc42152ab40)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42080e760, 0xc4221cfb90, 0xc42080e750)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 4590 [IO wait]:
internal/poll.runtime_pollWait(0x2ba42518c2f0, 0x72, 0x3ff2fd8)
	/usr/local/go/src/runtime/netpoll.go:173 +0x5e
internal/poll.(*pollDesc).wait(0xc421224718, 0x72, 0x314ed00, 0x3ff2fd8, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0xe5
internal/poll.(*pollDesc).waitRead(0xc421224718, 0xc42237c000, 0x2000, 0x2000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x4b
internal/poll.(*FD).Read(0xc421224700, 0xc42237c000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x22a
net.(*netFD).Read(0xc421224700, 0xc42237c000, 0x2000, 0x2000, 0xc41ff34f42, 0xc41ff34f41, 0x2)
	/usr/local/go/src/net/fd_unix.go:202 +0x66
net.(*conn).Read(0xc420d3a478, 0xc42237c000, 0x2000, 0x2000, 0xc4212dd6f0, 0x8135ad, 0xc420e14000)
	/usr/local/go/src/net/net.go:176 +0x85
crypto/tls.(*block).readFromUntil(0xc4206c8600, 0x2ba425183378, 0xc420d3a478, 0x5, 0xc420d3a478, 0x2ba422fbb6c8)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc421368e00, 0x2d4c917, 0xc421368f20, 0x37)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc421368e00, 0xc4207a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc42341a540, 0xc42029cff8, 0x9, 0x9, 0x0, 0x0, 0xc4212ddc58)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x3149340, 0xc42341a540, 0xc42029cff8, 0x9, 0x9, 0x9, 0x816e0a, 0xc4208fc800, 0x10)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3149340, 0xc42341a540, 0xc42029cff8, 0x9, 0x9, 0xc4203ddc88, 0x2, 0x867fa2)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42029cff8, 0x9, 0x9, 0x3149340, 0xc42341a540, 0x0, 0x0, 0xc421636870, 0x3155ae0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42029cfc0, 0xc4216379e0, 0xc421637900, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42342c000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1129 +0x1fe
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:265 +0x10fc

goroutine 4722 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x316e020, 0xc421645710)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:738 +0x174
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420696f90, 0xc4221cfb90, 0xc423655280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
@cockroach-teamcity cockroach-teamcity added this to the 2.1 milestone May 26, 2018
@cockroach-teamcity cockroach-teamcity added C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot. labels May 26, 2018
@petermattis petermattis mentioned this issue May 30, 2018
Closed
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/89690e007f3f4bab973a107754f471d140d5373f

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=686671&tab=buildLog

1 similar comment
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/89690e007f3f4bab973a107754f471d140d5373f

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=686671&tab=buildLog

@tbg tbg assigned windchan7 and unassigned petermattis May 30, 2018
@tbg tbg added the A-kv-client Relating to the KV client and the KV interface. label May 30, 2018
@tbg
Copy link
Member

tbg commented May 30, 2018

[12:18:31] : [Step 2/2] F180526 12:18:31.168356 4166 storage/replica.go:3873 [n1,s1,r4/1:/System/{NodeLive…-tsd}] while purging index %d: while purging index 56: IO error: /go/src/github.com/cockroachdb/cockroach/artifacts/cockroach-auxiliary583040763/sideloading/4/4.1/i56.t6: File not found

@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/89690e007f3f4bab973a107754f471d140d5373f

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=687287&tab=buildLog

1 similar comment
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/89690e007f3f4bab973a107754f471d140d5373f

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=687287&tab=buildLog

@windchan7
Copy link
Contributor

#26259

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@windchan7 windchan7

Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Projects
None yet
Milestone
2.1
Development

No branches or pull requests
4 participants
@tbg @petermattis @cockroach-teamcity @windchan7