Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

github.com/cockroachdb/cockroach/pkg/ccl/backupccl: TestBackupRestoreResume/restore failed under stress #26120

Closed
cockroach-teamcity opened this issue May 26, 2018 · 3 comments
Closed

github.com/cockroachdb/cockroach/pkg/ccl/backupccl: TestBackupRestoreResume/restore failed under stress #26120

cockroach-teamcity opened this issue May 26, 2018 · 3 comments
Assignees
@windchan7
Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Milestone
2.1

Comments

@cockroach-teamcity
Copy link
Member

SHA: https://github.com/cockroachdb/cockroach/commits/cbbc13cd30b14bc16cd8ebe56be093f264465ebe

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=680968&tab=buildLog


goroutine 45959 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x54a2c01, 0xc4204bc780, 0x0, 0x12)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:956 +0xdd
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x4adbf40, 0xc400000004, 0x453bced, 0x12, 0xf21, 0xc420e50e00, 0xd1)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:834 +0x7f4
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x37be100, 0xc42410b770, 0xc400000004, 0x2, 0x321fdf1, 0x6, 0xc422307b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:154 +0x31e
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x37be100, 0xc42410b770, 0x1, 0x4, 0x321fdf1, 0x6, 0xc422307b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:55 +0x9a
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x37be100, 0xc42410b770, 0x321fdf1, 0x6, 0xc422307b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:149 +0x90
github.com/cockroachdb/cockroach/pkg/storage.fatalOnRaftReadyErr(0x37be100, 0xc42410b770, 0x323fac3, 0x16, 0x37985c0, 0xc4212e4a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3873 +0x219
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x37be100, 0xc42410b770, 0xc4227ec000, 0x37be100)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3576 +0x215
github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc424bd2480, 0x37be100, 0xc42410b770, 0xc423856ea0, 0xc422307e98, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3178 +0x169
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc424bd2480, 0x37be100, 0xc4237d0d50, 0xd)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3563 +0x302
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:226 +0x2f2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4244050c0, 0xc4277cc120, 0xc4244050b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1 [chan receive, 1 minutes]:
testing.(*T).Run(0xc420106960, 0x3240566, 0x17, 0x334c9b8, 0xc420607c01)
	/usr/local/go/src/testing/testing.go:825 +0x597
testing.runTests.func1(0xc420106960)
	/usr/local/go/src/testing/testing.go:1063 +0xa5
testing.tRunner(0xc420106960, 0xc420607d68)
	/usr/local/go/src/testing/testing.go:777 +0x16e
testing.runTests(0xc4203292a0, 0x4885b00, 0x2b, 0x2b, 0xbeba5e8a074d8cc0)
	/usr/local/go/src/testing/testing.go:1061 +0x4e2
testing.(*M).Run(0xc420480a00, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x2ce
github.com/cockroachdb/cockroach/pkg/ccl/backupccl.TestMain(0xc420480a00)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/backupccl/main_test.go:31 +0x180
main.main()
	_testmain.go:136 +0x22b

goroutine 6 [syscall, 8 minutes]:
os/signal.signal_recv(0x90ab41)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x4f

goroutine 8 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0x132
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x1b5

goroutine 9 [chan receive, 8 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x1cd

goroutine 34 [select, 8 minutes, locked to thread]:
runtime.gopark(0x3352088, 0x0, 0x3221579, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0xf9
runtime.selectgo(0xc420091f50, 0xc4200b82a0)
	/usr/local/go/src/runtime/select.go:392 +0x11d4
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x19f
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 51 [select]:
github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.(*worker).start(0xc4203310c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:144 +0x18f
created by github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.init.0
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:29 +0x9b

goroutine 45763 [chan receive]:
testing.(*T).Run(0xc423a364b0, 0x3223661, 0x7, 0xc4227e6960, 0x1)
	/usr/local/go/src/testing/testing.go:825 +0x597
github.com/cockroachdb/cockroach/pkg/ccl/backupccl_test.TestBackupRestoreResume(0xc423a364b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backup_test.go:777 +0x482
testing.tRunner(0xc423a364b0, 0x334c9b8)
	/usr/local/go/src/testing/testing.go:777 +0x16e
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:824 +0x565

goroutine 45979 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xee99abf700000bc6)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4244053a0, 0xc4277cc120, 0xc424405390)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46825 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x7a5164b20000077c)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bbac90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bbac90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963a20, 0xc4228fc630, 0xc424963a10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 921 [chan receive, 7 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4201fe900)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46004 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x37be100, 0xc4209b0450)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:296 +0x1af
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4213e6930, 0xc4277cc120, 0xc423b34b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45954 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xefc5de4500000be6)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194cb10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194cb10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424405000, 0xc4277cc120, 0xc424404ff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47171 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x3518fd5200000642)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42337e240)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42337e240)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422f8c5f0, 0xc421304f30, 0xc422f8c5e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47414 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420cbf500, 0x60, 0x3109020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:240 +0x155
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc420cbf500, 0x58, 0xc421a815e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:251 +0x3d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc4220bfba0, 0x30ba3a0, 0xc42097c6f0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:525 +0xadb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc422585d80, 0x30ba3a0, 0xc42097c6f0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x67
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4211af540, 0xc42097c6c0, 0xc421a816a8, 0xa4de23)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:366 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x37d4080, 0xc4211af540, 0xc421610ec0, 0xc4226e82d0, 0x37be100, 0xc42097c6c0, 0x8d627d, 0xc420dca2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x37be100, 0xc42097c6c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4211af580, 0xc4228fc630, 0xc42097c690)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46819 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0xdcf40d9800000770)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bbaa80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bbaa80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963960, 0xc4228fc630, 0xc424963950)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46951 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e660)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828280, 0xc421304f30, 0xc426828260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 113 [chan receive, 8 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4207db440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 114 [chan receive, 8 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4207db560)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 47158 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x37be100, 0xc4207553b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:473 +0x295
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421670f70, 0xc421304f30, 0xc4204608c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 36340 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4234085a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 29513 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc423b36120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46806 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f4000000754)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba7b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba7b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249637c0, 0xc4228fc630, 0xc4249637b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45757 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc42230a000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 45772 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc42053e000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x31d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0x1651

goroutine 46960 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4209b7680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46664 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x37be100, 0xc422320870)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a88f80, 0xc4228fc630, 0xc421b9ec60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46912 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420755da0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 45835 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x37be100, 0xc42194c2a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x63
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404a60, 0xc4277cc120, 0xc42125f580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45924 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xd6af227d00000bd1)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194c4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194c4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404c00, 0xc4277cc120, 0xc424404bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46733 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x37be100, 0xc424bf20c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1558 +0x499
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828ae0, 0xc4228fc630, 0xc420c4db80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47194 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x2fa1b01800000625)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5320, 0xc421304f30, 0xc4216e5310)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45841 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xd6af227d00000bc1)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194c420)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194c420)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404ba0, 0xc4277cc120, 0xc424404b90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47193 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x77b113e400000623)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b8c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b8c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5300, 0xc421304f30, 0xc4216e52f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 29514 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc423b36240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46592 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e4b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4268280d0, 0xc421304f30, 0xc4268280c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47179 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x37be100, 0xc42337eff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422f8c8c0, 0xc421304f30, 0xc422d265e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47196 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0xb3fed17a0000061f)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5370, 0xc421304f30, 0xc4216e5350)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46593 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4268280f0, 0xc421304f30, 0xc4268280e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45963 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0x4da56e1f00000bd4)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424405160, 0xc4277cc120, 0xc424405150)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47267 [select]:
database/sql.(*DB).connectionOpener(0xc421284820, 0x37be040, 0xc42061f200)
	/usr/local/go/src/database/sql/sql.go:935 +0x13f
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:634 +0x2bd

goroutine 45917 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc423f325b0, 0x37be100, 0xc4237d10b0, 0xc4277cc120, 0x294c102c, 0xed29afb87, 0x0, 0xc42484b500, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x24e
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc42484b500, 0x58)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x13c
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420f77320, 0xc421f84e80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:319 +0x2e2
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x37be100, 0xc4237d10b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x392
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42034eab0, 0xc4277cc120, 0xc420ee6320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46968 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4236ef160, 0x30, 0x30d7cc0, 0x100000000000001, 0xc42236f5f0)
	<autogenerated>:1 +0x80
crypto/tls.(*listener).Accept(0xc4236ef1a0, 0xc420050128, 0x2f087a0, 0x4864a00, 0x31dab80)
	/usr/local/go/src/crypto/tls/tls.go:52 +0x53
net/http.(*Server).Serve(0xc4203bcf70, 0x37ab140, 0xc4236ef1a0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2770 +0x1f6
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x37be100, 0xc42236f590)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1108 +0x89
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828540, 0xc421304f30, 0xc4236ef1c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46730 [chan receive]:
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).execInternal(0xc421518870, 0x37be100, 0xc42348dd10, 0x32265f1, 0x9, 0x0, 0xc42348b640, 0x32b3a12, 0x52, 0xc420910220, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:490 +0x980
github.com/cockroachdb/cockroach/pkg/sql.(*internalExecutorImpl).queryInternal(0xc421518870, 0x37be100, 0xc424bf2180, 0x32265f1, 0x9, 0x0, 0xc42348b640, 0x32b3a12, 0x52, 0xc420910220, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:240 +0x112
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).Query(0xc421518870, 0x37be100, 0xc424bf2180, 0x32265f1, 0x9, 0x0, 0x32b3a12, 0x52, 0xc420910220, 0x2, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/internal.go:226 +0x1a0
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).maybeAdoptJob(0xc4262a2630, 0x37be100, 0xc424bf2180, 0x37a0c40, 0xc423576200, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:485 +0x170
github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x37be100, 0xc424bf2180)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:272 +0x190
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828ab0, 0xc4228fc630, 0xc420958b10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 52260 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*hookFnNode).Next(0xc420d9bc80, 0x37be100, 0xc42348cd50, 0xc420eaa498, 0xc420eaa428, 0x37b21c0, 0xc420d9bc80, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/planhook.go:146 +0x1a4
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).forEachRow(0xc420eaa000, 0x37be100, 0xc42348cd50, 0xc420eaa498, 0xc420eaa428, 0x37b21c0, 0xc420d9bc80, 0xc42099e870, 0x92abc5, 0x92e1cc)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:764 +0x7d
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc420eaa000, 0x37be100, 0xc42348cd50, 0xc420eaa428, 0x3, 0x7f8edf33ec18, 0xc423a79b80, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:709 +0x2ff
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc420eaa000, 0x37be100, 0xc42348cd50, 0x37bf0c0, 0xc4208ecf00, 0xc423409320, 0x7, 0x7, 0xc424309f20, 0x1a, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:657 +0xbaf
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc420eaa000, 0x37be100, 0xc42348cd50, 0x37bf0c0, 0xc4208ecf00, 0xc423409320, 0x7, 0x7, 0xc424309f20, 0x1a, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:398 +0x1090
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc420eaa000, 0x37be100, 0xc42348cd50, 0x37bf0c0, 0xc4208ecf00, 0xc423409320, 0x7, 0x7, 0xc424309f20, 0x1a, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:92 +0x46f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc420eaa000, 0x37be040, 0xc420b81100, 0xc426829f10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1108 +0x1ebd
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc420dd0400, 0x37be040, 0xc420b81100, 0xc421281039, 0x4, 0xc421281043, 0x4, 0x0, 0x0, 0x37a2e40, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:401 +0x1e1
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc420dd0400, 0x37be040, 0xc420b81100, 0xc42150c000, 0x5400, 0x15000, 0xc422197c80, 0xc426829f10, 0xc426829f00, 0xc424c99ff0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:266 +0x1aa
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:265 +0x147b

goroutine 51718 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420cbab00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1373 +0x2ac
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420cbab00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:949 +0x247
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:940 +0x173

goroutine 45744 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x37be100, 0xc420f76030)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xbe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404000, 0xc4277cc120, 0xc42005c680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46949 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e600)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828230, 0xc421304f30, 0xc426828220)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 23902 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42290f7a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 47416 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc424014500, 0xc42168b130, 0x5, 0x5, 0xc421c6bd60, 0x8e79c0, 0x299ab60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x19c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc424014500, 0xc42168b130, 0x5, 0x5, 0x8e6c7c, 0xc421c6bd60, 0xc4221e3970)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x82
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc42097c900, 0xc42168b130, 0x5, 0x5, 0x1019163, 0xc421c6bd60, 0x1fffe0000114ab)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x71
io.ReadAtLeast(0x3798d60, 0xc42097c900, 0xc42168b130, 0x5, 0x5, 0x5, 0xc4221e3a48, 0x1043409, 0xc42053e800)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3798d60, 0xc42097c900, 0xc42168b130, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420cbf600, 0xc42168b130, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0x15d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42168b120, 0x7fffffff, 0xb, 0x0, 0x0, 0x0, 0x0, 0xe)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x99
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42168b120, 0x7f8ee4339050, 0x54a2f90, 0xc420cbf600, 0x0, 0x0, 0x30ba2c0, 0xc42457a680, 0x7fffffff, 0xc42722b560, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42042da20, 0x30ba2c0, 0xc42457a680, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:757 +0x1e9
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4211af600, 0x37be100, 0xc42097ca80, 0xc421297d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:434 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x37d40e0, 0xc4211af600, 0xc423a36b40, 0x37be100, 0xc42097ca80, 0x8d627d, 0xc420670c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x109
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x37be100, 0xc42097ca80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x75
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4211af620, 0xc4277cc120, 0xc42097ca20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46973 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc423879170)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 45929 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xee99abf700000bc8)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc4237d0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc4237d0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404cc0, 0xc4277cc120, 0xc424404ca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46666 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x37be100, 0xc422320900)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x350
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a88fd0, 0xc4228fc630, 0xc421b9eca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46731 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x37be100, 0xc424bba780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1420 +0x149
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828ac0, 0xc4228fc630, 0xc423a3c040)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46741 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x37be100, 0xc4237d0030)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424962000, 0xc4228fc630, 0xc422712000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45820 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x37be100, 0xc4281f6090)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x350
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4213e6830, 0xc4277cc120, 0xc423b34a40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 41199 [chan receive, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc421132b40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46715 [select]:
database/sql.(*DB).connectionOpener(0xc4248903c0, 0x37be040, 0xc420fa8640)
	/usr/local/go/src/database/sql/sql.go:935 +0x13f
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:634 +0x2bd

goroutine 46936 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x37be100, 0xc421ae4c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249625a0, 0xc421304f30, 0xc4237f71a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46646 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x37be100, 0xc421852210)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xbe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5690, 0xc4228fc630, 0xc42143eb80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46754 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f4000000789)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc423b6fbf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc423b6fbf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249630f0, 0xc4228fc630, 0xc4249630e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45914 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x37be100, 0xc4237d11d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42034ea70, 0xc4277cc120, 0xc420ee62c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 51729 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4211b0ac0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc4206de180, 0xc421543a00, 0xc4211b0ac0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4205cfb00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 46804 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x5b46c5bd00000755)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963780, 0xc4228fc630, 0xc424963770)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46929 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x37be100, 0xc42080b980)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:738 +0x174
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e4d70, 0xc421304f30, 0xc420387920)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45976 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0xd6af227d00000bfb)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194cd20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194cd20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424405340, 0xc4277cc120, 0xc424405330)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47248 [select]:
github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x37be100, 0xc423ba8900)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:108 +0x23f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5b00, 0xc421304f30, 0xc423ba87e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46958 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37be100, 0xc42100e7b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828380, 0xc421304f30, 0xc426828370)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46497 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x37be100, 0xc4253cdad0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1161 +0x5a5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423a88480, 0xc4228fc630, 0xc423a88470)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47207 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x2bc171820000062f)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141bbc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141bbc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e55b0, 0xc421304f30, 0xc4216e5550)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 54870 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc421b5c4b0, 0xc42457ae30, 0x5, 0x5, 0xc420c6d220, 0x8e79c0, 0x299ab60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x19c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421b5c4b0, 0xc42457ae30, 0x5, 0x5, 0x8e6c7c, 0xc420c6d220, 0xc4237828f0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x82
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc423381ec0, 0xc42457ae30, 0x5, 0x5, 0x1019163, 0xc420c6d220, 0x1fffe00000002d)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x71
io.ReadAtLeast(0x3798d60, 0xc423381ec0, 0xc42457ae30, 0x5, 0x5, 0x5, 0xc4237829c8, 0x1041979, 0xc4207f6000)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3798d60, 0xc423381ec0, 0xc42457ae30, 0x5, 0x5, 0xc42348a398, 0xc4209108e0, 0xc42348a380)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42417d400, 0xc42457ae30, 0x5, 0x5, 0xc42365cc00, 0x16, 0x600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0x15d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42457ae20, 0x7fffffff, 0xb, 0x0, 0x7210, 0xc420041220, 0x0, 0xe)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x99
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42457ae20, 0x7f8ee4339050, 0x54a2f90, 0xc42417d400, 0x0, 0x0, 0x30f5ce0, 0xc420910920, 0x7fffffff, 0xc4207de660, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc421846c30, 0x30f5ce0, 0xc420910920, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:539 +0x27c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4242b4900, 0x30f5ce0, 0xc420910920, 0x3292630, 0x35)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x67
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*distSQLFlowStreamClient).Recv(0xc4203173c0, 0xc4210e8500, 0x3292630, 0x35)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/api.pb.go:331 +0x83
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*outbox).listenForDrainSignalFromConsumer.func1(0x37be040, 0xc4210e8500)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/outbox.go:328 +0xfb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4277cc120, 0x37be040, 0xc4210e8500, 0xc4225a80d1, 0xd, 0x37e5320, 0xc42262bcc0, 0xc42457ae80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xf4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:259 +0x14d

goroutine 47050 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x111f8ccf00000648)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc421b35200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc421b35200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc422f8c4d0, 0xc421304f30, 0xc422f8c4a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 12967 [chan receive, 6 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42290ea20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 46331 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc4224907c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 46814 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f4000000769)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba9c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba9c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4249638c0, 0xc4228fc630, 0xc4249638b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47522 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc421930000, 0xc423adf000, 0x37be100, 0xc420bac7b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0x120
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x1200

goroutine 46772 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x46bf2f400000078d)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba390)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba390)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963430, 0xc4228fc630, 0xc424963420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47432 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4209b8240, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc4214d43c0, 0xc4216e5be0, 0xc4209b8240)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4207f6000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 45958 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0x4da56e1f00000bdb)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194cb70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194cb70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4244050a0, 0xc4277cc120, 0xc424405080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47146 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x77b113e40000064b)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b5f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b5f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5070, 0xc421304f30, 0xc4216e5060)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 45944 [semacquire]:
sync.runtime_notifyListWait(0xc420fa82d0, 0x2a20103200000be3)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420fa82c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc421304240, 0x37be100, 0xc42194c960)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42194c960)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424404ea0, 0xc4277cc120, 0xc424404e90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 47152 [semacquire]:
sync.runtime_notifyListWait(0xc420c69190, 0x2fa1b01800000652)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420c69180)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4262a3560, 0x37be100, 0xc42141b7a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc42141b7a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4216e5140, 0xc421304f30, 0xc4216e5130)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46736 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startLeaseRenewer.func1(0x37be100, 0xc424bf2120)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1607 +0x4b2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc426828b70, 0xc4228fc630, 0xc426828b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 46783 [semacquire]:
sync.runtime_notifyListWait(0xc4224a5110, 0x9bdc470e0000075b)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4224a5100)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4249a95f0, 0x37be100, 0xc424bba570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37be100, 0xc424bba570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc424963640, 0xc4228fc630, 0xc4249635d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 54890 [runnable]:
@cockroach-teamcity cockroach-teamcity added this to the 2.1 milestone May 26, 2018
@cockroach-teamcity cockroach-teamcity added C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot. labels May 26, 2018
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/915a266778aab15c98fc7e18639583fbe6532399

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=684084&tab=buildLog

1 similar comment
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/915a266778aab15c98fc7e18639583fbe6532399

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=684084&tab=buildLog

@tbg tbg assigned windchan7 and unassigned benesch May 30, 2018
@tbg tbg added the A-kv-client Relating to the KV client and the KV interface. label May 30, 2018
@windchan7
Copy link
Contributor

#26259

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@windchan7 windchan7

Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Projects
None yet
Milestone
2.1
Development

No branches or pull requests
4 participants
@benesch @tbg @cockroach-teamcity @windchan7