Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

github.com/cockroachdb/cockroach/pkg/ccl/importccl: TestImportCSVStmt/empty-file failed under stress #26084

Closed
cockroach-teamcity opened this issue May 25, 2018 · 1 comment
Closed

github.com/cockroachdb/cockroach/pkg/ccl/importccl: TestImportCSVStmt/empty-file failed under stress #26084

cockroach-teamcity opened this issue May 25, 2018 · 1 comment
Assignees
@windchan7
Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Milestone
2.1

Comments

@cockroach-teamcity
Copy link
Member

SHA: https://github.com/cockroachdb/cockroach/commits/20a78c0d80bac6bf557598776680612c6c588df5

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=679669&tab=buildLog


I180525 11:17:54.224250 33636 storage/replica_command.go:863  [split,n2,s2,r225/2:/{Table/63/3/4…-Max}] initiating a split of this range at key /Table/64 [r226]
I180525 11:17:54.733169 2474 storage/replica_proposal.go:202  [n2,s2,r225/2:/{Table/63/3/4…-Max}] new range lease repl=(n2,s2):2 seq=4 start=1527247051.160735748,0 epo=1 pro=1527247051.160748990,0 following repl=(n2,s2):2 seq=4 start=1527247051.160735748,0 epo=1 pro=1527247051.160748990,0
I180525 11:17:54.787358 1976 server/status/runtime.go:219  [n1] runtime stats: 1.5 GiB RSS, 592 goroutines, 68 MiB/28 MiB/122 MiB GO alloc/idle/total, 210 MiB/247 MiB CGO alloc/total, 1496.73cgo/sec, 0.79/0.06 %(u/s)time, 0.01 %gc (4x)
I180525 11:17:55.660920 33693 ccl/importccl/read_import_proc.go:76  [import-distsql,n1] could not fetch file size; falling back to per-file progress: <nil>
I180525 11:17:56.523240 33902 ccl/importccl/read_import_proc.go:76  [import-distsql,n1] could not fetch file size; falling back to per-file progress: <nil>
F180525 11:17:57.338775 2564 storage/replica.go:3873  [n2,s2,r56/2:/Table/53/3/{2801/"…-3467/"…}] while purging index %d: while purging index 115: IO error: /go/src/github.com/cockroachdb/cockroach/artifacts/cockroach-auxiliary921324148/sideloading/56/56.2/i115.t97: File not found
goroutine 2564 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x53f5b01, 0xc4204ac960, 0x0, 0x12)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:956 +0xdd
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x4a2ee60, 0xc400000004, 0x44a77e2, 0x12, 0xf21, 0xc423335d10, 0xe7)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:834 +0x7f4
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x3747ca0, 0xc4239de6c0, 0xc400000004, 0x2, 0x31bcf26, 0x6, 0xc42452fb88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:154 +0x31e
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x3747ca0, 0xc4239de6c0, 0x1, 0x4, 0x31bcf26, 0x6, 0xc42452fb88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:55 +0x9a
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x3747ca0, 0xc4239de6c0, 0x31bcf26, 0x6, 0xc42452fb88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:149 +0x90
github.com/cockroachdb/cockroach/pkg/storage.fatalOnRaftReadyErr(0x3747ca0, 0xc4239de6c0, 0x31dd074, 0x16, 0x37236e0, 0xc4203810c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3873 +0x219
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x3747ca0, 0xc4239de6c0, 0xc4217eea80, 0x3747ca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3576 +0x215
github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc421514000, 0x3747ca0, 0xc4239de6c0, 0xc4216d8820, 0xc42452fe98, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3178 +0x169
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc421514000, 0x3747ca0, 0xc420ed3950, 0x38)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3563 +0x302
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420ed3950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:226 +0x2f2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420ed3950)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42039cb50, 0xc420ce66c0, 0xc42039cb30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1 [chan receive, 4 minutes]:
testing.(*T).Run(0xc42024a960, 0x31d1912, 0x11, 0x32e8010, 0xc4205a7c01)
	/usr/local/go/src/testing/testing.go:825 +0x597
testing.runTests.func1(0xc42024a960)
	/usr/local/go/src/testing/testing.go:1063 +0xa5
testing.tRunner(0xc42024a960, 0xc4205a7d68)
	/usr/local/go/src/testing/testing.go:777 +0x16e
testing.runTests(0xc420332aa0, 0x47e8b60, 0x10, 0x10, 0xbeba19d56c319ae9)
	/usr/local/go/src/testing/testing.go:1061 +0x4e2
testing.(*M).Run(0xc420448c00, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x2ce
github.com/cockroachdb/cockroach/pkg/ccl/importccl.TestMain(0xc420448c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/importccl/main_test.go:31 +0x180
main.main()
	_testmain.go:78 +0x22b

goroutine 21 [syscall, 4 minutes]:
os/signal.signal_recv(0x90ab41)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x4f

goroutine 3 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1158 +0x132
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x1b5

goroutine 4 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x1cd

goroutine 5 [select, 4 minutes, locked to thread]:
runtime.gopark(0x32ed6d8, 0x0, 0x31be61e, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0xf9
runtime.selectgo(0xc420499f50, 0xc4204ad500)
	/usr/local/go/src/runtime/select.go:392 +0x11d4
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x19f
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 23 [select]:
github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.(*worker).start(0xc4203ff400)
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:144 +0x18f
created by github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.init.0
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:29 +0x9b

goroutine 2549 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc420fe9800)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421305a00, 0xc420ce66c0, 0xc421068d20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2554 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc420fe9980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421305a60, 0xc420ce66c0, 0xc421068dc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1697 [chan receive]:
testing.(*T).Run(0xc4218a6690, 0x31c59ef, 0xa, 0xc4207c4f00, 0xc4218fe301)
	/usr/local/go/src/testing/testing.go:825 +0x597
github.com/cockroachdb/cockroach/pkg/ccl/importccl.TestImportCSVStmt(0xc4218a6690)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/importccl/import_stmt_test.go:451 +0x1216
testing.tRunner(0xc4218a6690, 0x32e8010)
	/usr/local/go/src/testing/testing.go:777 +0x16e
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:824 +0x565

goroutine 1759 [semacquire]:
sync.runtime_notifyListWait(0xc421b89490, 0xc965910f0000590d)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421b89480)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ce6fc0, 0x3747ca0, 0xc42205f080)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc42205f080)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204ece90, 0xc421215170, 0xc4204ece80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2728 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x3747ca0, 0xc4212ab9e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x1a2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f720, 0xc420ce6630, 0xc42043f700)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2724 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1(0x3747ca0, 0xc420d4aff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:775 +0x25d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f630, 0xc420ce6630, 0xc4204e3000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2493 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc4208b56e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203ccfe0, 0xc420ce6630, 0xc4203ccfd0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1995 [select]:
github.com/cockroachdb/cockroach/pkg/kv.GRPCTransportFactory.func1(0x3747c20, 0xc4200de058)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport_race.go:96 +0x376
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421215170, 0x3747c20, 0xc4200de058, 0xc42140c560, 0x17, 0x0, 0x0, 0xc4202e8ac0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xf4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:259 +0x14d

goroutine 1902 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc420ef1cc0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc421657e00, 0xc420407460, 0xc420ef1cc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc421893d40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 2314 [IO wait]:
internal/poll.runtime_pollWait(0x7f19ad2c8f38, 0x72, 0x47fe018)
	/usr/local/go/src/runtime/netpoll.go:173 +0x5e
internal/poll.(*pollDesc).wait(0xc42034f418, 0x72, 0x3726400, 0x47fe018, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0xe5
internal/poll.(*pollDesc).waitRead(0xc42034f418, 0xc423e68000, 0x4000, 0x4000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x4b
internal/poll.(*FD).Read(0xc42034f400, 0xc423e68000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x22a
net.(*netFD).Read(0xc42034f400, 0xc423e68000, 0x4000, 0x4000, 0x0, 0x0, 0x141d)
	/usr/local/go/src/net/fd_unix.go:202 +0x66
net.(*conn).Read(0xc420866028, 0xc423e68000, 0x4000, 0x4000, 0xc4210c96f0, 0x8b722d, 0xc42003a000)
	/usr/local/go/src/net/net.go:176 +0x85
crypto/tls.(*block).readFromUntil(0xc4212fbc50, 0x7f19ade39208, 0xc420866028, 0x5, 0xc420866028, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc421414000, 0x32ed817, 0xc421414120, 0xc423e6800d)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc421414000, 0xc420b50000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc421898c60, 0xc42150a2d8, 0x9, 0x9, 0xc4210c5048, 0xc4210c9c48, 0x92aade)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x3720040, 0xc421898c60, 0xc42150a2d8, 0x9, 0x9, 0x9, 0xc4210c9d01, 0x8baa8a, 0xc420083000)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3720040, 0xc421898c60, 0xc42150a2d8, 0x9, 0x9, 0xc4210c5050, 0xc425e23500, 0x90bc22)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42150a2d8, 0x9, 0x9, 0x3720040, 0xc421898c60, 0x0, 0xc400000000, 0xc425e23500, 0x1414)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42150a2a0, 0xc421b7a540, 0xc421b7a500, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421408000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1129 +0x1fe
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:265 +0x10fc

goroutine 2367 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x3747ca0, 0xc4212fbb90)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1190 +0x2fe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204ec2d0, 0xc420ce66c0, 0xc4204ec2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2008 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc421dd0000, 0xc4200e5cc0, 0xc4203b91b0, 0xf, 0xc42143f080, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:575 +0x5a1
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x3747ca0, 0xc4208acb70)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:507 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f230, 0xc421215170, 0xc4208acb40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2479 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xc36da193000042ce)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420ac3980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420ac3980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e7a0, 0xc420ce66c0, 0xc42043e790)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2471 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xff043d49000042d2)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420f36a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420f36a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e640, 0xc420ce66c0, 0xc42043e630)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2473 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0x3a44fe11000042d3)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420f36a50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420f36a50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e680, 0xc420ce66c0, 0xc42043e670)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2474 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0x2378c5ed000042db)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc4212175f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc4212175f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e6a0, 0xc420ce66c0, 0xc42043e690)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2638 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x3747ca0, 0xc4212ab950)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:738 +0x174
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f490, 0xc420ce6630, 0xc4204e2f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2475 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xff043d49000042a7)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc421217620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc421217620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e6c0, 0xc420ce66c0, 0xc42043e6b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2477 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0x4e5dfd02000042ac)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420ac38c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420ac38c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e730, 0xc420ce66c0, 0xc42043e710)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2476 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xf46622b5000042ba)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420ac38f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420ac38f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e6e0, 0xc420ce66c0, 0xc42043e6d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 63 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42089d200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 64 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc42089d320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 2268 [select, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc4211d6720)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203042a0, 0xc420ce66c0, 0xc420304280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1680 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x3747ca0, 0xc4221ef650)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42031f250, 0xc421215170, 0xc42219ab40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2470 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x3747ca0, 0xc420f36a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x63
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e620, 0xc420ce66c0, 0xc420404740)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2352 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc4210f4900, 0xc42135e100, 0x3747ca0, 0xc4211d7c80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0x120
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x1200

goroutine 2422 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9(0x3747ca0, 0xc421916120)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1219 +0xb0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203049f0, 0xc420ce66c0, 0xc421964ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2420 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x3747ca0, 0xc4211d6d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1112 +0xbe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304910, 0xc420ce66c0, 0xc4211f9500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2421 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc42191e5e0, 0x32ec8e8, 0xc421aff180, 0x375db20, 0xc4221634a0)
	<autogenerated>:1 +0x80
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc421aff180, 0x3745560, 0xc42191e5e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:544 +0x2e1
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x3747ca0, 0xc4211d6d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1124 +0x8d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304920, 0xc420ce66c0, 0xc42191e700)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1761 [semacquire]:
sync.runtime_notifyListWait(0xc421b89490, 0xc965910f000058f2)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421b89480)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ce6fc0, 0x3747ca0, 0xc42205f2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc42205f2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204eced0, 0xc421215170, 0xc4204ecec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2409 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc42190c030)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304560, 0xc420ce66c0, 0xc420304510)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2416 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x3747ca0, 0xc4211d6960)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1077 +0x84
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304890, 0xc420ce66c0, 0xc42191e640)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1681 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420ef0a80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 1819 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x3747ca0, 0xc4212eeff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x1a2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421275650, 0xc421215170, 0xc421275630)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2586 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc421ada000, 0xc4218fe300, 0x3747ca0, 0xc420fe9080)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0x120
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x1200

goroutine 2478 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0x2378c5ed000042d8)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420ac3920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420ac3920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e760, 0xc420ce66c0, 0xc42043e740)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2415 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x3747ca0, 0xc4219160f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304800, 0xc420ce66c0, 0xc42191e540)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2520 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xc8645366000042a6)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc4212ab230)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc4212ab230)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f0b0, 0xc420ce66c0, 0xc42043f0a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2361 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc42112e000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 2490 [semacquire, 4 minutes]:
sync.runtime_notifyListWait(0xc4211ee608, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4211ee5f8)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4211ee4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0xb3
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x607

goroutine 2946 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42121f340)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1373 +0x2ac
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc42121f340)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:949 +0x247
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:940 +0x173

goroutine 1813 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x3747ca0, 0xc421d7ab10)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x83
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421274ce0, 0xc421215170, 0xc4203d4780)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1915 [semacquire]:
sync.runtime_notifyListWait(0xc421b89490, 0xc965910f0000590f)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421b89480)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ce6fc0, 0x3747ca0, 0xc421d0fb90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc421d0fb90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204ed010, 0xc421215170, 0xc4204ed000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2411 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc4219c3440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 2686 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc420b002d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421275670, 0xc420ce6630, 0xc42038d5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1940 [semacquire]:
sync.runtime_notifyListWait(0xc421b89490, 0x785b3962000058ed)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421b89480)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ce6fc0, 0x3747ca0, 0xc4215aaa50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc4215aaa50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204ed330, 0xc421215170, 0xc4204ed320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1814 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x3747ca0, 0xc421d0f320)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1161 +0x5a5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421274d00, 0xc421215170, 0xc421274cf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1928 [semacquire]:
sync.runtime_notifyListWait(0xc421b89490, 0xc965910f00005910)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421b89480)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ce6fc0, 0x3747ca0, 0xc4215aa900)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc4215aa900)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204ed1b0, 0xc421215170, 0xc4204ed1a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2512 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xf46622b5000042b2)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420f36b10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420f36b10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043ee80, 0xc420ce66c0, 0xc42043ee50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1822 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x3747ca0, 0xc420f645a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:473 +0x295
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421275730, 0xc421215170, 0xc4217fa500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 6276 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc421658640, 0xc42162fd10, 0xc420452770, 0xf, 0xc42100e420, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:575 +0x5a1
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x3747ca0, 0xc421814960)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:507 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc423ba2530, 0xc420ce6630, 0xc421814900)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2373 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc420ce66c0, 0xc4211e10a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:122 +0x11d
created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler
	/go/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:119 +0x57

goroutine 2662 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc421917470)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203cd100, 0xc420ce6630, 0xc4203cd0f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2364 [semacquire]:
sync.runtime_notifyListWait(0xc4201b1988, 0x134b)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4201b1978)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4201b1860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0xb3
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x607

goroutine 2536 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0x12c25f33000042b7)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc4212ab980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc4212ab980)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42039c950, 0xc420ce66c0, 0xc42039c940)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2661 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc4208b57d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203cd0e0, 0xc420ce6630, 0xc4203cd0d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1766 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc4221ef2f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421274600, 0xc421215170, 0xc4212745f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1732 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4208c6240, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc420648360, 0xc420356110, 0xc4208c6240)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420b0c000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:276 +0x1ab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:273 +0x162c

goroutine 2552 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc420fe98c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421305a30, 0xc420ce66c0, 0xc421068d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1769 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc421d0efc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421274660, 0xc421215170, 0xc421274650)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2410 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc4211d6870)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304580, 0xc420ce66c0, 0xc420304570)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2551 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc420fe9710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421305a20, 0xc420ce66c0, 0xc421068d60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2583 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc42101c100, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc421200420, 0xc42039cdf0, 0xc42101c100)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4210d2000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 2398 [select, 4 minutes]:
database/sql.(*DB).connectionResetter(0xc421075180, 0x3747be0, 0xc4216d1b80)
	/usr/local/go/src/database/sql/sql.go:948 +0x155
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:635 +0x2f3

goroutine 2634 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc420c2e000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x31d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0x1651

goroutine 2443 [semacquire, 4 minutes]:
sync.runtime_SemacquireMutex(0xc422063234, 0x900000000)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc422063230)
	/usr/local/go/src/sync/mutex.go:134 +0x172
sync.(*Once).Do(0xc422063230, 0xc420dddf50)
	/usr/local/go/src/sync/once.go:40 +0x63
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func15(0x3747ca0, 0xc420ac2540)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1388 +0xb9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421304f20, 0xc420ce66c0, 0xc42037c740)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2621 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc421392db0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 2939 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc4206c0600, 0xc42135fc00, 0x3747ca0, 0xc420cc59b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0x120
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x1200

goroutine 2482 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc42191e5c0, 0x28, 0xc420d00720, 0x7f19b4c019b0, 0x0)
	<autogenerated>:1 +0x80
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc4210a0088, 0x3747ca0, 0xc420d006f0, 0xc420ce66c0, 0x3745560, 0xc42191e5c0, 0xc420d00720, 0xc420ddf6a8, 0xa4de23)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:119 +0x61
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func18(0x3747ca0, 0xc420d006f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1580 +0x2cd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421275020, 0xc420ce66c0, 0xc420cdac40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2553 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc420fe9920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421305a50, 0xc420ce66c0, 0xc421068da0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1763 [chan receive, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x3747ca0, 0xc421d0ef30)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:321 +0x6c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e9c0, 0xc421215170, 0xc42149cca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2389 [select]:
net.(*pipe).read(0xc420be2900, 0xc421adcc00, 0x400, 0x400, 0x16a5, 0x2000, 0xc4219136f0)
	/usr/local/go/src/net/pipe.go:164 +0x290
net.(*pipe).Read(0xc420be2900, 0xc421adcc00, 0x400, 0x400, 0xc4204e36c0, 0xc421913720, 0xbdcc7a)
	/usr/local/go/src/net/pipe.go:147 +0x61
crypto/tls.(*block).readFromUntil(0xc4212fbe00, 0x7f19b3abc478, 0xc420be2900, 0x5, 0xc420be2900, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc421512000, 0x32ed817, 0xc421512120, 0xc421adcc0d)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc421512000, 0xc42183c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc421abc060, 0xc42032a1f8, 0x9, 0x9, 0x7f19a58d7988, 0xc421ac3a40, 0x8e79c0)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x3720040, 0xc421abc060, 0xc42032a1f8, 0x9, 0x9, 0x9, 0xc421ac3a40, 0xc4205a6900, 0xc421913cc8)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3720040, 0xc421abc060, 0xc42032a1f8, 0x9, 0x9, 0x8e792a, 0xc4205a5800, 0x90bc22)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42032a1f8, 0x9, 0x9, 0x3720040, 0xc421abc060, 0x0, 0x0, 0xc421ac3a40, 0xc421913da7)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42032a1c0, 0xc421877b80, 0xc421877b00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421794000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1129 +0x1fe
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:265 +0x10fc

goroutine 2663 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc4208b5800)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203cd120, 0xc420ce6630, 0xc4203cd110)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2225 [select, 4 minutes]:
database/sql.(*DB).connectionOpener(0xc420efe320, 0x3747be0, 0xc421723c40)
	/usr/local/go/src/database/sql/sql.go:935 +0x13f
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:634 +0x2bd

goroutine 1767 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc421d0ef90)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421274620, 0xc421215170, 0xc421274610)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2393 [select]:
net.(*pipe).read(0xc420be2880, 0xc4217f5000, 0x400, 0x400, 0x16a5, 0x2000, 0xc421d91680)
	/usr/local/go/src/net/pipe.go:164 +0x290
net.(*pipe).Read(0xc420be2880, 0xc4217f5000, 0x400, 0x400, 0xc42037c200, 0xc421d916b0, 0xbdcc7a)
	/usr/local/go/src/net/pipe.go:147 +0x61
crypto/tls.(*block).readFromUntil(0xc4211d6de0, 0x7f19b3abc478, 0xc420be2880, 0x5, 0xc420be2880, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc4210c6e00, 0x32ed817, 0xc4210c6f20, 0xc4217f500d)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc4210c6e00, 0xc421312000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc42114e780, 0xc4202aeff8, 0x9, 0x9, 0x1060100000020, 0x8, 0x557bfe345a50136)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x3720040, 0xc42114e780, 0xc4202aeff8, 0x9, 0x9, 0x9, 0x372d060, 0xc423cb3e40, 0x8f1040)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x3720040, 0xc42114e780, 0xc4202aeff8, 0x9, 0x9, 0x8, 0x8, 0x90bc22)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202aeff8, 0x9, 0x9, 0x3720040, 0xc42114e780, 0x0, 0x0, 0xc421877c60, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202aefc0, 0xc421877c60, 0xc421877c60, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420b0c200, 0xc420ac2450, 0x32ec940)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:427 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc421aff180, 0x37607c0, 0xc420b0c200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:676 +0x16f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2()
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:631 +0x54
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4210f9240, 0xc421aff180, 0x7f19b1a21000, 0xc420b0c200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:639 +0x35
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x7c7

goroutine 2660 [select, 4 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3747ca0, 0xc421917440)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203cd0c0, 0xc420ce6630, 0xc4203cd0b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2391 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc421412480, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc421abc120, 0xc4203820b0, 0xc421412480)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420b0c200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:276 +0x1ab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:273 +0x162c

goroutine 2336 [semacquire]:
sync.runtime_notifyListWait(0xc4210c53d0, 0xff043d49000042c4)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc4210c53c0)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42135a900, 0x3747ca0, 0xc420d017a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x3747ca0, 0xc420d017a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42039c840, 0xc420ce66c0, 0xc42039c830)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2377 [select, 4 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc42190c1e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 1728 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x3747ca0, 0xc4215ab050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:586 +0x1c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202e8560, 0xc421215170, 0xc421068500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2390 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4216d0040, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc421abc0c0, 0xc420382060, 0xc4216d0040)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc421794000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
@cockroach-teamcity cockroach-teamcity added this to the 2.1 milestone May 25, 2018
@cockroach-teamcity cockroach-teamcity added C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot. labels May 25, 2018
@tbg tbg assigned windchan7 and unassigned dt May 30, 2018
@tbg tbg added the A-kv-client Relating to the KV client and the KV interface. label May 30, 2018
@windchan7
Copy link
Contributor

#26259

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@windchan7 windchan7

Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Projects
None yet
Milestone
2.1
Development

No branches or pull requests
4 participants
@dt @tbg @cockroach-teamcity @windchan7