Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

github.com/cockroachdb/cockroach/pkg/ccl/importccl: TestImportCSVStmt/schema-in-file-sstsize failed under stress #26123

Closed
cockroach-teamcity opened this issue May 26, 2018 · 1 comment
Closed

github.com/cockroachdb/cockroach/pkg/ccl/importccl: TestImportCSVStmt/schema-in-file-sstsize failed under stress #26123

cockroach-teamcity opened this issue May 26, 2018 · 1 comment
Assignees
@windchan7
Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Milestone
2.1

Comments

@cockroach-teamcity
Copy link
Member

SHA: https://github.com/cockroachdb/cockroach/commits/cbbc13cd30b14bc16cd8ebe56be093f264465ebe

Parameters:

TAGS=
GOFLAGS=-race

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=681590&tab=buildLog


I180526 11:10:35.900653 22455 storage/replica_command.go:863  [split,n2,s2,r131/2:/Table/5{8-9/1/783}] initiating a split of this range at key /Table/59 [r63]
I180526 11:10:36.493462 2571 storage/replica_proposal.go:202  [n2,s2,r131/2:/Table/5{8-9/1/783}] new range lease repl=(n2,s2):2 seq=4 start=1527333019.187386845,0 epo=1 pro=1527333019.187401415,0 following repl=(n2,s2):2 seq=4 start=1527333019.187386845,0 epo=1 pro=1527333019.187401415,0
I180526 11:10:37.491299 1833 server/status/runtime.go:219  [n1] runtime stats: 1.2 GiB RSS, 590 goroutines, 48 MiB/37 MiB/109 MiB GO alloc/idle/total, 190 MiB/225 MiB CGO alloc/total, 2415.71cgo/sec, 0.88/0.05 %(u/s)time, 0.03 %gc (6x)
I180526 11:10:41.017576 2513 server/status/runtime.go:219  [n2] runtime stats: 1.2 GiB RSS, 628 goroutines, 78 MiB/13 MiB/109 MiB GO alloc/idle/total, 192 MiB/228 MiB CGO alloc/total, 1091.61cgo/sec, 0.78/0.05 %(u/s)time, 0.02 %gc (5x)
I180526 11:10:41.916955 2776 server/status/runtime.go:219  [n3] runtime stats: 1.2 GiB RSS, 609 goroutines, 55 MiB/31 MiB/109 MiB GO alloc/idle/total, 191 MiB/228 MiB CGO alloc/total, 1010.54cgo/sec, 0.76/0.05 %(u/s)time, 0.02 %gc (5x)
F180526 11:10:42.914224 2584 storage/replica.go:3873  [n2,s2,r145/2:/Table/59/2/"{H"/788-L"/3210}] while purging index %d: while purging index 28: IO error: /go/src/github.com/cockroachdb/cockroach/artifacts/cockroach-auxiliary451336369/sideloading/145/145.2/i28.t13: File not found
goroutine 2584 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x54b0301, 0xc4204a48a0, 0x0, 0x12)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:956 +0xdd
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x4ae9680, 0xc400000004, 0x454828d, 0x12, 0xf21, 0xc4207a84b0, 0xe4)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:834 +0x7f4
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x37c4140, 0xc420c5f020, 0xc400000004, 0x2, 0x3228e05, 0x6, 0xc4252f5b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:154 +0x31e
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x37c4140, 0xc420c5f020, 0x1, 0x4, 0x3228e05, 0x6, 0xc4252f5b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:55 +0x9a
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x37c4140, 0xc420c5f020, 0x3228e05, 0x6, 0xc4252f5b88, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:149 +0x90
github.com/cockroachdb/cockroach/pkg/storage.fatalOnRaftReadyErr(0x37c4140, 0xc420c5f020, 0x3249a5d, 0x16, 0x379e800, 0xc4213c5660)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3873 +0x219
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x37c4140, 0xc420c5f020, 0xc422f3ea80, 0x37c4140)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3576 +0x215
github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc421665b00, 0x37c4140, 0xc420c5f020, 0xc422d3e820, 0xc4241cfe98, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3178 +0x169
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc421665b00, 0x37c4140, 0xc421f4c690, 0x91)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3563 +0x302
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4213b4bd0, 0x37c4140, 0xc421f4c690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:226 +0x2f2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc421f4c690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203247f0, 0xc4201b0fc0, 0xc4203247e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1 [chan receive, 2 minutes]:
testing.(*T).Run(0xc420228960, 0x323e16b, 0x11, 0x3354e48, 0xc42062dc01)
	/usr/local/go/src/testing/testing.go:825 +0x597
testing.runTests.func1(0xc420228960)
	/usr/local/go/src/testing/testing.go:1063 +0xa5
testing.tRunner(0xc420228960, 0xc42062dd68)
	/usr/local/go/src/testing/testing.go:777 +0x16e
testing.runTests(0xc4202fbd20, 0x488aac0, 0x12, 0x12, 0xbeba6dde5668f9e2)
	/usr/local/go/src/testing/testing.go:1061 +0x4e2
testing.(*M).Run(0xc4206eea00, 0x0)
	/usr/local/go/src/testing/testing.go:978 +0x2ce
github.com/cockroachdb/cockroach/pkg/ccl/importccl.TestMain(0xc4206eea00)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/importccl/main_test.go:31 +0x180
main.main()
	_testmain.go:82 +0x22b

goroutine 6 [syscall, 3 minutes]:
os/signal.signal_recv(0x90ab41)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x4f

goroutine 8 [semacquire]:
sync.runtime_SemacquireMutex(0x4ae96d8, 0x900000000)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0x4ae96d4)
	/usr/local/go/src/sync/mutex.go:134 +0x172
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0x4ae96d4)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x3b
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1163 +0x160
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:591 +0x1b5

goroutine 9 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:598 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:592 +0x1cd

goroutine 10 [select, 3 minutes, locked to thread]:
runtime.gopark(0x335a528, 0x0, 0x322a599, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0xf9
runtime.selectgo(0xc42008ff50, 0xc4200b9920)
	/usr/local/go/src/runtime/select.go:392 +0x11d4
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x19f
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 26 [select]:
github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.(*worker).start(0xc420302e00)
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:144 +0x18f
created by github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.init.0
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:29 +0x9b

goroutine 1750 [chan receive]:
testing.(*T).Run(0xc420228a50, 0x3249509, 0x16, 0xc4238f84e0, 0xc422053301)
	/usr/local/go/src/testing/testing.go:825 +0x597
github.com/cockroachdb/cockroach/pkg/ccl/importccl.TestImportCSVStmt(0xc420228a50)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/importccl/import_stmt_test.go:452 +0x1216
testing.tRunner(0xc420228a50, 0x3354e48)
	/usr/local/go/src/testing/testing.go:777 +0x16e
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:824 +0x565

goroutine 2692 [semacquire, 2 minutes]:
sync.runtime_notifyListWait(0xc420caa948, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420caa938)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420caa820)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:648 +0xb3
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:635 +0x607

goroutine 2048 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4214ce8c0, 0xc421782460, 0xc42054c190, 0xf, 0xc420d62cc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:575 +0x5a1
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x37c4140, 0xc420fb83f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:507 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f0300, 0xc421f53170, 0xc420fb83c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1791 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc42015a510)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420386030, 0xc421f53170, 0xc420386020)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2535 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x37c4140, 0xc420fb9170)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x1a2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203241c0, 0xc4201b0fc0, 0xc4203241b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2698 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc42183e510)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d3cd0, 0xc420faa360, 0xc4202d3ca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2682 [select, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4207862c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1373 +0x2ac
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc4207862c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:949 +0x247
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:940 +0x173

goroutine 2983 [select, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc4213b0600, 0xc4218d8280, 0x37c4140, 0xc420667920)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:321 +0x120
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x1200

goroutine 2224 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420495180, 0x3357d78, 0xc421f53170, 0x37c4140, 0xc4218fccf0)
	<autogenerated>:1 +0x80
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc420d64018, 0x37c4140, 0xc4218fccf0, 0xc421f53170, 0x37c1880, 0xc420495180, 0xc4218fcd20, 0xc420d376a8, 0xa4de23)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:119 +0x61
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func18(0x37c4140, 0xc4218fccf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1580 +0x2cd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d24630, 0xc421f53170, 0xc420bd8600)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2284 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4212f4ac0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc421c78ea0, 0xc42036c440, 0xc4212f4ac0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4213cc000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:298 +0x101
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:296 +0x14cf

goroutine 3017 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420f67450, 0xc4203405b0, 0x5, 0x5, 0xc421c9a5e0, 0x8e79c0, 0x29a1010)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x19c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420f67450, 0xc4203405b0, 0x5, 0x5, 0x8e6c7c, 0xc421c9a5e0, 0xc424db9970)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:131 +0x82
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420feaf00, 0xc4203405b0, 0x5, 0x5, 0xfe8bb3, 0xc421c9a5e0, 0x1fffe00004dc5f)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:394 +0x71
io.ReadAtLeast(0x379efa0, 0xc420feaf00, 0xc4203405b0, 0x5, 0x5, 0x5, 0xc424db9a48, 0x1012e59, 0xc420f60600)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x379efa0, 0xc420feaf00, 0xc4203405b0, 0x5, 0x5, 0xc4251dfd10, 0xc4251dfd10, 0xc424db9ad0)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420c3c800, 0xc4203405b0, 0x5, 0x5, 0x8ae8ed, 0xc421c86240, 0x8e79c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:378 +0x15d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203405a0, 0x7fffffff, 0xb, 0x0, 0x92af86, 0xc420bb3b10, 0x0, 0xe)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:452 +0x99
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203405a0, 0x7f09bf41f590, 0x54b06d0, 0xc420c3c800, 0x0, 0x0, 0x30c2120, 0xc425ba7aa0, 0x7fffffff, 0xc422ab4540, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:578 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42098a2c0, 0x30c2120, 0xc425ba7aa0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:757 +0x1e9
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4202d2350, 0x37c4140, 0xc420feaff0, 0xc424961520)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:434 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x37da000, 0xc4202d2350, 0xc4214962d0, 0x37c4140, 0xc420feaff0, 0x8d627d, 0xc421463ac0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x109
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x37c4140, 0xc420feaff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x75
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d2370, 0xc420faa360, 0xc420feaf90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2697 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc420fb9710)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d3c80, 0xc420faa360, 0xc4202d3c20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2695 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc4210aa990)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d3bb0, 0xc420faa360, 0xc4202d3ba0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2694 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc4210aa900)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d3b90, 0xc420faa360, 0xc4202d3b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2693 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc420fb96b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202d3b20, 0xc420faa360, 0xc4202d3b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2679 [select, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc4213bed50)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x205
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x4d

goroutine 2985 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc421463cc0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc4219cb8c0, 0xc420550810, 0xc421463cc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420f60600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:276 +0x1ab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:273 +0x162c

goroutine 2509 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x37c4140, 0xc420d28a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:738 +0x174
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420324000, 0xc4201b0fc0, 0xc4203122e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2988 [select, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420c62000, 0x60, 0x3110c20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:240 +0x155
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc420c62000, 0x58, 0xc420ad8de0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:251 +0x3d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc420546820, 0x30c2200, 0xc420f68480, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:525 +0xadb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc422052480, 0x30c2200, 0xc420f68480, 0x692dd0f223332eaf, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:405 +0x67
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4204f8000, 0xc420f68450, 0xc420ad8ea8, 0xa4de23)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:366 +0x83
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x37d9fa0, 0xc4204f8000, 0xc42037e840, 0xc4217a12c0, 0x37c4140, 0xc420f68450, 0x8d627d, 0xc42147e2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x37c4140, 0xc420f68450)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204f80f0, 0xc421f53170, 0xc420f68420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 55 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420795200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 56 [chan receive, 3 minutes]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420795320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1210 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:82 +0x4bb

goroutine 2334 [chan receive, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x37c4140, 0xc421108360)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x7d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f0220, 0xc420faa360, 0xc421a221a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1792 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc42015a5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420386050, 0xc421f53170, 0xc420386040)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2944 [select, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420b85c00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x1d4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x294

goroutine 2014 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0xc36e44a200002e57)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc420667ef0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc420667ef0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203429c0, 0xc421f53170, 0xc4203429a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1988 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0x7934171900002e38)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc4b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc4b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203425e0, 0xc421f53170, 0xc4203425d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1990 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0xb80804200002e45)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc510)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc510)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420342640, 0xc421f53170, 0xc420342630)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1989 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0xdb0acc00002e51)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc480)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc480)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420342610, 0xc421f53170, 0xc420342600)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2753 [semacquire]:
sync.runtime_notifyListWait(0xc420bb3b10, 0xf83b15290000260d)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420bb3b00)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa900, 0x37c4140, 0xc421211f50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc421211f50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b93d0, 0xc420faa360, 0xc4203b93c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1592 [select, 2 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc421b58000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1373 +0x2ac
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc421b58000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:949 +0x247
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:940 +0x173

goroutine 2870 [semacquire]:
sync.runtime_notifyListWait(0xc420bb3b10, 0xf83b15290000262c)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420bb3b00)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa900, 0x37c4140, 0xc421b0cc60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc421b0cc60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9470, 0xc420faa360, 0xc4203b9460)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1986 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0xc36e44a200002e2b)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc450)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc450)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420342580, 0xc421f53170, 0xc420342570)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1979 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0xd84201fc00002e48)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc330)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc330)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420342440, 0xc421f53170, 0xc420342430)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2872 [semacquire]:
sync.runtime_notifyListWait(0xc420bb3b10, 0xd3e3932800002630)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420bb3b00)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa900, 0x37c4140, 0xc4218cc630)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218cc630)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b94b0, 0xc420faa360, 0xc4203b94a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2470 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc4217b6c00, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc421aa70e0, 0xc4203f1a30, 0xc4217b6c00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420590800)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:276 +0x1ab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:273 +0x162c

goroutine 1984 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0xdb0acc00002e2a)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc3c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc3c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420342530, 0xc421f53170, 0xc420342510)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2442 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc420692210)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420410670, 0xc4201b0fc0, 0xc420410660)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1790 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc42058cd50)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420386010, 0xc421f53170, 0xc420386000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2868 [semacquire]:
sync.runtime_notifyListWait(0xc420bb3b10, 0x4355d4b1000025fd)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc420bb3b00)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa900, 0x37c4140, 0xc421b0cc90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc421b0cc90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203b9430, 0xc420faa360, 0xc4203b9420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1836 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x37c4140, 0xc420666660)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1124 +0x632
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bc2420, 0xc421f53170, 0xc4217fa200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2510 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x37c4140, 0xc420d289f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:700 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420324010, 0xc4201b0fc0, 0xc420312420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 23465 [runnable]:
github.com/cockroachdb/cockroach/pkg/roachpb.Span.Contains(0xc4205a1560, 0x35, 0x60, 0xc4205a1560, 0x36, 0x60, 0xc423f8c8d0, 0x30, 0x30, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/data.go:1432 +0x2ed
github.com/cockroachdb/cockroach/pkg/storage/spanset.(*SpanSet).CheckAllowed(0xc421ef7620, 0x1, 0xc423f8c8d0, 0x30, 0x30, 0x0, 0x0, 0x0, 0xc422f62f78, 0xc4250301a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/spanset/spanset.go:117 +0x16b
github.com/cockroachdb/cockroach/pkg/storage/spanset.spanSetWriter.Merge(0x7f09bba5c270, 0xc420dca000, 0xc421ef7620, 0xc423f8c8d0, 0x30, 0x30, 0x0, 0x0, 0xc421f48980, 0x40, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/spanset/batch.go:280 +0xbc
github.com/cockroachdb/cockroach/pkg/storage/engine.MVCCMerge(0x37c4140, 0xc420c5e7e0, 0x7f09bba5dc18, 0xc420a20370, 0xc4230d43f0, 0xc423f8c8d0, 0x30, 0x30, 0x15322d4f6a83e4ea, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/mvcc.go:1587 +0x33f
github.com/cockroachdb/cockroach/pkg/storage/batcheval.Merge(0x37c4140, 0xc420c5e7e0, 0x7f09bba5dc18, 0xc420a20370, 0x37f7540, 0xc421432150, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/batcheval/cmd_merge.go:41 +0x194
github.com/cockroachdb/cockroach/pkg/storage.evaluateCommand(0x37c4140, 0xc420c5e7e0, 0xc4247a7410, 0x8, 0x1bb, 0x7f09bba5dc18, 0xc420a20370, 0x37f7540, 0xc421432150, 0xc4230d43f0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:101 +0x373
github.com/cockroachdb/cockroach/pkg/storage.evaluateBatch(0x37c4140, 0xc420c5e7e0, 0xc4247a7410, 0x8, 0x7f09bba5dc18, 0xc420a20370, 0x37f7540, 0xc421432150, 0xc4230d43f0, 0x15322d4f6a83e4ea, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:5527 +0x42e
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateWriteBatchWithLocalRetries(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0xc4247a7410, 0x8, 0x37f7540, 0xc421432150, 0xc4230d43f0, 0x15322d4f6a83e4ea, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:5304 +0x44b
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateWriteBatch(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0xc4247a7410, 0x8, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:5280 +0x298
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateProposal(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0xc4247a7410, 0x8, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2935 +0x17e
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).requestToProposal(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0xc4247a7410, 0x8, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2883 +0x118
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0x15322d2f19efbcd0, 0x0, 0xc4202ff300, 0x200000002, 0x3, 0x0, 0xc4202ff310, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3144 +0x52d
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2788 +0x51a
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2637 +0xbf
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc422ffe000, 0x37c4140, 0xc420c5e7e0, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1876 +0x7ce
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc421665b00, 0x37c4140, 0xc420c5e630, 0x15322d4f6a83e4ea, 0x0, 0x200000002, 0x3, 0x5, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2905 +0x752
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421924630, 0x37c4140, 0xc420c5e5d0, 0x0, 0x0, 0x200000002, 0x3, 0x5, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:185 +0x127
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x37c4140, 0xc420c5e5d0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:921 +0x1e0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc4201b0fc0, 0x37c4140, 0xc420c5e5d0, 0x323cced, 0x10, 0xc425035938, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:243 +0xfb
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4209ed800, 0x37c4140, 0xc420c5e5d0, 0xc423227af0, 0xc420c5e5d0, 0x1be16f9, 0x37c4080)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:908 +0x363
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4209ed800, 0x37c4140, 0xc420c5e5d0, 0xc423227af0, 0x1f, 0x1, 0x7f09b8a16c7a)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:949 +0xb8
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4218f6770, 0xc423227ae0, 0x37c4080, 0xc4247a45c0, 0xc4207c85f0, 0xc420c95eb0, 0x8d627d)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:242 +0xa8f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4218f6770, 0x37c4080, 0xc4247a45c0, 0xc4215ec2d0, 0xf, 0x0, 0x0, 0x200000002, 0x3, 0x5, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:273 +0xa1
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4218f6770, 0x37c4080, 0xc4247a45c0, 0xc423227a40, 0xc421ef7500)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:224 +0xd8
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:222 +0x3a6

goroutine 3276 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc421d283c0, 0xc421c6f400, 0xc421999390, 0xf, 0xc4210ccba0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:575 +0x5a1
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x37c4140, 0xc42132efc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:507 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ef110, 0xc4201b0fc0, 0xc42132ef90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 2485 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc420f60200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:869 +0x31d
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:281 +0x1651

goroutine 6693 [IO wait]:
internal/poll.runtime_pollWait(0x7f09bf463130, 0x72, 0x489fd58)
	/usr/local/go/src/runtime/netpoll.go:173 +0x5e
internal/poll.(*pollDesc).wait(0xc42098d098, 0x72, 0x37a1500, 0x489fd58, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0xe5
internal/poll.(*pollDesc).waitRead(0xc42098d098, 0xc42069d000, 0x400, 0x400)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x4b
internal/poll.(*FD).Read(0xc42098d080, 0xc42069d000, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x22a
net.(*netFD).Read(0xc42098d080, 0xc42069d000, 0x400, 0x400, 0x8, 0x7f09b8a14560, 0xc423449640)
	/usr/local/go/src/net/fd_unix.go:202 +0x66
net.(*conn).Read(0xc4204a8500, 0xc42069d000, 0x400, 0x400, 0xc4234496f0, 0x8b722d, 0xc42003a000)
	/usr/local/go/src/net/net.go:176 +0x85
crypto/tls.(*block).readFromUntil(0xc4217034d0, 0x7f09bf474568, 0xc4204a8500, 0x5, 0xc4204a8500, 0x7f09b8a0df3c)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc422fff500, 0x335a617, 0xc422fff620, 0x7f09b8a158b1)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc422fff500, 0xc4243f2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc423223f20, 0xc421aca038, 0x9, 0x9, 0x8bc8a0, 0x7f09be2e81a8, 0x67)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x379b140, 0xc423223f20, 0xc421aca038, 0x9, 0x9, 0x9, 0x8baa8a, 0xc4204c8800, 0x10)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x379b140, 0xc423223f20, 0xc421aca038, 0x9, 0x9, 0xc421ce5248, 0x2, 0x90bc22)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc421aca038, 0x9, 0x9, 0x379b140, 0xc423223f20, 0x0, 0x0, 0xc422851980, 0x37a8f00)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc421aca000, 0xc421456e10, 0xc421456e00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc424048fc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1129 +0x1fe
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:265 +0x10fc

goroutine 23421 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0xc421390500, 0x91, 0xc421945400, 0x3, 0x3, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1298 +0x60c
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x91, 0xc421945400, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:393 +0x2bf
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:460 +0x266
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1067 +0x3de
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:739 +0x1bdc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1133 +0xb13
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:739 +0x1bdc
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4213904b0, 0x37c4140, 0xc4213771a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:655 +0x566
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc420b44540, 0x37c4140, 0xc421377140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x2c5
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420ff7620, 0x37c4100, 0xc4219a4360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x107
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420ff7620, 0x37c4100, 0xc4219a4360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xb6
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x37c4100, 0xc4219a4360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x97
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x37c4100, 0xc4219a4360, 0xc42346bb08, 0xc423f0a000, 0xc42346bb18, 0x154d11b)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0x13e
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc420ff7620, 0x37c4100, 0xc4219a4360, 0xc423f0a000, 0xc4217302f0, 0x8b84a1)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0xee
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).PutInline(0xc420ff7620, 0x37c4100, 0xc4219a4360, 0x30c84c0, 0xc421cbf100, 0x319aa60, 0xc4217302f0, 0x40, 0xda4c4a)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:265 +0xe9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).setQueueLastProcessed(0xc422f3ea80, 0x37c4100, 0xc4219a4360, 0x32549aa, 0x1b, 0x15322d4f6ef8b8ab, 0x0, 0xc421ffc810, 0xc420060501)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1720 +0x1d6
github.com/cockroachdb/cockroach/pkg/storage.(*consistencyQueue).process(0xc4202e5fa0, 0x37c4100, 0xc4219a4360, 0xc422f3ea80, 0x0, 0x0, 0x0, 0x2, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/consistency_queue.go:109 +0x16a
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc4207b3500, 0x37c4140, 0xc421376f60, 0xc422f3ea80, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:734 +0x314
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2(0x37c4140, 0xc421376f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:617 +0xde
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4201b0fc0, 0x37c4140, 0xc421376f60, 0xc420f98140, 0x3f, 0x0, 0x0, 0xc421cbf0e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xf4
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:259 +0x14d

goroutine 2512 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x37c4140, 0xc420d28a50)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x350
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420324070, 0xc4201b0fc0, 0xc420312780)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 1985 [semacquire]:
sync.runtime_notifyListWait(0xc421689a90, 0x9c0aaa1c00002e42)
	/usr/local/go/src/runtime/sema.go:510 +0x11a
sync.(*Cond).Wait(0xc421689a80)
	/usr/local/go/src/sync/cond.go:56 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420faa5a0, 0x37c4140, 0xc4218fc180)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x9c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x37c4140, 0xc4218fc180)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420342560, 0xc421f53170, 0xc420342540)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 3376 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc420993540, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:293 +0x1e9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc422380300, 0xc420343bc0, 0xc420993540)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/controlbuf.go:370 +0x236
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420be2c00)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:276 +0x1ab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:273 +0x162c

goroutine 2987 [IO wait]:
internal/poll.runtime_pollWait(0x7f09bf47b840, 0x72, 0x489fd58)
	/usr/local/go/src/runtime/netpoll.go:173 +0x5e
internal/poll.(*pollDesc).wait(0xc420f7b818, 0x72, 0x37a1500, 0x489fd58, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0xe5
internal/poll.(*pollDesc).waitRead(0xc420f7b818, 0xc42309c000, 0x8000, 0x8000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x4b
internal/poll.(*FD).Read(0xc420f7b800, 0xc42309c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x22a
net.(*netFD).Read(0xc420f7b800, 0xc42309c000, 0x8000, 0x8000, 0xc41ff5d4bb, 0xc41ff5d4ba, 0x0)
	/usr/local/go/src/net/fd_unix.go:202 +0x66
net.(*conn).Read(0xc4210d62e8, 0xc42309c000, 0x8000, 0x8000, 0xc4209466e0, 0xc421901618, 0x335a408)
	/usr/local/go/src/net/net.go:176 +0x85
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420ef4140, 0xc42309c000, 0x8000, 0x8000, 0x10, 0xc4219016b0, 0xbdcc7a)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x241
crypto/tls.(*block).readFromUntil(0xc420ab89f0, 0x7f09bf474588, 0xc420ef4140, 0x5, 0xc420ef4140, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc420c53180, 0x335a617, 0xc420c532a0, 0x105802e)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc420c53180, 0xc42330a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc42075daa0, 0xc421498658, 0x9, 0x9, 0x92aade, 0xc421677308, 0x90bd4c)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x379b140, 0xc42075daa0, 0xc421498658, 0x9, 0x9, 0x9, 0x1, 0xc4214569f0, 0x0)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x379b140, 0xc42075daa0, 0xc421498658, 0x9, 0x9, 0x0, 0x0, 0x90bc22)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc421498658, 0x9, 0x9, 0x379b140, 0xc42075daa0, 0x0, 0x0, 0x0, 0x379b5e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc421498620, 0xc420ab9290, 0xc420ab9290, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420f60600, 0xc420b58210, 0x3359790)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:427 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4217a2000, 0x37dcdc0, 0xc420f60600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:676 +0x16f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2()
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:631 +0x54
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4219672c0, 0xc4217a2000, 0x7f09ba6be000, 0xc420f60600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:639 +0x35
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x7c7

goroutine 2486 [IO wait]:
internal/poll.runtime_pollWait(0x7f09bf463950, 0x72, 0x489fd58)
	/usr/local/go/src/runtime/netpoll.go:173 +0x5e
internal/poll.(*pollDesc).wait(0xc422052218, 0x72, 0x37a1500, 0x489fd58, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0xe5
internal/poll.(*pollDesc).waitRead(0xc422052218, 0xc4237c2000, 0x4000, 0x4000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x4b
internal/poll.(*FD).Read(0xc422052200, 0xc4237c2000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x22a
net.(*netFD).Read(0xc422052200, 0xc4237c2000, 0x4000, 0x4000, 0xc42175e820, 0x62, 0x0)
	/usr/local/go/src/net/fd_unix.go:202 +0x66
net.(*conn).Read(0xc4210d60a8, 0xc4237c2000, 0x4000, 0x4000, 0x55, 0x3ff3, 0x335a408)
	/usr/local/go/src/net/net.go:176 +0x85
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421896820, 0xc4237c2000, 0x4000, 0x4000, 0xc4217fa4c0, 0xc4249716b0, 0xbdcc7a)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x241
crypto/tls.(*block).readFromUntil(0xc421632810, 0x7f09bf474588, 0xc421896820, 0x5, 0xc421896820, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x114
crypto/tls.(*Conn).readRecord(0xc42175e700, 0x335a617, 0xc42175e820, 0xc4237c200d)
	/usr/local/go/src/crypto/tls/conn.go:595 +0x134
crypto/tls.(*Conn).Read(0xc42175e700, 0xc4216ec000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x160
bufio.(*Reader).Read(0xc4213b7980, 0xc4203e3c38, 0x9, 0x9, 0x92aade, 0xc420b85d48, 0x90bd4c)
	/usr/local/go/src/bufio/bufio.go:216 +0x740
io.ReadAtLeast(0x379b140, 0xc4213b7980, 0xc4203e3c38, 0x9, 0x9, 0x9, 0xc4204c8800, 0x50, 0x0)
	/usr/local/go/src/io/io.go:309 +0x94
io.ReadFull(0x379b140, 0xc4213b7980, 0xc4203e3c38, 0x9, 0x9, 0x4c, 0x4c, 0x90bc22)
	/usr/local/go/src/io/io.go:327 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4203e3c38, 0x9, 0x9, 0x379b140, 0xc4213b7980, 0x0, 0x0, 0x4c, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x8f
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4203e3c00, 0xc421632cf0, 0xc421632cf0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x10f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420f60200, 0xc4213952f0, 0x3359790)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:427 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42110b6c0, 0x37dcdc0, 0xc420f60200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:676 +0x16f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2()
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:631 +0x54
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4203db800, 0xc42110b6c0, 0x7f09ba6be000, 0xc420f60200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:639 +0x35
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x7c7

goroutine 2438 [select, 2 minutes]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x37c4140, 0xc4206921b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x1aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204105f0, 0xc4201b0fc0, 0xc4204105e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 3579 [runnable]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4214962d0, 0x2, 0xc42232a720, 0xc420f986c0, 0x37d9fa0, 0xc420325c70, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x764
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4214962d0, 0x2, 0x37c4140, 0xc420b783c0, 0x1, 0x0, 0xc421934d20, 0xc42232a720, 0xc420f986c0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x378
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4214962d0, 0x37c4140, 0xc420b783c0, 0x2, 0xc42232a720, 0xc420f986c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0x103
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x37c4140, 0xc420b783c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x7c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420325be0, 0xc420faa360, 0xc420b78390)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:193 +0x14b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:186 +0xbb

goroutine 3482 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420eefc20, 0xc42042d850, 0x5, 0x5, 0xc420447f00, 0x8e79c0, 0x29a1010)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:142 +0x19c
@cockroach-teamcity cockroach-teamcity added this to the 2.1 milestone May 26, 2018
@cockroach-teamcity cockroach-teamcity added C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot. labels May 26, 2018
@tbg tbg added the A-kv-client Relating to the KV client and the KV interface. label May 30, 2018
@tbg tbg assigned windchan7 and unassigned dt May 30, 2018
@windchan7
Copy link
Contributor

#26259

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@windchan7 windchan7

Labels
A-kv-client Relating to the KV client and the KV interface. C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Projects
None yet
Milestone
2.1
Development

No branches or pull requests
4 participants
@dt @tbg @cockroach-teamcity @windchan7