storage: TestRemovePlaceholderRace failed under stress #12574

cockroach-teamcity · 2016-12-24T09:05:16Z

SHA: https://github.com/cockroachdb/cockroach/commits/fd729f0de7d2fb5df7d9b1cca018d1416e263ced

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=true
TAGS=
GOFLAGS=-race

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=99779&tab=buildLog

	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:312 +0x15e
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4208ac160, 0x60d451, 0x3560178, 0x2b6471aea000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:283 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x3099340, 0xc4208ac160, 0xc420e29b00, 0xc42097d7a0, 0x2b6471aea000, 0xc420094428, 0xc421318f48, 0xda280d)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x6d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0x113
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421874ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21018 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42097d7a0, 0x2b6400000003, 0xc4214268a0, 0xc420e29b00, 0x3099340, 0xc4208ac160, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x907
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42097d7a0, 0x3, 0x2b6471aea000, 0xc420094428, 0x1, 0x0, 0xc421ab1680, 0xc4214268a0, 0xc420e29b00, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x439
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42097d7a0, 0x2b6471aea000, 0xc420094428, 0x2b6400000003, 0xc4214268a0, 0xc420e29b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x116
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xfe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421874de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21029 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4205bf0c0, 0xc421a758b0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:140 +0x89c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420ea2960, 0xc421a758b0, 0x5, 0x5, 0x641581, 0xc420215040, 0x4)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:325 +0x79
io.ReadAtLeast(0x307e080, 0xc420ea2960, 0xc421a758b0, 0x5, 0x5, 0x5, 0xa53746, 0xc421789f90, 0xc4200f9bb8)
	/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x307e080, 0xc420ea2960, 0xc421a758b0, 0x5, 0x5, 0x77946c, 0xc420aadce0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc421a758a0, 0x7fffffff, 0x2, 0x8, 0x0, 0xc42166b580, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc421a758a0, 0x308fe00, 0x3f168f0, 0xc420ea2960, 0x0, 0x0, 0x20c17c0, 0xc42166b5a0, 0x7fffffff, 0x20, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x5e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420374bd0, 0x20c17c0, 0xc42166b5a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:516 +0x160
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc420984900, 0x2b6471aeb420, 0xc421966c00, 0xc421a3d680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:351 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x30993a0, 0xc420984900, 0xc42097d7a0, 0x65a4e0, 0xc420c04788)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x61
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x61
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421a29c20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20846 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20730 [IO wait]:
net.runtime_pollWait(0x2b6471ab9808, 0x72, 0x611dc3)
	/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc420e5c370, 0x72, 0x60e678, 0x2006220)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc420e5c370, 0x3082140, 0xc4200940b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).accept(0xc420e5c310, 0x0, 0x307ef80, 0xc421b9a700)
	/usr/local/go/src/net/fd_unix.go:419 +0x2b8
net.(*TCPListener).accept(0xc421264208, 0xc420146170, 0xc4210c8de0, 0x636b3f)
	/usr/local/go/src/net/tcpsock_posix.go:132 +0x51
net.(*TCPListener).Accept(0xc421264208, 0x256e8f8, 0xc420593c80, 0x3099640, 0xc420146170)
	/usr/local/go/src/net/tcpsock.go:222 +0x50
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc420593c80, 0x308ec80, 0xc421264208, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:354 +0x1d1
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:56 +0x54
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421aca960)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20885 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc4000000a1)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9b8e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21031 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42097d7a0, 0x30993a0, 0xc420984b90, 0xc4211b8ba8, 0xc4211b8bb8)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x305
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x2148ba0, 0xc42097d7a0, 0x3096fa0, 0xc420375320, 0x1090768, 0x3090640)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:332 +0xda
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc420593c80, 0x30981a0, 0xc4220de990, 0xc4210e5b30, 0xc421b04780, 0x3065020, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:716 +0xc7d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc420593c80, 0x30981a0, 0xc4220de990, 0xc4210e5b30, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:800 +0x170e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421a5a310, 0xc420593c80, 0x30981a0, 0xc4220de990, 0xc4210e5b30)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:449 +0xb9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:450 +0xbb

goroutine 20958 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc40000016e)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421880360)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20830 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1313 +0x4b6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421b1d7d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20874 [IO wait]:
net.runtime_pollWait(0x2b6472b43940, 0x72, 0x611dc3)
	/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc422223c60, 0x72, 0x60e678, 0x2006220)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc422223c60, 0x3082140, 0xc4200940b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc422223c00, 0xc4215a2000, 0x8000, 0x8000, 0x0, 0x3082140, 0xc4200940b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42125e2b0, 0xc4215a2000, 0x8000, 0x8000, 0x42, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42125e2b0, 0xc4215a2000, 0x8000, 0x8000, 0x0, 0x7fbe, 0x1)
	<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc42065d980)
	/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc42065d980, 0xc421d3c2a8, 0x9, 0x9, 0xc4208ac1a0, 0xc4203b21a0, 0xc4208ed9d0)
	/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x3078d40, 0xc42065d980, 0xc421d3c2a8, 0x9, 0x9, 0x9, 0xc4208ed9b0, 0x63caec, 0xc421874f98)
	/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x3078d40, 0xc42065d980, 0xc421d3c2a8, 0x9, 0x9, 0x65a600, 0xc420e2af40, 0xc4208edaa0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc421d3c2a8, 0x9, 0x9, 0x3078d40, 0xc42065d980, 0x0, 0xc400000000, 0xc4208edad0, 0x11a8d78)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc421d3c270, 0x3084a80, 0xc421e2ea50, 0x0, 0xc42067e2c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0xf2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc421a2e2d0, 0xc421e2ea50, 0xc421e2ea50, 0xc4208edce0, 0x1)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4200abd40, 0xc421a2e360)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4203bfbc0, 0x30981a0, 0xc4200abd40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:451 +0x1d6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc4203bfbc0, 0x3099640, 0xc42125e2b0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:438 +0x4f1
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4203bfbc0, 0x3099640, 0xc42125e2b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:415 +0x5c3
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:384 +0x6b4

goroutine 20862 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:995 +0x69b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421a5bb80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20969 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1313 +0x4b6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421a419b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20814 [semacquire]:
sync.runtime_notifyListWait(0xc42174e850, 0xc4000001fe)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc42174e840)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420e35c30, 0xc420d118c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1aec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20845 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf560)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20871 [IO wait]:
net.runtime_pollWait(0x2b6472b43b80, 0x72, 0x611dc3)
	/usr/local/go/src/runtime/netpoll.go:160 +0x5e
net.(*pollDesc).wait(0xc422223950, 0x72, 0x60e678, 0x2006220)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x5b
net.(*pollDesc).waitRead(0xc422223950, 0x3082140, 0xc4200940b0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x42
net.(*netFD).Read(0xc4222238f0, 0xc421884000, 0x8000, 0x8000, 0x0, 0x3082140, 0xc4200940b0)
	/usr/local/go/src/net/fd_unix.go:243 +0x1d9
net.(*conn).Read(0xc42125e290, 0xc421884000, 0x8000, 0x8000, 0x13c, 0x0, 0x0)
	/usr/local/go/src/net/net.go:173 +0x97
net.(*TCPConn).Read(0xc42125e290, 0xc421884000, 0x8000, 0x8000, 0x0, 0x7ec4, 0x1)
	<autogenerated>:69 +0x73
bufio.(*Reader).fill(0xc42065d620)
	/usr/local/go/src/bufio/bufio.go:97 +0x197
bufio.(*Reader).Read(0xc42065d620, 0xc420175f18, 0x9, 0x9, 0xc400000000, 0xc400000001, 0xc4217b19d0)
	/usr/local/go/src/bufio/bufio.go:209 +0x6ac
io.ReadAtLeast(0x3078d40, 0xc42065d620, 0xc420175f18, 0x9, 0x9, 0x9, 0xc4217b19b0, 0x63caec, 0xc422225c28)
	/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x3078d40, 0xc42065d620, 0xc420175f18, 0x9, 0x9, 0x65a600, 0xc420415340, 0xc4217b1aa0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420175f18, 0x9, 0x9, 0x3078d40, 0xc42065d620, 0x0, 0xc400000000, 0xc4217b1ad0, 0x11a8d78)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420175ee0, 0x3084a80, 0xc421e233e0, 0x0, 0xc4201103c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:469 +0xf2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc421a9fce0, 0xc421e233e0, 0xc421e233e0, 0xc4217b1ce0, 0x1)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4200abc20, 0xc421a9fd70)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0x30a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4209f9680, 0x30981a0, 0xc4200abc20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:451 +0x1d6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc4209f9680, 0x3099640, 0xc42125e290, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:438 +0x4f1
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4209f9680, 0x3099640, 0xc42125e290)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:415 +0x5c3
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:384 +0x6b4

goroutine 20863 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1033 +0x4ce
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421a5bb90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20952 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc400000165)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc4218802a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20926 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4219db980, 0xc4202467e0, 0xc421824bf0, 0xf, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x892
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0xb3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421a418f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20834 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420b1b9a0, 0x2b6471aea000, 0xc420094428, 0xecff03149, 0xc426a43a03, 0x35564c0, 0xc42174e040, 0xc420d118c0, 0xc421aec000, 0x1130)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x2a7
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc421aec000, 0x58)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x162
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc421b055c0, 0xc421d0bb80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:339 +0x5b0
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x45a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421bae780)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20847 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf5c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20888 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc4000000a4)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9b940)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20928 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:995 +0x69b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421824d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20872 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc4200abc20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:674 +0xa56
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:139 +0xa91

goroutine 21030 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42097d7a0, 0xc400000001, 0xc421f0f740, 0xc420ef15c0, 0x3099340, 0xc421723580, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x907
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42097d7a0, 0x1, 0x2b6471aea000, 0xc420094428, 0x0, 0x0, 0xc420375050, 0xc421f0f740, 0xc420ef15c0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x439
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42097d7a0, 0x2b6471aea000, 0xc420094428, 0x2b6400000001, 0xc421f0f740, 0xc420ef15c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x116
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xfe
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc4219641e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20809 [semacquire]:
sync.runtime_notifyListWait(0xc42174e850, 0xc400000201)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc42174e840)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420e35c30, 0xc420d118c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1ae20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20804 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4219db980, 0xc421910a20, 0xc421862950, 0xf, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x892
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0xb3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421b1cf30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20806 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1004 +0x620
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421c0eb80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20725 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4217887c0, 0xc42169a750, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:140 +0x89c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc4200f22d0, 0xc42169a750, 0x5, 0x5, 0xc421b36aa0, 0x0, 0xc4210b4450)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:325 +0x79
io.ReadAtLeast(0x307e080, 0xc4200f22d0, 0xc42169a750, 0x5, 0x5, 0x5, 0x1f66640, 0xc4210b43f0, 0xc421b36bd0)
	/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x307e080, 0xc4200f22d0, 0xc42169a750, 0x5, 0x5, 0x133, 0x0, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42169a740, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0xc42105a280, 0xc4211cdba0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42169a740, 0x308fe00, 0x3f168f0, 0xc4200f22d0, 0x0, 0x0, 0x20c1c20, 0xc42169a780, 0x7fffffff, 0x20, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x5e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc421d44120, 0x20c1c20, 0xc42169a780, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:312 +0x15e
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftSnapshotClient).Recv(0xc421cee710, 0xc421b36ec0, 0x6f06db, 0xc421cee710)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:314 +0x86
github.com/cockroachdb/cockroach/pkg/storage.snapshotClientWithBreaker.Recv(0x3099400, 0xc421cee710, 0xc42222cea0, 0xc42222cea0, 0xc421788800, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:612 +0x43
github.com/cockroachdb/cockroach/pkg/storage.(*snapshotClientWithBreaker).Recv(0xc42169a760, 0xc421788800, 0x0, 0x0)
	<autogenerated>:662 +0x8d
github.com/cockroachdb/cockroach/pkg/storage.sendSnapshot(0x2b6471aeb420, 0xc421e2e150, 0x3086180, 0xc42169a760, 0x307ccc0, 0xc4219dba80, 0x63, 0x57, 0xc42097c4e0, 0xc42159a460, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3179 +0x198
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendSnapshot(0xc42097d7a0, 0x2b6471aeb420, 0xc421e2e150, 0xc4219dba80, 0x63, 0x57, 0xc42097c4e0, 0xc42159a460, 0xc421cee550, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:656 +0x399
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).ChangeReplicas.func1(0xc421aec000, 0x2b6471aeb420, 0xc421e2e150, 0xc421cee2a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:3226 +0x8a3
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).ChangeReplicas(0xc421aec000, 0x2b6471aeb420, 0xc421e2e150, 0x200000000, 0x2, 0xc421ab8d20, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:3232 +0x5f4
github.com/cockroachdb/cockroach/pkg/storage_test.TestRemovePlaceholderRace(0xc420593b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_raft_test.go:1966 +0x487
testing.tRunner(0xc420593b00, 0x256cec8)
	/usr/local/go/src/testing/testing.go:610 +0xca
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:646 +0x530

goroutine 20951 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc400000173)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421880280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21021 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42097d7a0, 0x30993a0, 0xc4208ac1d0, 0xc42130dba8, 0xc42130dbb8)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x305
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x2148ba0, 0xc42097d7a0, 0x3096fa0, 0xc421ab17a0, 0x0, 0x1ab60f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:332 +0xda
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4203bfbc0, 0x30981a0, 0xc4200abd40, 0xc42203a780, 0xc421a40600, 0x3065020, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:716 +0xc7d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4203bfbc0, 0x30981a0, 0xc4200abd40, 0xc42203a780, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:800 +0x170e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421afdaf0, 0xc4203bfbc0, 0x30981a0, 0xc4200abd40, 0xc42203a780)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:449 +0xb9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:450 +0xbb

goroutine 20859 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4211662c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x7d7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:494 +0x9c5

goroutine 20811 [semacquire]:
sync.runtime_notifyListWait(0xc42174e850, 0xc4000001ff)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc42174e840)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420e35c30, 0xc420d118c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1ae60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20918 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc42047d540, 0x2b6471aea000, 0xc420094428)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:209 +0x1011
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:193 +0xc0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421c0fdf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21486 [runnable]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420d117a0, 0x2d62af6, 0x13, 0x12d, 0x0, 0x0, 0xc42219aa40, 0x2b6471aea000, 0xc420094428)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:260
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x2b0

goroutine 20838 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4219b9a40, 0xc421baeb70, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:140 +0x89c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421883a40, 0xc421baeb70, 0x5, 0x5, 0x0, 0x0, 0xc421b9a720)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:325 +0x79
io.ReadAtLeast(0x307e080, 0xc421883a40, 0xc421baeb70, 0x5, 0x5, 0x5, 0xc421b93230, 0xc4220de9b8, 0xc4220de9c8)
	/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x307e080, 0xc421883a40, 0xc421baeb70, 0x5, 0x5, 0x8, 0x24, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc421baeb60, 0x7fffffff, 0x7, 0x0, 0x6049d5, 0xc421aa0b38, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc421baeb60, 0x308fe00, 0x3f168f0, 0xc421883a40, 0x0, 0x0, 0x20cd840, 0xc4217c5200, 0x7fffffff, 0x40, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x5e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420e39c20, 0x20cd840, 0xc4217c5200, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:516 +0x160
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc421aa85b0, 0xc4ffffffff, 0x0, 0x180002)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:228 +0x86
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc420593d70, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x4b
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc420593d40, 0x2b6471aeb3a8, 0xc4219b9b00, 0xc4201582e0, 0xc4219b9b40, 0xc420ba9f40, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:321 +0x641
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x14e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421c41720)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20815 [semacquire]:
sync.runtime_notifyListWait(0xc42174e850, 0xc400000203)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc42174e840)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420e35c30, 0xc420d118c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1aee0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20887 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc4000000a2)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9b920)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20886 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc4000000a0)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9b900)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20949 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc400000170)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421880240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20729 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:49 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421b059e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21028 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420ef0a40, 0xc421a75850, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:140 +0x89c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc4210e5950, 0xc421a75850, 0x5, 0x5, 0x0, 0x8, 0x149325d8b45006a2)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:325 +0x79
io.ReadAtLeast(0x307e080, 0xc4210e5950, 0xc421a75850, 0x5, 0x5, 0x5, 0x611df5, 0xc420094370, 0x10)
	/usr/local/go/src/io/io.go:307 +0xb2
io.ReadFull(0x307e080, 0xc4210e5950, 0xc421a75850, 0x5, 0x5, 0xc421b53bb8, 0xd0d8d6, 0x3557a20)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc421a75840, 0x7fffffff, 0x1f729c0, 0xc420094330, 0x1f733c0, 0xc420094370, 0xc421b53c10, 0xd0242f)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc421a75840, 0x308fe00, 0x3f168f0, 0xc4210e5950, 0x0, 0x0, 0x20c18a0, 0xc420012540, 0x7fffffff, 0x30, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x5e
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc421d446c0, 0x20c18a0, 0xc420012540, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:312 +0x15e
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4209848e0, 0x2b6471aeb420, 0xc4200123f0, 0xc422197590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:283 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x3099340, 0xc4209848e0, 0xc420f29b00, 0xc42097d7a0, 0x2b6471aea000, 0xc420094428, 0xc420d5d748, 0xda280d)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x6d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0x113
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421a29bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21027 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func2(0x3098140, 0xc4211cdb00, 0xc4210e5950, 0xc421d446c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:204 +0x54e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:224 +0x10a5

goroutine 20799 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc4218082c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20798 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc4218082a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20800 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc4218082e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20801 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421808300)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20884 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc4000000a6)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9b8c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20829 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1227 +0x19d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1b0a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20883 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc40000009e)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9b8a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20968 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1313 +0x4b6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421a41980)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20896 [semacquire]:
sync.runtime_notifyListWait(0xc421789f90, 0xc40000009d)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc421789f80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420aadce0, 0xc4220defc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421b9ba40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20925 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42098e000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:751 +0x7d7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:494 +0x9c5

goroutine 20897 [semacquire]:
sync.runtime_Semacquire(0xc420aadd84)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc420aadd78)
	/usr/local/go/src/sync/waitgroup.go:131 +0xbf
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420aadce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x42
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3406 +0x67
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421a5bc20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20931 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc42021dae0, 0x2b6471aea000, 0xc420094428, 0xecff03149, 0xc4293b4c77, 0x35564c0, 0xc42174e040, 0xc4220defc0, 0x0, 0xc420a4f748)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x2a7
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x637
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf660)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20930 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf640)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20805 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x94
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421d1a800)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20824 [semacquire]:
sync.runtime_notifyListWait(0xc42174e850, 0xc4000001f8)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc42174e840)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420e35c30, 0xc420d118c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1b000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20861 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x94
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d117a0, 0xc421b9b7e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20924 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420246900)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1015 +0x8c9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:235 +0x1500

goroutine 20825 [semacquire]:
sync.runtime_Semacquire(0xc420e35cd4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc420e35cc8)
	/usr/local/go/src/sync/waitgroup.go:131 +0xbf
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420e35c30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x42
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3406 +0x67
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421c0f2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20833 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1227 +0x19d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421d1b1e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20947 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc400000166)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421880200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20955 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc400000168)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421880300)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20849 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf620)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20917 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc421b4af00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:375 +0x64
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:376 +0x8fb

goroutine 20717 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0x495
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421bae6c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20959 [semacquire]:
sync.runtime_notifyListWait(0xc4206d3a90, 0xc40000016a)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc4206d3a80)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420ec96b0, 0xc421ab0990)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421ab0990, 0xc421880380)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 21503 [semacquire]:
sync.runtime_Semacquire(0xc4203bfcb4)
	/usr/local/go/src/runtime/sema.go:47 +0x30
sync.(*Mutex).Lock(0xc4203bfcb0)
	/usr/local/go/src/sync/mutex.go:85 +0xde
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:369 +0x56
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func2(0xc42188e440, 0x8, 0xc4216ec700, 0x15, 0x20, 0x149325d8d134fdfa, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:377 +0x3e
github.com/cockroachdb/cockroach/pkg/gossip.(*infoStore).runCallbacks.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:277 +0x12a
github.com/cockroachdb/cockroach/pkg/gossip.(*infoStore).runCallbacks.func2(0x2b6471aea000, 0xc420094428)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:299 +0x14a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420d117a0, 0x2d62af6, 0x13, 0x12d, 0x0, 0x0, 0xc42188e470, 0x2b6471aea000, 0xc420094428)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xed
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x2b0

goroutine 20898 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3432 +0x482
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421a5bc30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20820 [semacquire]:
sync.runtime_notifyListWait(0xc42174e850, 0xc4000001fb)
	/usr/local/go/src/runtime/sema.go:267 +0x12f
sync.(*Cond).Wait(0xc42174e840)
	/usr/local/go/src/sync/cond.go:57 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420e35c30, 0xc420d118c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xfd
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420d118c0, 0xc421d1af80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

goroutine 20899 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3463 +0x1c5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421a5bc40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74


ERROR: exit status 255

make: *** [stress] Error 1
57 runs completed, 1 failures, over 12m40s
Makefile:147: recipe for target 'stress' failed

The text was updated successfully, but these errors were encountered:

a-robinson · 2016-12-27T22:47:10Z

The actual failure:

[04:05:15][Step 2/2] === RUN   TestRemovePlaceholderRace
[04:05:15][Step 2/2] I161224 09:05:13.648828 20725 storage/store.go:1258  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
[04:05:15][Step 2/2] I161224 09:05:13.649047 20725 gossip/gossip.go:292  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:39332" > attrs:<> locality:<> 
[04:05:15][Step 2/2] W161224 09:05:13.676857 20725 gossip/gossip.go:1130  [n?] no incoming or outgoing connections
[04:05:15][Step 2/2] I161224 09:05:13.685053 20725 storage/store.go:1258  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
[04:05:15][Step 2/2] I161224 09:05:13.685268 20725 gossip/gossip.go:292  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:56089" > attrs:<> locality:<> 
[04:05:15][Step 2/2] I161224 09:05:13.687539 20835 gossip/client.go:125  [n2] started gossip client to 127.0.0.1:39332
[04:05:15][Step 2/2] W161224 09:05:13.772258 20725 gossip/gossip.go:1130  [n?] no incoming or outgoing connections
[04:05:15][Step 2/2] I161224 09:05:13.777673 20795 gossip/client.go:125  [n3] started gossip client to 127.0.0.1:39332
[04:05:15][Step 2/2] I161224 09:05:13.784229 20725 gossip/gossip.go:292  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:35421" > attrs:<> locality:<> 
[04:05:15][Step 2/2] I161224 09:05:13.797405 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot dcccfbf2 at index 15
[04:05:15][Step 2/2] I161224 09:05:13.810435 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 34, log entries: 5
[04:05:15][Step 2/2] I161224 09:05:13.811526 20996 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c421ae7680] applying preemptive snapshot at index 15 (id=dcccfbf2, encoded size=6645, 1 rocksdb batches, 5 log entries)
[04:05:15][Step 2/2] I161224 09:05:13.812385 20996 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c421ae7680] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:13.814812 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2 
[04:05:15][Step 2/2] I161224 09:05:13.820141 21011 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:2}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
[04:05:15][Step 2/2] I161224 09:05:13.827089 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 9e55d821 at index 17
[04:05:15][Step 2/2] I161224 09:05:13.828911 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 37, log entries: 7
[04:05:15][Step 2/2] I161224 09:05:13.829929 21001 storage/replica_raftstorage.go:633  [s3,r1/?:{-},@c4205ba000] applying preemptive snapshot at index 17 (id=9e55d821, encoded size=8827, 1 rocksdb batches, 7 log entries)
[04:05:15][Step 2/2] I161224 09:05:13.831096 21001 storage/replica_raftstorage.go:641  [s3,r1/?:/M{in-ax},@c4205ba000] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:13.834586 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 
[04:05:15][Step 2/2] I161224 09:05:13.841906 21030 storage/raft_transport.go:437  raft transport stream to node 1 established
[04:05:15][Step 2/2] I161224 09:05:13.848514 21006 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:3}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.133329 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:4 
[04:05:15][Step 2/2] I161224 09:05:14.140497 20904 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:2}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.152414 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 33898cbc at index 21
[04:05:15][Step 2/2] I161224 09:05:14.156268 20848 storage/store.go:2087  [replicaGC,s2,r1/2:/M{in-ax},@c421ae7680] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.163664 21004 storage/store.go:3136  [s2,r1/2:/M{in-ax},@c421ae7680] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.166193 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 43, log entries: 11
[04:05:15][Step 2/2] I161224 09:05:14.168821 20989 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c4216c4900] applying preemptive snapshot at index 21 (id=33898cbc, encoded size=13333, 1 rocksdb batches, 11 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.170145 20989 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c4216c4900] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.176343 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:4 
[04:05:15][Step 2/2] I161224 09:05:14.188683 20991 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:4}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:4}]
[04:05:15][Step 2/2] I161224 09:05:14.195873 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:4 > next_replica_id:5 
[04:05:15][Step 2/2] W161224 09:05:14.196245 21028 storage/store.go:3144  [s1] got error from range 1, replica {2 2 2}: raft group deleted
[04:05:15][Step 2/2] I161224 09:05:14.206133 21024 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:4}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.214288 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 40242561 at index 25
[04:05:15][Step 2/2] I161224 09:05:14.216390 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 49, log entries: 15
[04:05:15][Step 2/2] I161224 09:05:14.219243 21004 storage/store.go:3136  [s2,r1/4:/M{in-ax},@c4216c4900] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.219690 20848 storage/store.go:2087  [replicaGC,s2,r1/4:/M{in-ax},@c4216c4900] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.220061 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:5 
[04:05:15][Step 2/2] I161224 09:05:14.226298 21009 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:5}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:5}]
[04:05:15][Step 2/2] I161224 09:05:14.237943 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:5 > next_replica_id:6 
[04:05:15][Step 2/2] I161224 09:05:14.243437 20821 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated Raft snapshot c7d30a97 at index 28
[04:05:15][Step 2/2] I161224 09:05:14.246135 21037 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 55, log entries: 18
[04:05:15][Step 2/2] I161224 09:05:14.246960 21054 storage/replica_raftstorage.go:633  [s2,r1/5:{-},@c421f68480] applying Raft snapshot at index 28 (id=c7d30a97, encoded size=21399, 1 rocksdb batches, 18 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.249005 21054 storage/replica_raftstorage.go:641  [s2,r1/5:/M{in-ax},@c421f68480] applied Raft snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.249876 21040 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:5}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.257803 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 87a408b6 at index 29
[04:05:15][Step 2/2] I161224 09:05:14.259583 21004 storage/store.go:3136  [s2,r1/5:/M{in-ax},@c421f68480] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.260205 20848 storage/store.go:2087  [replicaGC,s2,r1/5:/M{in-ax},@c421f68480] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.260294 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 55, log entries: 19
[04:05:15][Step 2/2] I161224 09:05:14.262066 21092 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c420ae8d80] applying preemptive snapshot at index 29 (id=87a408b6, encoded size=22389, 1 rocksdb batches, 19 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.263707 21092 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c420ae8d80] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.266338 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:6 
[04:05:15][Step 2/2] I161224 09:05:14.274239 21094 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:6}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:6}]
[04:05:15][Step 2/2] I161224 09:05:14.281211 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:6 > next_replica_id:7 
[04:05:15][Step 2/2] I161224 09:05:14.292274 20737 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:6}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] W161224 09:05:14.296762 21028 storage/store.go:3144  [s1] got error from range 1, replica {2 2 5}: raft group deleted
[04:05:15][Step 2/2] I161224 09:05:14.305333 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot cbb4a149 at index 33
[04:05:15][Step 2/2] I161224 09:05:14.314413 20848 storage/store.go:2087  [replicaGC,s2,r1/6:/M{in-ax},@c420ae8d80] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.314576 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 61, log entries: 23
[04:05:15][Step 2/2] I161224 09:05:14.316959 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:7 
[04:05:15][Step 2/2] I161224 09:05:14.334377 21126 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:7}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:7}]
[04:05:15][Step 2/2] I161224 09:05:14.342892 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:7 > next_replica_id:8 
[04:05:15][Step 2/2] I161224 09:05:14.352675 20810 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated Raft snapshot 6a6e7c65 at index 35
[04:05:15][Step 2/2] I161224 09:05:14.377999 21143 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 64, log entries: 25
[04:05:15][Step 2/2] I161224 09:05:14.378100 21148 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:7}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.381692 21146 storage/replica_raftstorage.go:633  [s2,r1/7:{-},@c42034e900] applying Raft snapshot at index 35 (id=6a6e7c65, encoded size=29218, 1 rocksdb batches, 25 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.400616 21146 storage/replica_raftstorage.go:641  [s2,r1/7:/M{in-ax},@c42034e900] applied Raft snapshot in 19ms [clear=0ms batch=0ms entries=18ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.418436 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 37e284ed at index 38
[04:05:15][Step 2/2] I161224 09:05:14.420305 21004 storage/store.go:3136  [s2,r1/7:/M{in-ax},@c42034e900] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.421137 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 68, log entries: 28
[04:05:15][Step 2/2] I161224 09:05:14.422689 21154 storage/replica_raftstorage.go:633  [s2,r1/7:/M{in-ax},@c42034e900] applying Raft snapshot at index 38 (id=37e284ed, encoded size=32303, 1 rocksdb batches, 28 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.428910 21154 storage/replica_raftstorage.go:641  [s2,r1/7:/M{in-ax},@c42034e900] applied Raft snapshot in 5ms [clear=0ms batch=0ms entries=1ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:14.429143 20848 storage/store.go:2087  [replicaGC,s2,r1/7:/M{in-ax},@c42034e900] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.444504 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:8 
[04:05:15][Step 2/2] I161224 09:05:14.480588 21100 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:8}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:8}]
[04:05:15][Step 2/2] I161224 09:05:14.492013 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:8 > next_replica_id:9 
[04:05:15][Step 2/2] I161224 09:05:14.511340 20824 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated Raft snapshot 4a549b0f at index 42
[04:05:15][Step 2/2] I161224 09:05:14.513580 21153 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:8}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.515203 21158 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 75, log entries: 32
[04:05:15][Step 2/2] I161224 09:05:14.522780 21188 storage/replica_raftstorage.go:633  [s2,r1/8:{-},@c4216c5200] applying Raft snapshot at index 42 (id=4a549b0f, encoded size=36706, 1 rocksdb batches, 32 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.526519 21188 storage/replica_raftstorage.go:641  [s2,r1/8:/M{in-ax},@c4216c5200] applied Raft snapshot in 4ms [clear=0ms batch=0ms entries=2ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.528279 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot dd64d248 at index 43
[04:05:15][Step 2/2] I161224 09:05:14.531082 21004 storage/store.go:3136  [s2,r1/8:/M{in-ax},@c4216c5200] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.543323 20848 storage/store.go:2087  [replicaGC,s2,r1/8:/M{in-ax},@c4216c5200] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.544945 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 75, log entries: 33
[04:05:15][Step 2/2] I161224 09:05:14.546937 21210 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c42203c000] applying preemptive snapshot at index 43 (id=dd64d248, encoded size=37696, 1 rocksdb batches, 33 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.549411 21210 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c42203c000] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.553637 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:9 
[04:05:15][Step 2/2] I161224 09:05:14.562066 21213 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:9}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:9}]
[04:05:15][Step 2/2] I161224 09:05:14.570502 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:9 > next_replica_id:10 
[04:05:15][Step 2/2] I161224 09:05:14.581133 21194 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:9}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.591303 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot d93d45cc at index 48
[04:05:15][Step 2/2] I161224 09:05:14.593911 21004 storage/store.go:3136  [s2,r1/9:/M{in-ax},@c42203c000] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.595401 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 82, log entries: 38
[04:05:15][Step 2/2] I161224 09:05:14.597286 20848 storage/store.go:2087  [replicaGC,s2,r1/9:/M{in-ax},@c42203c000] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.598914 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:10 
[04:05:15][Step 2/2] I161224 09:05:14.609446 21223 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:10}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:10}]
[04:05:15][Step 2/2] I161224 09:05:14.616403 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:10 > next_replica_id:11 
[04:05:15][Step 2/2] I161224 09:05:14.629150 20824 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated Raft snapshot 036989a6 at index 50
[04:05:15][Step 2/2] I161224 09:05:14.635305 21228 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:10}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.648907 21226 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 85, log entries: 40
[04:05:15][Step 2/2] I161224 09:05:14.649874 21266 storage/replica_raftstorage.go:633  [s2,r1/10:{-},@c4219de480] applying Raft snapshot at index 50 (id=036989a6, encoded size=45397, 1 rocksdb batches, 40 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.652836 21266 storage/replica_raftstorage.go:641  [s2,r1/10:/M{in-ax},@c4219de480] applied Raft snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
[04:05:15][Step 2/2] I161224 09:05:14.658669 21004 storage/store.go:3136  [s2,r1/10:/M{in-ax},@c4219de480] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.659003 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot f7477ba9 at index 52
[04:05:15][Step 2/2] I161224 09:05:14.661450 20848 storage/store.go:2087  [replicaGC,s2,r1/10:/M{in-ax},@c4219de480] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.669530 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 88, log entries: 42
[04:05:15][Step 2/2] I161224 09:05:14.679105 21283 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c421bcb680] applying preemptive snapshot at index 52 (id=f7477ba9, encoded size=47617, 1 rocksdb batches, 42 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.708461 21283 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c421bcb680] applied preemptive snapshot in 29ms [clear=0ms batch=0ms entries=28ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:14.714049 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:11 
[04:05:15][Step 2/2] I161224 09:05:14.720546 21068 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:11}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:11}]
[04:05:15][Step 2/2] I161224 09:05:14.735853 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:11 > next_replica_id:12 
[04:05:15][Step 2/2] I161224 09:05:14.745562 21293 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:11}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.751167 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 824eee31 at index 56
[04:05:15][Step 2/2] I161224 09:05:14.756683 21004 storage/store.go:3136  [s2,r1/11:/M{in-ax},@c421bcb680] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.758236 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 94, log entries: 46
[04:05:15][Step 2/2] I161224 09:05:14.762108 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:12 
[04:05:15][Step 2/2] I161224 09:05:14.771302 20848 storage/store.go:2087  [replicaGC,s2,r1/11:/M{in-ax},@c421bcb680] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.773376 21306 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:12}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:12}]
[04:05:15][Step 2/2] I161224 09:05:14.784054 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:12 > next_replica_id:13 
[04:05:15][Step 2/2] I161224 09:05:14.790708 20824 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated Raft snapshot 5ad2b9a1 at index 59
[04:05:15][Step 2/2] I161224 09:05:14.792369 21200 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:12}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.818354 21243 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 100, log entries: 49
[04:05:15][Step 2/2] I161224 09:05:14.822335 21166 storage/replica_raftstorage.go:633  [s2,r1/12:{-},@c420cb7200] applying Raft snapshot at index 59 (id=5ad2b9a1, encoded size=55683, 1 rocksdb batches, 49 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.832232 21166 storage/replica_raftstorage.go:641  [s2,r1/12:/M{in-ax},@c420cb7200] applied Raft snapshot in 10ms [clear=4ms batch=0ms entries=4ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:14.834213 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot f90e5509 at index 60
[04:05:15][Step 2/2] I161224 09:05:14.848665 21004 storage/store.go:3136  [s2,r1/12:/M{in-ax},@c420cb7200] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.851633 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 100, log entries: 50
[04:05:15][Step 2/2] I161224 09:05:14.852811 20848 storage/store.go:2087  [replicaGC,s2,r1/12:/M{in-ax},@c420cb7200] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.856934 21168 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c420540480] applying preemptive snapshot at index 60 (id=f90e5509, encoded size=56673, 1 rocksdb batches, 50 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.860247 21168 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c420540480] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:14.862817 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:13 
[04:05:15][Step 2/2] I161224 09:05:14.879369 21314 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:13}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:13}]
[04:05:15][Step 2/2] I161224 09:05:14.888322 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:13 > next_replica_id:14 
[04:05:15][Step 2/2] I161224 09:05:14.895505 21334 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:13}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.908320 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 940949fb at index 64
[04:05:15][Step 2/2] I161224 09:05:14.909089 21004 storage/store.go:3136  [s2,r1/13:/M{in-ax},@c420540480] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.909777 20848 storage/store.go:2087  [replicaGC,s2,r1/13:/M{in-ax},@c420540480] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.913210 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 106, log entries: 54
[04:05:15][Step 2/2] I161224 09:05:14.914936 21366 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c421d35680] applying preemptive snapshot at index 64 (id=940949fb, encoded size=61202, 1 rocksdb batches, 54 log entries)
[04:05:15][Step 2/2] I161224 09:05:14.918640 21366 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c421d35680] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=2ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:14.921299 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:14 
[04:05:15][Step 2/2] I161224 09:05:14.927546 21274 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:14}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:14}]
[04:05:15][Step 2/2] I161224 09:05:14.936607 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:14 > next_replica_id:15 
[04:05:15][Step 2/2] I161224 09:05:14.946398 21257 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:14}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:14.952255 21004 storage/store.go:3136  [s2,r1/14:/M{in-ax},@c421d35680] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:14.953366 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot c2958c5d at index 68
[04:05:15][Step 2/2] I161224 09:05:14.975310 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 112, log entries: 58
[04:05:15][Step 2/2] I161224 09:05:14.979888 20848 storage/store.go:2087  [replicaGC,s2,r1/14:/M{in-ax},@c421d35680] removing replica
[04:05:15][Step 2/2] I161224 09:05:14.983588 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:15 
[04:05:15][Step 2/2] I161224 09:05:14.996821 21356 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:15}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:15}]
[04:05:15][Step 2/2] W161224 09:05:15.003332 21028 storage/store.go:3144  [s1] got error from range 1, replica {2 2 14}: raft group deleted
[04:05:15][Step 2/2] I161224 09:05:15.011930 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:15 > next_replica_id:16 
[04:05:15][Step 2/2] I161224 09:05:15.019539 20810 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated Raft snapshot 205d6aa0 at index 71
[04:05:15][Step 2/2] I161224 09:05:15.022348 21318 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:15}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:15.031400 21369 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 118, log entries: 61
[04:05:15][Step 2/2] I161224 09:05:15.034630 21359 storage/replica_raftstorage.go:633  [s2,r1/15:{-},@c420d05200] applying Raft snapshot at index 71 (id=205d6aa0, encoded size=69262, 1 rocksdb batches, 61 log entries)
[04:05:15][Step 2/2] I161224 09:05:15.038831 21359 storage/replica_raftstorage.go:641  [s2,r1/15:/M{in-ax},@c420d05200] applied Raft snapshot in 4ms [clear=1ms batch=0ms entries=2ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:15.040329 21004 storage/store.go:3136  [s2,r1/15:/M{in-ax},@c420d05200] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.041005 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 9ebbf311 at index 72
[04:05:15][Step 2/2] I161224 09:05:15.043199 20848 storage/store.go:2087  [replicaGC,s2,r1/15:/M{in-ax},@c420d05200] removing replica
[04:05:15][Step 2/2] I161224 09:05:15.047631 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 118, log entries: 62
[04:05:15][Step 2/2] I161224 09:05:15.050197 21401 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c420a0a900] applying preemptive snapshot at index 72 (id=9ebbf311, encoded size=70252, 1 rocksdb batches, 62 log entries)
[04:05:15][Step 2/2] I161224 09:05:15.054746 21401 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c420a0a900] applied preemptive snapshot in 4ms [clear=1ms batch=0ms entries=2ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:15.057925 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:16 
[04:05:15][Step 2/2] I161224 09:05:15.082528 21389 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:16}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:16}]
[04:05:15][Step 2/2] I161224 09:05:15.089704 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:16 > next_replica_id:17 
[04:05:15][Step 2/2] I161224 09:05:15.109405 21328 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:16}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:15.134949 21004 storage/store.go:3136  [s2,r1/16:/M{in-ax},@c420a0a900] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.136852 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 2bb664a9 at index 77
[04:05:15][Step 2/2] I161224 09:05:15.138122 20848 storage/store.go:2087  [replicaGC,s2,r1/16:/M{in-ax},@c420a0a900] removing replica
[04:05:15][Step 2/2] I161224 09:05:15.155482 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 125, log entries: 67
[04:05:15][Step 2/2] I161224 09:05:15.157550 21406 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c42160fb00] applying preemptive snapshot at index 77 (id=2bb664a9, encoded size=75647, 1 rocksdb batches, 67 log entries)
[04:05:15][Step 2/2] I161224 09:05:15.166015 21406 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c42160fb00] applied preemptive snapshot in 8ms [clear=1ms batch=0ms entries=6ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:15.169109 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:17 
[04:05:15][Step 2/2] I161224 09:05:15.177661 21461 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:17}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:17}]
[04:05:15][Step 2/2] I161224 09:05:15.185348 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:17 > next_replica_id:18 
[04:05:15][Step 2/2] I161224 09:05:15.219784 21391 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:17}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:15.233418 21004 storage/store.go:3136  [s2,r1/17:/M{in-ax},@c42160fb00] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.234956 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 24abb2e7 at index 82
[04:05:15][Step 2/2] I161224 09:05:15.239331 20848 storage/store.go:2087  [replicaGC,s2,r1/17:/M{in-ax},@c42160fb00] removing replica
[04:05:15][Step 2/2] I161224 09:05:15.256003 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 132, log entries: 72
[04:05:15][Step 2/2] I161224 09:05:15.267902 21466 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c421b03200] applying preemptive snapshot at index 82 (id=24abb2e7, encoded size=81041, 1 rocksdb batches, 72 log entries)
[04:05:15][Step 2/2] I161224 09:05:15.274629 21466 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c421b03200] applied preemptive snapshot in 7ms [clear=1ms batch=0ms entries=5ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:15.277495 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:18 
[04:05:15][Step 2/2] I161224 09:05:15.286293 21480 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:18}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:18}]
[04:05:15][Step 2/2] I161224 09:05:15.293284 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:18 > next_replica_id:19 
[04:05:15][Step 2/2] I161224 09:05:15.305657 21374 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:18}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:15.311531 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 929fed8c at index 87
[04:05:15][Step 2/2] I161224 09:05:15.312909 21004 storage/store.go:3136  [s2,r1/18:/M{in-ax},@c421b03200] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.314447 20848 storage/store.go:2087  [replicaGC,s2,r1/18:/M{in-ax},@c421b03200] removing replica
[04:05:15][Step 2/2] I161224 09:05:15.317594 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 139, log entries: 77
[04:05:15][Step 2/2] I161224 09:05:15.319570 21496 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c420412480] applying preemptive snapshot at index 87 (id=929fed8c, encoded size=86435, 1 rocksdb batches, 77 log entries)
[04:05:15][Step 2/2] I161224 09:05:15.324425 21496 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c420412480] applied preemptive snapshot in 5ms [clear=1ms batch=0ms entries=3ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:15.327100 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:19 
[04:05:15][Step 2/2] I161224 09:05:15.343907 21470 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:19}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:19}]
[04:05:15][Step 2/2] I161224 09:05:15.353611 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:19 > next_replica_id:20 
[04:05:15][Step 2/2] I161224 09:05:15.360085 21508 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:19}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:15.364952 21004 storage/store.go:3136  [s2,r1/19:/M{in-ax},@c420412480] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.366086 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 31bec9ca at index 91
[04:05:15][Step 2/2] I161224 09:05:15.369947 20848 storage/store.go:2087  [replicaGC,s2,r1/19:/M{in-ax},@c420412480] removing replica
[04:05:15][Step 2/2] I161224 09:05:15.377077 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 145, log entries: 81
[04:05:15][Step 2/2] I161224 09:05:15.379494 21424 storage/replica_raftstorage.go:633  [s2,r1/?:{-},@c420ff9b00] applying preemptive snapshot at index 91 (id=31bec9ca, encoded size=90963, 1 rocksdb batches, 81 log entries)
[04:05:15][Step 2/2] I161224 09:05:15.385450 21424 storage/replica_raftstorage.go:641  [s2,r1/?:/M{in-ax},@c420ff9b00] applied preemptive snapshot in 6ms [clear=1ms batch=0ms entries=4ms commit=1ms]
[04:05:15][Step 2/2] I161224 09:05:15.388260 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:20 
[04:05:15][Step 2/2] I161224 09:05:15.399864 21501 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:20}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:20}]
[04:05:15][Step 2/2] I161224 09:05:15.411198 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:20 > next_replica_id:21 
[04:05:15][Step 2/2] I161224 09:05:15.419454 21532 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:20}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] I161224 09:05:15.425941 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot 29f3090e at index 95
[04:05:15][Step 2/2] I161224 09:05:15.432790 20725 storage/store.go:3278  [s1,r1/1:/M{in-ax},@c421aec000] streamed snapshot: kv pairs: 151, log entries: 85
[04:05:15][Step 2/2] I161224 09:05:15.436248 21004 storage/store.go:3136  [s2,r1/20:/M{in-ax},@c420ff9b00] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.436493 20848 storage/store.go:2087  [replicaGC,s2,r1/20:/M{in-ax},@c420ff9b00] removing replica
[04:05:15][Step 2/2] I161224 09:05:15.441514 21004 storage/store.go:3136  [s2,r1/20:/M{in-ax},@c420ff9b00] added to replica GC queue (peer suggestion)
[04:05:15][Step 2/2] I161224 09:05:15.445124 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:21 
[04:05:15][Step 2/2] I161224 09:05:15.466258 21517 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:21}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3} {NodeID:2 StoreID:2 ReplicaID:21}]
[04:05:15][Step 2/2] I161224 09:05:15.477909 20725 storage/replica_command.go:3263  [s1,r1/1:/M{in-ax},@c421aec000] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:3 store_id:3 replica_id:3 > replicas:<node_id:2 store_id:2 replica_id:21 > next_replica_id:22 
[04:05:15][Step 2/2] I161224 09:05:15.484918 21521 storage/replica.go:2364  [s1,r1/1:/M{in-ax},@c421aec000] proposing REMOVE_REPLICA {NodeID:2 StoreID:2 ReplicaID:21}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:3 StoreID:3 ReplicaID:3}]
[04:05:15][Step 2/2] E161224 09:05:15.501722 21004 storage/store.go:3134  [s2,r1/21:{-},@c420d05680] unable to add to replica GC queue: replica not initialized
[04:05:15][Step 2/2] I161224 09:05:15.502556 20725 storage/replica_raftstorage.go:468  [s1,r1/1:/M{in-ax},@c421aec000] generated preemptive snapshot abcb828f at index 99
[04:05:15][Step 2/2] I161224 09:05:15.510358 20848 storage/store.go:2087  [replicaGC,s2,r1/21:{-},@c420d05680] removing replica
[04:05:15][Step 2/2] F161224 09:05:15.510555 20848 storage/store.go:2105  [replicaGC,s2,r1/21:{-},@c420d05680] unexpectedly overlapped by <nil>
[04:05:15][Step 2/2] goroutine 20848 [running]:
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x3f16601, 0xc41e6e7520, 0x35564c0, 0xc4205bec00)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:849 +0xc2
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x3557a20, 0xc400000004, 0x2d66f73, 0x10, 0x839, 0xc4221554a0, 0x45)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:714 +0x99a
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x2b6472b392d0, 0xc421c46f60, 0xc400000004, 0x2, 0x219b530, 0x1d, 0xc420f61830, 0x1, 0x1)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:146 +0x331
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x2b6472b392d0, 0xc421c46f60, 0x1, 0x4, 0x219b530, 0x1d, 0xc420f61830, 0x1, 0x1)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:88 +0x9a
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x2b6472b392d0, 0xc421c46f60, 0x219b530, 0x1d, 0xc420f61830, 0x1, 0x1)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:172 +0x90
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/storage.(*Store).removeReplicaImpl(0xc421a3a000, 0x2b6472b392d0, 0xc421c46f60, 0xc420d05680, 0x1, 0x3f168f0, 0x0, 0x0, 0xc421824ec8, 0x2, ...)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2105 +0x44e
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/storage.(*Store).RemoveReplica(0xc421a3a000, 0x2b6472b392d0, 0xc421c46f60, 0xc420d05680, 0x1, 0x3f168f0, 0x0, 0x0, 0xc421824ec8, 0x2, ...)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2078 +0x115
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/storage.(*replicaGCQueue).process(0xc421b9b480, 0x2b6472b392d0, 0xc421c46f60, 0xc420d05680, 0xc42020b800, 0x13, 0x20, 0x0, 0xc42108e960)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_gc_queue.go:206 +0xa8f
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc421e49e00, 0x2b6471aeb420, 0xc422102cf0, 0xc420d05680, 0xc42174e040, 0x0, 0x0)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:574 +0x5b2
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2()
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:498 +0x119
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc4220defc0, 0xc420f61e50, 0x0, 0x0)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x10e
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:505 +0x45a
[04:05:15][Step 2/2] github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4220defc0, 0xc421baf600)
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x8b
[04:05:15][Step 2/2] created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
[04:05:15][Step 2/2] 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x74

cockroach-teamcity · 2017-03-08T09:27:12Z

SHA: https://github.com/cockroachdb/cockroach/commits/6880337159ac1b1984fa0136032db9b5572d80ed

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=false
TAGS=
GOFLAGS=-race

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=172955&tab=buildLog

cockroach-teamcity · 2017-03-21T08:28:53Z

SHA: https://github.com/cockroachdb/cockroach/commits/593db202fdbd598cb1bf0281e9567c6080b82d76

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=false
TAGS=
GOFLAGS=-race

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=189107&tab=buildLog

cockroach-teamcity added O-robot Originated from a bot. C-test-failure Broken test (automatically or manually discovered). labels Dec 24, 2016

jordanlewis mentioned this issue Dec 28, 2016

github.com/cockroachdb/cockroach/pkg/storage: TestRemovePlaceholderRace failed under stress #12589

Closed

a-robinson mentioned this issue Jan 5, 2017

github.com/cockroachdb/cockroach/pkg/storage: TestRemovePlaceholderRace failed under stress #12699

Closed

a-robinson mentioned this issue Jan 24, 2017

github.com/cockroachdb/cockroach/pkg/storage: TestRemovePlaceholderRace failed under stress #13113

Closed

tamird assigned rjnn Feb 6, 2017

petermattis changed the title ~~github.com/cockroachdb/cockroach/pkg/storage: TestRemovePlaceholderRace failed under stress~~ storage: TestRemovePlaceholderRace failed under stress Feb 23, 2017

petermattis added this to the 1.0 milestone Feb 23, 2017

tamird mentioned this issue Mar 16, 2017

stability: crash on uninitialized replica #14193

Closed

a-robinson mentioned this issue Mar 17, 2017

storage: panic in raftGroup.Step while running under stressrace #14231

Closed

a-robinson assigned a-robinson and unassigned rjnn Mar 19, 2017

a-robinson mentioned this issue Mar 20, 2017

storage: Don't allow preemptive snapshots on initialized replicas #14261

Closed

a-robinson mentioned this issue Mar 22, 2017

storage: Don't process uninitialized replicas in the queue #14306

Merged

a-robinson closed this as completed in #14306 Mar 22, 2017

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

storage: TestRemovePlaceholderRace failed under stress #12574

storage: TestRemovePlaceholderRace failed under stress #12574

cockroach-teamcity commented Dec 24, 2016

a-robinson commented Dec 27, 2016

cockroach-teamcity commented Mar 8, 2017

cockroach-teamcity commented Mar 21, 2017

storage: TestRemovePlaceholderRace failed under stress #12574

storage: TestRemovePlaceholderRace failed under stress #12574

Comments

cockroach-teamcity commented Dec 24, 2016

a-robinson commented Dec 27, 2016

cockroach-teamcity commented Mar 8, 2017

cockroach-teamcity commented Mar 21, 2017