PD may panic when exit

[lance6716]

Bug Report

What did you do?

found it in adding a lightning integration test

What did you expect to see?

What did you see instead?

[negroni] PANIC: runtime error: invalid memory address or nil pointer dereference
goroutine 96187 [running]:
github.com/urfave/negroni.(*Recovery).ServeHTTP.func1()
        /go/pkg/mod/github.com/urfave/[email protected]/recovery.go:159 +0xc7
panic({0x2ab4580, 0x4d8a540})
        /usr/local/go/src/runtime/panic.go:884 +0x213
github.com/tikv/pd/server/cluster.(*RaftCluster).GetStoreMinResolvedTS(0x0, 0xc00254e570?)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/server/cluster/cluster.go:2479 +0x41
github.com/tikv/pd/server/api.(*minResolvedTSHandler).GetStoreMinResolvedTS(0xc00129dfa0, {0x7f932465c570?, 0xc006831028}, 0x2b3d300?)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/server/api/min_resolved_ts.go:61 +0xf6
github.com/urfave/negroni.WrapFunc.func1({0x7f932465c570, 0xc006831028}, 0x2b3d300?, 0xc005aa57c0)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:56 +0x31
github.com/urfave/negroni.HandlerFunc.ServeHTTP(0x18?, {0x7f932465c570?, 0xc006831028?}, 0x297c2c0?, 0xc005aac2b8?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:29 +0x33
github.com/urfave/negroni.middleware.ServeHTTP({{0x3b02260?, 0xc001334100?}, 0xc000012a68?}, {0x7f932465c570, 0xc006831028}, 0x1000000e6e4f049?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/tikv/pd/server/api.(*rateLimitMiddleware).ServeHTTP(0x7f935c72c5b8?, {0x7f932465c570?, 0xc006831028?}, 0xc005a9b950?, 0x0?)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/server/api/middleware.go:170 +0x31e
github.com/urfave/negroni.middleware.ServeHTTP({{0x3afeb60?, 0xc0002b0c38?}, 0xc000012a50?}, {0x7f932465c570, 0xc006831028}, 0xc005a9b950?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/tikv/pd/server/api.(*auditMiddleware).ServeHTTP(0xc0002b0c30, {0x7f932465c570, 0xc006831028}, 0xc005c89d00, 0xc005aa5780)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/server/api/middleware.go:150 +0x58e
github.com/urfave/negroni.middleware.ServeHTTP({{0x3afeb40?, 0xc0002b0c30?}, 0xc000012a38?}, {0x7f932465c570, 0xc006831028}, 0x3af1701?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/tikv/pd/server/api.(*requestInfoMiddleware).ServeHTTP(0x8?, {0x7f932465c570, 0xc006831028}, 0xc005c89c00, 0xc005aa5760)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/server/api/middleware.go:76 +0x2fa
github.com/urfave/negroni.middleware.ServeHTTP({{0x3afeb80?, 0xc0002b0c28?}, 0xc000012a08?}, {0x7f932465c570, 0xc006831028}, 0xc005c89c00?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/urfave/negroni.(*Negroni).ServeHTTP(0xc001311a40, {0x7f932465c570?, 0xc006831020}, 0x4ebcb98?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:96 +0x125
github.com/tikv/pd/server/api.clusterMiddleware.Middleware.func1({0x7f932465c570?, 0xc006831020}, 0xc005c89b00)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/server/api/middleware.go:99 +0x1ea
net/http.HandlerFunc.ServeHTTP(0xc005c89a00?, {0x7f932465c570?, 0xc006831020?}, 0x0?)
        /usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000801800, {0x7f932465c570, 0xc006831020}, 0xc005c89900)
        /go/pkg/mod/github.com/gorilla/[email protected]/mux.go:210 +0x1cf
github.com/urfave/negroni.Wrap.func1({0x7f932465c570, 0xc006831020}, 0x2b3d300?, 0xc005aa5740)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:46 +0x4b
github.com/urfave/negroni.HandlerFunc.ServeHTTP(0x4e7f420?, {0x7f932465c570?, 0xc006831020?}, 0x2d591c0?, 0xc005b10c78?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:29 +0x33
github.com/urfave/negroni.middleware.ServeHTTP({{0x3b02260?, 0xc000012ed0?}, 0xc000012f30?}, {0x7f932465c570, 0xc006831020}, 0x18?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/tikv/pd/pkg/utils/apiutil/serverapi.(*redirector).ServeHTTP(0xc0013492e0, {0x7f932465c570, 0xc006831020}, 0xc005c89900, 0xc005aa5720)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/pkg/utils/apiutil/serverapi/middleware.go:135 +0xf8
github.com/urfave/negroni.middleware.ServeHTTP({{0x3afea40?, 0xc0013492e0?}, 0xc000012f18?}, {0x7f932465c570, 0xc006831020}, 0xc000298000?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/tikv/pd/pkg/utils/apiutil/serverapi.(*runtimeServiceValidator).ServeHTTP(0x8?, {0x7f932465c570, 0xc006831020}, 0x0?, 0xc005aa5700)
        /home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/pd/pkg/utils/apiutil/serverapi/middleware.go:47 +0xcb
github.com/urfave/negroni.middleware.ServeHTTP({{0x3afea60?, 0xc0005e7540?}, 0xc000012ee8?}, {0x7f932465c570, 0xc006831020}, 0x29b97e0?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/urfave/negroni.(*Negroni).ServeHTTP(0xc0013422a0, {0x7f932465c570?, 0xc006831018}, 0x2c461a0?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:96 +0x125
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000801740, {0x7f932465c570, 0xc006831018}, 0xc005c89700)
        /go/pkg/mod/github.com/gorilla/[email protected]/mux.go:210 +0x1cf
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000801680, {0x7f932465c570, 0xc006831018}, 0xc005c89200)
        /go/pkg/mod/github.com/gorilla/[email protected]/mux.go:210 +0x1cf
github.com/urfave/negroni.Wrap.func1({0x7f932465c570, 0xc006831018}, 0x2b3d300?, 0xc005aa56e0)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:46 +0x4b
github.com/urfave/negroni.HandlerFunc.ServeHTTP(0xeb31b0?, {0x7f932465c570?, 0xc006831018?}, 0x12df496?, 0x2d8baeb?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:29 +0x33
github.com/urfave/negroni.middleware.ServeHTTP({{0x3b02260?, 0xc0000131d0?}, 0xc000013200?}, {0x7f932465c570, 0xc006831018}, 0x2b3d300?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/urfave/negroni.(*Recovery).ServeHTTP(0x8?, {0x7f932465c570?, 0xc006831018?}, 0x0?, 0xc004734af8?)
        /go/pkg/mod/github.com/urfave/[email protected]/recovery.go:193 +0x86
github.com/urfave/negroni.middleware.ServeHTTP({{0x3afec80?, 0xc0004c9b30?}, 0xc0000131e8?}, {0x7f932465c570, 0xc006831018}, 0xc005a9b8f0?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:38 +0xb6
github.com/urfave/negroni.(*Negroni).ServeHTTP(0xc000618ed0, {0x3b156a0?, 0xc006831008}, 0xc00254e2d0?)
        /go/pkg/mod/github.com/urfave/[email protected]/negroni.go:96 +0x125
net/http.(*ServeMux).ServeHTTP(0x3b156a0?, {0x3b156a0, 0xc006831008}, 0xc005c89200)
        /usr/local/go/src/net/http/server.go:2500 +0x149
go.etcd.io/etcd/embed.(*accessController).ServeHTTP(0xc0004487e0, {0x3b156a0, 0xc006831008}, 0xc005c89200)
        /go/pkg/mod/go.etcd.io/[email protected]/embed/serve.go:381 +0x4eb
net/http.serverHandler.ServeHTTP({0x12295c0?}, {0x3b156a0, 0xc006831008}, 0xc005c89200)
        /usr/local/go/src/net/http/server.go:2936 +0x316
net/http.initALPNRequest.ServeHTTP({{0x3b17c18?, 0xc001d632c0?}, 0xc0017ece00?, {0xc0002e85a0?}}, {0x3b156a0, 0xc006831008}, 0xc005c89200)
        /usr/local/go/src/net/http/server.go:3545 +0x245
golang.org/x/net/http2.(*serverConn).runHandler(0x4ebcb98?, 0x0?, 0x0?, 0xc004734fb8?)
        /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:2314 +0x83
created by golang.org/x/net/http2.(*serverConn).processHeaders
        /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:2028 +0x68a

What version of PD are you using (pd-server -V)?

$ bin/pd-server -V
Release Version: v7.3.0-alpha
Edition: Community
Git Commit Hash: 0934e641129d8bac3c5975372b2ce492b81fd9db
Git Branch: heads/refs/tags/v7.3.0-alpha
UTC Build Time:  2023-06-19 02:24:53

[rleungx]

/cc @HuSharp

[HuSharp]

please add pd version :) @lance6716

[lance6716]

please add pd version :) @lance6716

updated