diff --git a/server/monitor.go b/server/monitor.go index 0a2815b943b..c0c2bc08e83 100644 --- a/server/monitor.go +++ b/server/monitor.go @@ -1421,6 +1421,7 @@ func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) { Routes LeafNodes Gateways + Raft Groups Health Probe Help @@ -1436,6 +1437,7 @@ func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) { s.basePath(RoutezPath), s.basePath(LeafzPath), s.basePath(GatewayzPath), + s.basePath(RaftzPath), s.basePath(HealthzPath), ) } @@ -3741,3 +3743,138 @@ func (s *Server) profilez(opts *ProfilezOptions) *ProfilezStatus { Profile: buffer.Bytes(), } } + +type RaftzGroup struct { + ID string `json:"id"` + State string `json:"state"` + Size int `json:"size"` + QuorumNeeded int `json:"quorum_needed"` + Observer bool `json:"observer,omitempty"` + Paused bool `json:"paused,omitempty"` + Committed uint64 `json:"committed"` + Applied uint64 `json:"applied"` + CatchingUp bool `json:"catching_up,omitempty"` + Leader string `json:"leader,omitempty"` + EverHadLeader bool `json:"ever_had_leader"` + Term uint64 `json:"term"` + Vote string `json:"voted_for,omitempty"` + PTerm uint64 `json:"pterm"` + PIndex uint64 `json:"pindex"` + IPQPropLen int `json:"ipq_proposal_len"` + IPQEntryLen int `json:"ipq_entry_len"` + IPQRespLen int `json:"ipq_resp_len"` + IPQApplyLen int `json:"ipq_apply_len"` + WAL StreamState `json:"wal"` + WALError error `json:"wal_error,omitempty"` + Peers map[string]RaftzGroupPeer `json:"peers"` +} + +type RaftzGroupPeer struct { + Name string `json:"name"` + Known bool `json:"known"` + LastReplicatedIndex uint64 `json:"last_replicated_index,omitempty"` + LastSeen string `json:"last_seen,omitempty"` +} + +func (s *Server) HandleRaftz(w http.ResponseWriter, r *http.Request) { + if s.raftNodes == nil { + w.WriteHeader(404) + w.Write([]byte("No Raft nodes registered")) + return + } + + gfilter := r.URL.Query().Get("group") + afilter := r.URL.Query().Get("acc") + if afilter == "" { + afilter = s.SystemAccount().Name + } + + groups := map[string]RaftNode{} + infos := map[string]map[string]RaftzGroup{} // account -> group ID + + s.rnMu.RLock() + if gfilter != _EMPTY_ { + if rg, ok := s.raftNodes[gfilter]; ok && rg != nil { + if n, ok := rg.(*raft); ok { + if n.accName == afilter { + groups[gfilter] = rg + } + } + } + } else { + for name, rg := range s.raftNodes { + if rg == nil { + continue + } + if n, ok := rg.(*raft); ok { + if n.accName != afilter { + continue + } + groups[name] = rg + } + } + } + s.rnMu.RUnlock() + + if len(groups) == 0 { + w.WriteHeader(404) + w.Write([]byte("No Raft nodes found, does the specified account/group exist?")) + return + } + + for name, rg := range groups { + n, ok := rg.(*raft) + if n == nil || !ok { + continue + } + if _, ok := infos[n.accName]; !ok { + infos[n.accName] = map[string]RaftzGroup{} + } + // Only take the lock once, using the public RaftNode functions would + // cause us to take and release the locks over and over again. + n.RLock() + info := RaftzGroup{ + ID: n.id, + State: RaftState(n.state.Load()).String(), + Size: n.csz, + QuorumNeeded: n.qn, + Observer: n.observer, + Paused: n.paused, + Committed: n.commit, + Applied: n.applied, + CatchingUp: n.catchup != nil, + Leader: n.leader, + EverHadLeader: n.pleader, + Term: n.term, + Vote: n.vote, + PTerm: n.pterm, + PIndex: n.pindex, + IPQPropLen: n.prop.len(), + IPQEntryLen: n.entry.len(), + IPQRespLen: n.resp.len(), + IPQApplyLen: n.apply.len(), + WALError: n.werr, + Peers: map[string]RaftzGroupPeer{}, + } + n.wal.FastState(&info.WAL) + for id, p := range n.peers { + if id == n.id { + continue + } + peer := RaftzGroupPeer{ + Name: s.serverNameForNode(id), + Known: p.kp, + LastReplicatedIndex: p.li, + } + if p.ts > 0 { + peer.LastSeen = time.Since(time.Unix(0, p.ts)).String() + } + info.Peers[id] = peer + } + n.RUnlock() + infos[n.accName][name] = info + } + + b, _ := json.MarshalIndent(infos, "", " ") + ResponseHandler(w, r, b) +} diff --git a/server/server.go b/server/server.go index 426d96208fa..fc8c886381c 100644 --- a/server/server.go +++ b/server/server.go @@ -2892,6 +2892,7 @@ const ( JszPath = "/jsz" HealthzPath = "/healthz" IPQueuesPath = "/ipqueuesz" + RaftzPath = "/raftz" ) func (s *Server) basePath(p string) string { @@ -3006,6 +3007,8 @@ func (s *Server) startMonitoring(secure bool) error { mux.HandleFunc(s.basePath(HealthzPath), s.HandleHealthz) // IPQueuesz mux.HandleFunc(s.basePath(IPQueuesPath), s.HandleIPQueuesz) + // Raftz + mux.HandleFunc(s.basePath(RaftzPath), s.HandleRaftz) // Do not set a WriteTimeout because it could cause cURL/browser // to return empty response or unable to display page if the