ui: dispay circuit breakers in problem ranges and range status

This PR adds changes to the reports/problemranges and reports/range pages. Ranges with replicas that have a circuit breaker will show up as problem ranges and the circuit breaker error will show up as a row on the status page. Release note (ui change): display circuit breakers in problems ranges and range status
cockroachdb · Feb 10, 2022 · 0ffc720 · 0ffc720
1 parent d10188f
commit 0ffc720
Show file tree

Hide file tree

Showing 9 changed files with 41 additions and 1 deletion.
diff --git a/docs/generated/http/full.md b/docs/generated/http/full.md
@@ -1314,6 +1314,7 @@ RangeProblems describes issues reported by a range. For internal use only.
 | no_lease | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  |  | [reserved](#support-status) |
 | quiescent_equals_ticking | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  | Quiescent ranges do not tick by definition, but we track this in two different ways and suspect that they're getting out of sync. If the replica's quiescent flag doesn't agree with the store's list of replicas that are ticking, warn about it. | [reserved](#support-status) |
 | raft_log_too_large | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  | When the raft log is too large, it can be a symptom of other issues. | [reserved](#support-status) |
+| circuit_breaker_error | [bool](#cockroach.server.serverpb.RaftDebugResponse-bool) |  |  | [reserved](#support-status) |
 
 
 
@@ -1520,6 +1521,7 @@ RangeProblems describes issues reported by a range. For internal use only.
 | no_lease | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  |  | [reserved](#support-status) |
 | quiescent_equals_ticking | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  | Quiescent ranges do not tick by definition, but we track this in two different ways and suspect that they're getting out of sync. If the replica's quiescent flag doesn't agree with the store's list of replicas that are ticking, warn about it. | [reserved](#support-status) |
 | raft_log_too_large | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  | When the raft log is too large, it can be a symptom of other issues. | [reserved](#support-status) |
+| circuit_breaker_error | [bool](#cockroach.server.serverpb.RangesResponse-bool) |  |  | [reserved](#support-status) |
 
 
 
@@ -3049,6 +3051,7 @@ Support status: [reserved](#support-status)
 | overreplicated_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
 | quiescent_equals_ticking_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
 | raft_log_too_large_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
+| circuit_breaker_error_range_ids | [int64](#cockroach.server.serverpb.ProblemRangesResponse-int64) | repeated |  | [reserved](#support-status) |
 
 
 
@@ -3344,6 +3347,7 @@ RangeProblems describes issues reported by a range. For internal use only.
 | no_lease | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  |  | [reserved](#support-status) |
 | quiescent_equals_ticking | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  | Quiescent ranges do not tick by definition, but we track this in two different ways and suspect that they're getting out of sync. If the replica's quiescent flag doesn't agree with the store's list of replicas that are ticking, warn about it. | [reserved](#support-status) |
 | raft_log_too_large | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  | When the raft log is too large, it can be a symptom of other issues. | [reserved](#support-status) |
+| circuit_breaker_error | [bool](#cockroach.server.serverpb.RangeResponse-bool) |  |  | [reserved](#support-status) |
 
 
 

diff --git a/docs/generated/swagger/spec.json b/docs/generated/swagger/spec.json
@@ -1022,6 +1022,10 @@
       "type": "object",
       "title": "RangeProblems describes issues reported by a range. For internal use only.",
       "properties": {
+        "circuit_breaker_error": {
+          "type": "boolean",
+          "x-go-name": "CircuitBreakerError"
+        },
         "leader_not_lease_holder": {
           "type": "boolean",
           "x-go-name": "LeaderNotLeaseHolder"

diff --git a/pkg/server/problem_ranges.go b/pkg/server/problem_ranges.go
@@ -129,6 +129,10 @@ func (s *statusServer) ProblemRanges(
 					problems.RaftLogTooLargeRangeIDs =
 						append(problems.RaftLogTooLargeRangeIDs, info.State.Desc.RangeID)
 				}
+				if info.Problems.CircuitBreakerError {
+					problems.CircuitBreakerErrorRangeIDs =
+						append(problems.CircuitBreakerErrorRangeIDs, info.State.Desc.RangeID)
+				}
 			}
 			sort.Sort(roachpb.RangeIDSlice(problems.UnavailableRangeIDs))
 			sort.Sort(roachpb.RangeIDSlice(problems.RaftLeaderNotLeaseHolderRangeIDs))
@@ -138,6 +142,7 @@ func (s *statusServer) ProblemRanges(
 			sort.Sort(roachpb.RangeIDSlice(problems.OverreplicatedRangeIDs))
 			sort.Sort(roachpb.RangeIDSlice(problems.QuiescentEqualsTickingRangeIDs))
 			sort.Sort(roachpb.RangeIDSlice(problems.RaftLogTooLargeRangeIDs))
+			sort.Sort(roachpb.RangeIDSlice(problems.CircuitBreakerErrorRangeIDs))
 			response.ProblemsByNodeID[resp.nodeID] = problems
 		case <-ctx.Done():
 			return nil, status.Errorf(codes.DeadlineExceeded, ctx.Err().Error())

diff --git a/pkg/server/serverpb/status.proto b/pkg/server/serverpb/status.proto
@@ -389,6 +389,7 @@ message RangeProblems {
 
   // When the raft log is too large, it can be a symptom of other issues.
   bool raft_log_too_large = 7;
+  bool circuit_breaker_error = 9;
 }
 
 // RangeStatistics describes statistics reported by a range. For internal use
@@ -1044,6 +1045,11 @@ message ProblemRangesResponse {
       (gogoproto.casttype) =
           "github.com/cockroachdb/cockroach/pkg/roachpb.RangeID"
     ];
+    repeated int64 circuit_breaker_error_range_ids = 10 [
+      (gogoproto.customname) = "CircuitBreakerErrorRangeIDs",
+      (gogoproto.casttype) = 
+          "github.com/cockroachdb/cockroach/pkg/roachpb.RangeID"
+    ];
   }
   reserved 1 to 7;
   // NodeID is the node that submitted all the requests.

diff --git a/pkg/server/status.go b/pkg/server/status.go
@@ -1993,6 +1993,7 @@ func (s *statusServer) rangesHelper(
 				NoLease:                metrics.Leader && !metrics.LeaseValid && !metrics.Quiescent,
 				QuiescentEqualsTicking: raftStatus != nil && metrics.Quiescent == metrics.Ticking,
 				RaftLogTooLarge:        metrics.RaftLogTooLarge,
+				CircuitBreakerError:    len(state.CircuitBreakerError) > 0,
 			},
 			LeaseStatus:                 metrics.LeaseStatus,
 			Quiescent:                   metrics.Quiescent,

diff --git a/pkg/ui/workspaces/db-console/src/views/reports/containers/problemRanges/connectionsTable.tsx b/pkg/ui/workspaces/db-console/src/views/reports/containers/problemRanges/connectionsTable.tsx
@@ -71,6 +71,10 @@ const connectionTableColumns: ConnectionTableColumn[] = [
     title: "Raft log too large",
     extract: problem => problem.raft_log_too_large_range_ids.length,
   },
+  {
+    title: "Circuit breaker error",
+    extract: problem => problem.circuit_breaker_error_range_ids.length,
+  },
   {
     title: "Total",
     extract: problem => {
@@ -82,7 +86,8 @@ const connectionTableColumns: ConnectionTableColumn[] = [
         problem.underreplicated_range_ids.length +
         problem.overreplicated_range_ids.length +
         problem.quiescent_equals_ticking_range_ids.length +
-        problem.raft_log_too_large_range_ids.length
+        problem.raft_log_too_large_range_ids.length +
+        problem.circuit_breaker_error_range_ids.length
       );
     },
   },

diff --git a/pkg/ui/workspaces/db-console/src/views/reports/containers/problemRanges/index.tsx b/pkg/ui/workspaces/db-console/src/views/reports/containers/problemRanges/index.tsx
@@ -203,6 +203,11 @@ export class ProblemRanges extends React.Component<ProblemRangesProps, {}> {
           problems={problems}
           extract={problem => problem.quiescent_equals_ticking_range_ids}
         />
+        <ProblemRangeList
+          name="Circuit breaker error"
+          problems={problems}
+          extract={problem => problem.circuit_breaker_error_range_ids}
+        />
       </div>
     );
   }

diff --git a/pkg/ui/workspaces/db-console/src/views/reports/containers/range/rangeTable.tsx b/pkg/ui/workspaces/db-console/src/views/reports/containers/range/rangeTable.tsx
@@ -239,6 +239,11 @@ const rangeTableDisplayList: RangeTableRow[] = [
     display: "Closed timestamp LAI - side transport (centralized state)",
     compareToLeader: false,
   },
+  {
+    variable: "circuitBreakerError",
+    display: "Circuit Breaker Error",
+    compareToLeader: false,
+  },
 ];
 
 const rangeTableEmptyContent: RangeTableCellContent = {
@@ -854,6 +859,9 @@ export default class RangeTable extends React.Component<RangeTableProps, {}> {
             ? "range-table__cell--warning"
             : "",
         ),
+        circuitBreakerError: this.createContent(
+          info.state.circuit_breaker_error,
+        ),
       });
     });
 

diff --git a/pkg/ui/workspaces/db-console/styl/pages/reports.styl b/pkg/ui/workspaces/db-console/styl/pages/reports.styl
@@ -311,6 +311,8 @@ $reports-table
   list-style-type none
   margin 0
   padding 0
+  white-space normal
+  overflow-x auto
 
 .debug-table
   margin 0 0 40px