[Feature] Deployment & Members Condition metrics

CHANGELOG.md

@@ -10,6 +10,7 @@
 - (Feature) JobScheduler Volumes, Probes, Lifecycle and Ports integration
 - (Feature) Merge ArangoDB Usage Metrics
 - (Bugfix) Check Connection to the ArangoDB before creating Backup
+- (Feature) Deployment & Members Condition metrics
 
 ## [1.2.38](https://github.com/arangodb/kube-arangodb/tree/1.2.38) (2024-02-22)
 - (Feature) Extract GRPC Server

docs/generated/metrics/README.md

@@ -24,11 +24,13 @@ has_toc: false
 |                  [arangodb_operator_agency_cache_member_serving](./arangodb_operator_agency_cache_member_serving.md)                  | arangodb_operator |   agency_cache    |  Gauge  | Determines if agency member is reachable                                              |
 |                         [arangodb_operator_agency_cache_present](./arangodb_operator_agency_cache_present.md)                         | arangodb_operator |   agency_cache    |  Gauge  | Determines if local agency cache is present                                           |
 |                         [arangodb_operator_agency_cache_serving](./arangodb_operator_agency_cache_serving.md)                         | arangodb_operator |   agency_cache    |  Gauge  | Determines if agency is serving                                                       |
+|                        [arangodb_operator_deployment_conditions](./arangodb_operator_deployment_conditions.md)                        | arangodb_operator |    deployment     |  Gauge  | Representation of the ArangoDeployment condition state (true/false)                   |
 |                            [arangodb_operator_engine_assertions](./arangodb_operator_engine_assertions.md)                            | arangodb_operator |      engine       | Counter | Number of assertions invoked during Operator runtime                                  |
 |                            [arangodb_operator_engine_ops_alerts](./arangodb_operator_engine_ops_alerts.md)                            | arangodb_operator |      engine       | Counter | Counter for actions which requires ops attention                                      |
 |                      [arangodb_operator_engine_panics_recovered](./arangodb_operator_engine_panics_recovered.md)                      | arangodb_operator |      engine       | Counter | Number of Panics recovered inside Operator reconciliation loop                        |
 |             [arangodb_operator_kubernetes_client_request_errors](./arangodb_operator_kubernetes_client_request_errors.md)             | arangodb_operator | kubernetes_client | Counter | Number of Kubernetes Client request errors                                            |
 |                   [arangodb_operator_kubernetes_client_requests](./arangodb_operator_kubernetes_client_requests.md)                   | arangodb_operator | kubernetes_client | Counter | Number of Kubernetes Client requests                                                  |
+|                           [arangodb_operator_members_conditions](./arangodb_operator_members_conditions.md)                           | arangodb_operator |      members      |  Gauge  | Representation of the ArangoMember condition state (true/false)                       |
 |      [arangodb_operator_members_unexpected_container_exit_codes](./arangodb_operator_members_unexpected_container_exit_codes.md)      | arangodb_operator |      members      | Counter | Counter of unexpected restarts in pod (Containers/InitContainers/EphemeralContainers) |
 |                           [arangodb_operator_rebalancer_enabled](./arangodb_operator_rebalancer_enabled.md)                           | arangodb_operator |    rebalancer     |  Gauge  | Determines if rebalancer is enabled                                                   |
 |                     [arangodb_operator_rebalancer_moves_current](./arangodb_operator_rebalancer_moves_current.md)                     | arangodb_operator |    rebalancer     |  Gauge  | Define how many moves are currently in progress                                       |

docs/generated/metrics/arangodb_operator_deployment_conditions.md

@@ -0,0 +1,19 @@
+---
+layout: page
+title: arangodb_operator_deployment_conditions
+parent: List of available metrics
+---
+
+# arangodb_operator_deployment_conditions (Gauge)
+
+## Description
+
+Representation of the ArangoDeployment condition state (true/false)
+
+## Labels
+
+|   Label   | Description          |
+|:---------:|:---------------------|
+| namespace | Deployment Namespace |
+|   name    | Deployment Name      |
+| condition | Condition Name       |

docs/generated/metrics/arangodb_operator_members_conditions.md

@@ -0,0 +1,20 @@
+---
+layout: page
+title: arangodb_operator_members_conditions
+parent: List of available metrics
+---
+
+# arangodb_operator_members_conditions (Gauge)
+
+## Description
+
+Representation of the ArangoMember condition state (true/false)
+
+## Labels
+
+|   Label   | Description          |
+|:---------:|:---------------------|
+| namespace | Deployment Namespace |
+|   name    | Deployment Name      |
+|  member   | Member ID            |
+| condition | Condition Name       |

internal/metrics.go.tmpl

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

internal/metrics.item.go.tmpl

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

internal/metrics.yaml

@@ -219,6 +219,18 @@ namespaces:
             description: "DeploymentReplication Namespace"
           - key: name
             description: "DeploymentReplication Name"
+    deployment:
+      conditions:
+        shortDescription: "Representation of the ArangoDeployment condition state (true/false)"
+        description: "Representation of the ArangoDeployment condition state (true/false)"
+        type: "Gauge"
+        labels:
+          - key: namespace
+            description: "Deployment Namespace"
+          - key: name
+            description: "Deployment Name"
+          - key: condition
+            description: "Condition Name"
     members:
       unexpected_container_exit_codes:
         shortDescription: "Counter of unexpected restarts in pod (Containers/InitContainers/EphemeralContainers)"
@@ -239,6 +251,19 @@ namespaces:
             description: "ExitCode"
           - key: reason
             description: "Reason"
+      conditions:
+        shortDescription: "Representation of the ArangoMember condition state (true/false)"
+        description: "Representation of the ArangoMember condition state (true/false)"
+        type: "Gauge"
+        labels:
+          - key: namespace
+            description: "Deployment Namespace"
+          - key: name
+            description: "Deployment Name"
+          - key: member
+            description: "Member ID"
+          - key: condition
+            description: "Condition Name"
     engine:
       panics_recovered:
         shortDescription: "Number of Panics recovered inside Operator reconciliation loop"

pkg/deployment/deployment_inspector.go

@@ -153,6 +153,18 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 		d.metrics.Deployment.Propagated = updated.Status.Conditions.IsTrue(api.ConditionTypeSpecPropagated)
 		d.metrics.Deployment.UpToDate = updated.Status.Conditions.IsTrue(api.ConditionTypeUpToDate)
 
+		d.metrics.Conditions.RefreshDeployment(updated.Status.Conditions,
+			api.ConditionTypeSpecAccepted,
+			api.ConditionTypeSpecPropagated,
+			api.ConditionTypeUpToDate)
+
+		d.metrics.Conditions.RefreshMembers(updated.Status.Members.AsList(),
+			api.ConditionTypeServing,
+			api.ConditionTypeScheduled,
+			api.ConditionTypeReachable,
+			api.ConditionTypeStarted,
+			api.ConditionTypeReady)
+
 		// Is the deployment in failed state, if so, give up.
 		if d.GetPhase() == api.DeploymentPhaseFailed {
 			d.log.Debug("Deployment is in Failed state.")
@@ -482,6 +494,11 @@ func (d *Deployment) isUpToDateStatus(status api.DeploymentStatus) (upToDate boo
 			reason = "PVC is resizing"
 			return
 		}
+		if !member.Conditions.IsTrue(api.ConditionTypeReady) {
+			upToDate = false
+			reason = "Not all members are ready"
+			return
+		}
 	}
 
 	return

pkg/deployment/metrics.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -41,6 +41,8 @@ type Metrics struct {
 	Deployment struct {
 		Accepted, UpToDate, Propagated bool
 	}
+
+	Conditions ConditionsMetrics
 }
 
 func (d *Deployment) CollectMetrics(m metrics.PushMetric) {
@@ -90,4 +92,7 @@ func (d *Deployment) CollectMetrics(m metrics.PushMetric) {
 	if r := d.resources; r != nil {
 		r.CollectMetrics(m)
 	}
+
+	// Conditions
+	d.metrics.Conditions.CollectMetrics(d.namespace, d.name, m)
 }

pkg/deployment/metrics_conditions.go

@@ -0,0 +1,93 @@
+//
+// DISCLAIMER
+//
+// Copyright 2024 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package deployment
+
+import (
+	"sync"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/generated/metric_descriptions"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/metrics"
+)
+
+type ConditionsMetricsMap map[api.ConditionType]bool
+
+type ConditionsMetrics struct {
+	lock sync.Mutex
+
+	conditions ConditionsMetricsMap
+
+	memberConditions map[string]ConditionsMetricsMap
+}
+
+func (c *ConditionsMetrics) CollectMetrics(namespace, name string, m metrics.PushMetric) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	for k, v := range c.conditions {
+		m.Push(metric_descriptions.ArangodbOperatorDeploymentConditionsGauge(util.BoolSwitch[float64](v, 1, 0), namespace, name, string(k)))
+	}
+
+	for member := range c.memberConditions {
+		for k, v := range c.memberConditions[member] {
+			m.Push(metric_descriptions.ArangodbOperatorMembersConditionsGauge(util.BoolSwitch[float64](v, 1, 0), namespace, name, member, string(k)))
+		}
+	}
+}
+
+func (c *ConditionsMetrics) RefreshDeployment(conditions api.ConditionList, types ...api.ConditionType) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.conditions = c.extractConditionsMap(conditions, types...)
+}
+
+func (c *ConditionsMetrics) RefreshMembers(members api.DeploymentStatusMemberElements, types ...api.ConditionType) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if len(members) == 0 {
+		c.memberConditions = nil
+		return
+	}
+
+	ret := make(map[string]ConditionsMetricsMap, len(members))
+
+	for _, member := range members {
+		ret[member.Member.ID] = c.extractConditionsMap(member.Member.Conditions, types...)
+	}
+
+	c.memberConditions = ret
+}
+
+func (c *ConditionsMetrics) extractConditionsMap(conditions api.ConditionList, types ...api.ConditionType) ConditionsMetricsMap {
+	if len(types) == 0 {
+		return nil
+	}
+
+	ret := make(ConditionsMetricsMap, len(types))
+	for _, t := range types {
+		ret[t] = conditions.IsTrue(t)
+	}
+
+	return ret
+}

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -466,6 +466,8 @@ func groupReadyForRestart(context PlanBuilderContext, status api.DeploymentStatu
 		return true, "Bootstrap not completed, restart is allowed"
 	}
 
+	members := status.Members.MembersOfGroup(group)
+
 	// If current member did not become ready even once. Kill it
 	if !member.Conditions.IsTrue(api.ConditionTypeStarted) {
 		return true, "Member is not started"
@@ -476,10 +478,17 @@ func groupReadyForRestart(context PlanBuilderContext, status api.DeploymentStatu
 		return true, "Member is not serving"
 	}
 
-	if !status.Members.MembersOfGroup(group).AllMembersServing() {
+	if !members.AllMembersServing() {
 		return false, "Not all members are serving"
 	}
 
+	if member.Conditions.IsTrue(api.ConditionTypeReady) {
+		// Our pod is ready, lets check other pods
+		if !members.AllMembersReady() {
+			return false, "Not all members are ready"
+		}
+	}
+
 	switch group {
 	case api.ServerGroupDBServers:
 		agencyState, ok := context.GetAgencyCache()