[Fix] 1.2.16-preview-1 fix pack (#1104)

docs/generated/metrics/README.md

@@ -23,5 +23,6 @@
 |                   [arangodb_operator_rebalancer_moves_succeeded](./arangodb_operator_rebalancer_moves_succeeded.md)                   | arangodb_operator |  rebalancer  | Counter | Define how many moves succeeded                                                       |
 |          [arangodb_operator_resources_arangodeployment_accepted](./arangodb_operator_resources_arangodeployment_accepted.md)          | arangodb_operator |  resources   |  Gauge  | Defines if ArangoDeployment has been accepted                                         |
 |  [arangodb_operator_resources_arangodeployment_immutable_errors](./arangodb_operator_resources_arangodeployment_immutable_errors.md)  | arangodb_operator |  resources   | Counter | Counter for deployment immutable errors                                               |
+|   [arangodb_operator_resources_arangodeployment_status_restores](./arangodb_operator_resources_arangodeployment_status_restores.md)   | arangodb_operator |  resources   | Counter | Counter for deployment status restored                                                |
 |          [arangodb_operator_resources_arangodeployment_uptodate](./arangodb_operator_resources_arangodeployment_uptodate.md)          | arangodb_operator |  resources   |  Gauge  | Defines if ArangoDeployment is uptodate                                               |
 | [arangodb_operator_resources_arangodeployment_validation_errors](./arangodb_operator_resources_arangodeployment_validation_errors.md) | arangodb_operator |  resources   | Counter | Counter for deployment validation errors                                              |

docs/generated/metrics/arangodb_operator_resources_arangodeployment_status_restores.md

@@ -0,0 +1,12 @@
+# arangodb_operator_resources_arangodeployment_status_restores (Counter)
+
+## Description
+
+Counter for deployment status restored
+
+## Labels
+
+|   Label   | Description          |
+|:---------:|:---------------------|
+| namespace | Deployment Namespace |
+|   name    | Deployment Name      |

internal/metrics.yaml

@@ -150,6 +150,15 @@ namespaces:
           - key: name
             description: "Deployment Name"
     resources:
+      arangodeployment_status_restores:
+        shortDescription: "Counter for deployment status restored"
+        description: "Counter for deployment status restored"
+        type: "Counter"
+        labels:
+          - key: namespace
+            description: "Deployment Namespace"
+          - key: name
+            description: "Deployment Name"
       arangodeployment_validation_errors:
         shortDescription: "Counter for deployment validation errors"
         description: "Counter for deployment validation errors"

pkg/apis/deployment/v1/member_status.go

@@ -77,8 +77,9 @@ type MemberStatus struct {
 	// Endpoint definition how member should be reachable
 	Endpoint *string `json:"endpoint,omitempty"`
 	// Topology define topology member status assignment
-	Topology *TopologyMemberStatus `json:"topology,omitempty"`
-	Pod      *MemberPodStatus      `json:"pod,omitempty"`
+	Topology     *TopologyMemberStatus `json:"topology,omitempty"`
+	Pod          *MemberPodStatus      `json:"pod,omitempty"`
+	SecondaryPod *MemberPodStatus      `json:"secondaryPod,omitempty"`
 
 	// deprecated
 	// SideCarSpecs contains list of specifications specified for side cars
@@ -104,6 +105,7 @@ func (s MemberStatus) Equal(other MemberStatus) bool {
 		util.TimeCompareEqual(s.CreatedAt, other.CreatedAt) &&
 		s.PersistentVolumeClaimName == other.PersistentVolumeClaimName &&
 		s.Pod.Equal(other.Pod) &&
+		s.SecondaryPod.Equal(other.SecondaryPod) &&
 		s.Conditions.Equal(other.Conditions) &&
 		s.IsInitialized == other.IsInitialized &&
 		s.CleanoutJobID == other.CleanoutJobID &&

pkg/apis/deployment/v2alpha1/member_status.go

@@ -77,8 +77,9 @@ type MemberStatus struct {
 	// Endpoint definition how member should be reachable
 	Endpoint *string `json:"endpoint,omitempty"`
 	// Topology define topology member status assignment
-	Topology *TopologyMemberStatus `json:"topology,omitempty"`
-	Pod      *MemberPodStatus      `json:"pod,omitempty"`
+	Topology     *TopologyMemberStatus `json:"topology,omitempty"`
+	Pod          *MemberPodStatus      `json:"pod,omitempty"`
+	SecondaryPod *MemberPodStatus      `json:"secondaryPod,omitempty"`
 
 	// deprecated
 	// SideCarSpecs contains list of specifications specified for side cars
@@ -104,6 +105,7 @@ func (s MemberStatus) Equal(other MemberStatus) bool {
 		util.TimeCompareEqual(s.CreatedAt, other.CreatedAt) &&
 		s.PersistentVolumeClaimName == other.PersistentVolumeClaimName &&
 		s.Pod.Equal(other.Pod) &&
+		s.SecondaryPod.Equal(other.SecondaryPod) &&
 		s.Conditions.Equal(other.Conditions) &&
 		s.IsInitialized == other.IsInitialized &&
 		s.CleanoutJobID == other.CleanoutJobID &&

pkg/deployment/deployment_inspector.go

@@ -114,12 +114,26 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 
 		// Ensure that status is up to date
 		if !d.currentObjectStatus.Equal(updated.Status) {
+			d.metrics.Errors.StatusRestores++
 			if err := d.updateCRStatus(ctxReconciliation, *d.currentObjectStatus); err != nil {
 				d.log.Err(err).Error("Unable to refresh status")
 				return minInspectionInterval // Retry ASAP
 			}
 		}
 
+		// Ensure that fields are recovered
+		currentStatus := d.GetStatus()
+		if updated, err := RecoverStatus(&currentStatus, RecoverPodDetails); err != nil {
+			d.log.Err(err).Error("Unable to recover status")
+			return minInspectionInterval // Retry ASAP
+		} else if updated {
+			d.metrics.Errors.StatusRestores++
+			if err := d.updateCRStatus(ctxReconciliation, currentStatus); err != nil {
+				d.log.Err(err).Error("Unable to refresh status")
+				return minInspectionInterval // Retry ASAP
+			}
+		}
+
 		d.currentObject = updated
 
 		d.metrics.Deployment.Accepted = updated.Status.Conditions.IsTrue(api.ConditionTypeSpecAccepted)

pkg/deployment/deployment_status_pod_details_recovery.go

@@ -0,0 +1,45 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package deployment
+
+import api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+
+func RecoverPodDetails(in *api.DeploymentStatus) (changed bool, _ error) {
+	changed = false
+	for _, m := range in.Members.AsList() {
+		if m.Member.Pod == nil {
+			// Pod is nil, recovery might be needed
+			if m.Member.PodName != "" {
+				m.Member.Pod = &api.MemberPodStatus{
+					Name:        m.Member.PodName,
+					UID:         m.Member.PodUID,
+					SpecVersion: m.Member.PodSpecVersion,
+				}
+
+				if err := in.Members.Update(m.Member, m.Group); err != nil {
+					return false, err
+				}
+				changed = true
+			}
+		}
+	}
+	return
+}

pkg/deployment/deployment_status_recovery.go

@@ -0,0 +1,39 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package deployment
+
+import api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+
+type RecoverStatusFunc func(in *api.DeploymentStatus) (bool, error)
+
+func RecoverStatus(in *api.DeploymentStatus, fs ...RecoverStatusFunc) (bool, error) {
+	var changed bool
+	for _, f := range fs {
+		if f != nil {
+			if c, err := f(in); err != nil {
+				return false, err
+			} else if c {
+				changed = true
+			}
+		}
+	}
+	return changed, nil
+}

pkg/deployment/metrics.go

@@ -33,7 +33,7 @@ type Metrics struct {
 	}
 
 	Errors struct {
-		DeploymentValidationErrors, DeploymentImmutableErrors uint64
+		DeploymentValidationErrors, DeploymentImmutableErrors, StatusRestores uint64
 	}
 
 	Deployment struct {
@@ -48,6 +48,7 @@ func (d *Deployment) CollectMetrics(m metrics.PushMetric) {
 
 	m.Push(metric_descriptions.ArangodbOperatorResourcesArangodeploymentValidationErrorsCounter(float64(d.metrics.Errors.DeploymentValidationErrors), d.namespace, d.name))
 	m.Push(metric_descriptions.ArangodbOperatorResourcesArangodeploymentImmutableErrorsCounter(float64(d.metrics.Errors.DeploymentImmutableErrors), d.namespace, d.name))
+	m.Push(metric_descriptions.ArangodbOperatorResourcesArangodeploymentStatusRestoresCounter(float64(d.metrics.Errors.StatusRestores), d.namespace, d.name))
 
 	if d.metrics.Deployment.Accepted {
 		m.Push(metric_descriptions.ArangodbOperatorResourcesArangodeploymentAcceptedGauge(1, d.namespace, d.name))

pkg/deployment/reconcile/plan_builder_utils.go

@@ -98,6 +98,7 @@ func removeMemberConditionActionV2(actionReason string, conditionType api.Condit
 		AddParam(setConditionActionV2KeyType, setConditionActionV2KeyTypeRemove)
 }
 
+//nolint:unparam
 func updateMemberConditionActionV2(actionReason string, conditionType api.ConditionType, group api.ServerGroup, member string, status bool, reason, message, hash string) api.Action {
 	statusBool := core.ConditionTrue
 	if !status {