Skip to content

Commit

Permalink
Merge pull request #165 from arangodb/bugfix/deployment-finalizer
Browse files Browse the repository at this point in the history
Added finalizer on deployment, used to remove child finalizers on delete
  • Loading branch information
ewoutp authored Jun 8, 2018
2 parents c68289f + df09665 commit d12ad72
Show file tree
Hide file tree
Showing 4 changed files with 219 additions and 91 deletions.
10 changes: 7 additions & 3 deletions pkg/deployment/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,13 +345,17 @@ func (d *Deployment) updateCRStatus(force ...bool) error {
}

// Send update to API server
ns := d.apiObject.GetNamespace()
depls := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns)
update := d.apiObject.DeepCopy()
attempt := 0
for {
attempt++
update.Status = d.status
ns := d.apiObject.GetNamespace()
newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Update(update)
if update.GetDeletionTimestamp() == nil {
ensureFinalizers(update)
}
newAPIObject, err := depls.Update(update)
if err == nil {
// Update internal object
d.apiObject = newAPIObject
Expand All @@ -361,7 +365,7 @@ func (d *Deployment) updateCRStatus(force ...bool) error {
// API object may have been changed already,
// Reload api object and try again
var current *api.ArangoDeployment
current, err = d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Get(update.GetName(), metav1.GetOptions{})
current, err = depls.Get(update.GetName(), metav1.GetOptions{})
if err == nil {
update = current.DeepCopy()
continue
Expand Down
116 changes: 116 additions & 0 deletions pkg/deployment/deployment_finalizers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package deployment

import (
"context"

"github.com/rs/zerolog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)

// ensureFinalizers adds all required finalizers to the given deployment (in memory).
func ensureFinalizers(depl *api.ArangoDeployment) {
for _, f := range depl.GetFinalizers() {
if f == constants.FinalizerDeplRemoveChildFinalizers {
// Finalizer already set
return
}
}
// Set finalizers
depl.SetFinalizers(append(depl.GetFinalizers(), constants.FinalizerDeplRemoveChildFinalizers))
}

// runDeploymentFinalizers goes through the list of ArangoDeployoment finalizers to see if they can be removed.
func (d *Deployment) runDeploymentFinalizers(ctx context.Context) error {
log := d.deps.Log
var removalList []string

depls := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.GetNamespace())
updated, err := depls.Get(d.apiObject.GetName(), metav1.GetOptions{})
if err != nil {
return maskAny(err)
}
for _, f := range updated.ObjectMeta.GetFinalizers() {
switch f {
case constants.FinalizerDeplRemoveChildFinalizers:
log.Debug().Msg("Inspecting 'remove child finalizers' finalizer")
if err := d.inspectRemoveChildFinalizers(ctx, log, updated); err == nil {
removalList = append(removalList, f)
} else {
log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet")
}
}
}
// Remove finalizers (if needed)
if len(removalList) > 0 {
if err := removeDeploymentFinalizers(log, d.deps.DatabaseCRCli, updated, removalList); err != nil {
log.Debug().Err(err).Msg("Failed to update ArangoDeployment (to remove finalizers)")
return maskAny(err)
}
}
return nil
}

// inspectRemoveChildFinalizers checks the finalizer condition for remove-child-finalizers.
// It returns nil if the finalizer can be removed.
func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerolog.Logger, depl *api.ArangoDeployment) error {
if err := d.removePodFinalizers(); err != nil {
return maskAny(err)
}
if err := d.removePVCFinalizers(); err != nil {
return maskAny(err)
}

return nil
}

// removeDeploymentFinalizers removes the given finalizers from the given PVC.
func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, depl *api.ArangoDeployment, finalizers []string) error {
depls := cli.DatabaseV1alpha().ArangoDeployments(depl.GetNamespace())
getFunc := func() (metav1.Object, error) {
result, err := depls.Get(depl.GetName(), metav1.GetOptions{})
if err != nil {
return nil, maskAny(err)
}
return result, nil
}
updateFunc := func(updated metav1.Object) error {
updatedDepl := updated.(*api.ArangoDeployment)
result, err := depls.Update(updatedDepl)
if err != nil {
return maskAny(err)
}
*depl = *result
return nil
}
if err := k8sutil.RemoveFinalizers(log, finalizers, getFunc, updateFunc); err != nil {
return maskAny(err)
}
return nil
}
175 changes: 91 additions & 84 deletions pkg/deployment/deployment_inspector.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,105 +46,112 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration
ctx := context.Background()

// Check deployment still exists
if _, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) {
updated, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{})
if k8sutil.IsNotFound(err) {
// Deployment is gone
log.Info().Msg("Deployment is gone")
d.Delete()
return nextInterval
}

// Is the deployment in failed state, if so, give up.
if d.status.Phase == api.DeploymentPhaseFailed {
log.Debug().Msg("Deployment is in Failed state.")
return nextInterval
}
} else if updated != nil && updated.GetDeletionTimestamp() != nil {
// Deployment is marked for deletion
if err := d.runDeploymentFinalizers(ctx); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("ArangoDeployment finalizer inspection failed", err, d.apiObject))
}
} else {
// Is the deployment in failed state, if so, give up.
if d.status.Phase == api.DeploymentPhaseFailed {
log.Debug().Msg("Deployment is in Failed state.")
return nextInterval
}

// Inspect secret hashes
if err := d.resources.ValidateSecretHashes(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Secret hash validation failed", err, d.apiObject))
}
// Inspect secret hashes
if err := d.resources.ValidateSecretHashes(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Secret hash validation failed", err, d.apiObject))
}

// Is the deployment in a good state?
if d.status.Conditions.IsTrue(api.ConditionTypeSecretsChanged) {
log.Debug().Msg("Condition SecretsChanged is true. Revert secrets before we can continue")
return nextInterval
}
// Is the deployment in a good state?
if d.status.Conditions.IsTrue(api.ConditionTypeSecretsChanged) {
log.Debug().Msg("Condition SecretsChanged is true. Revert secrets before we can continue")
return nextInterval
}

// Ensure we have image info
if retrySoon, err := d.ensureImages(d.apiObject); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Image detection failed", err, d.apiObject))
} else if retrySoon {
nextInterval = minInspectionInterval
}
// Ensure we have image info
if retrySoon, err := d.ensureImages(d.apiObject); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Image detection failed", err, d.apiObject))
} else if retrySoon {
nextInterval = minInspectionInterval
}

// Inspection of generated resources needed
if err := d.resources.InspectPods(ctx); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Pod inspection failed", err, d.apiObject))
}
if err := d.resources.InspectPVCs(ctx); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("PVC inspection failed", err, d.apiObject))
}
// Inspection of generated resources needed
if err := d.resources.InspectPods(ctx); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Pod inspection failed", err, d.apiObject))
}
if err := d.resources.InspectPVCs(ctx); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("PVC inspection failed", err, d.apiObject))
}

// Check members for resilience
if err := d.resilience.CheckMemberFailure(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Member failure detection failed", err, d.apiObject))
}
// Check members for resilience
if err := d.resilience.CheckMemberFailure(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Member failure detection failed", err, d.apiObject))
}

// Create scale/update plan
if err := d.reconciler.CreatePlan(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Plan creation failed", err, d.apiObject))
}
// Create scale/update plan
if err := d.reconciler.CreatePlan(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Plan creation failed", err, d.apiObject))
}

// Execute current step of scale/update plan
retrySoon, err := d.reconciler.ExecutePlan(ctx)
if err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Plan execution failed", err, d.apiObject))
}
if retrySoon {
nextInterval = minInspectionInterval
}
// Execute current step of scale/update plan
retrySoon, err := d.reconciler.ExecutePlan(ctx)
if err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Plan execution failed", err, d.apiObject))
}
if retrySoon {
nextInterval = minInspectionInterval
}

// Ensure all resources are created
if err := d.resources.EnsureSecrets(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Secret creation failed", err, d.apiObject))
}
if err := d.resources.EnsureServices(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Service creation failed", err, d.apiObject))
}
if err := d.resources.EnsurePVCs(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("PVC creation failed", err, d.apiObject))
}
if err := d.resources.EnsurePods(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Pod creation failed", err, d.apiObject))
}
// Ensure all resources are created
if err := d.resources.EnsureSecrets(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Secret creation failed", err, d.apiObject))
}
if err := d.resources.EnsureServices(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Service creation failed", err, d.apiObject))
}
if err := d.resources.EnsurePVCs(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("PVC creation failed", err, d.apiObject))
}
if err := d.resources.EnsurePods(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Pod creation failed", err, d.apiObject))
}

// Create access packages
if err := d.createAccessPackages(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("AccessPackage creation failed", err, d.apiObject))
}
// Create access packages
if err := d.createAccessPackages(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("AccessPackage creation failed", err, d.apiObject))
}

// Inspect deployment for obsolete members
if err := d.resources.CleanupRemovedMembers(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Removed member cleanup failed", err, d.apiObject))
}
// Inspect deployment for obsolete members
if err := d.resources.CleanupRemovedMembers(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Removed member cleanup failed", err, d.apiObject))
}

// At the end of the inspect, we cleanup terminated pods.
if err := d.resources.CleanupTerminatedPods(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Pod cleanup failed", err, d.apiObject))
// At the end of the inspect, we cleanup terminated pods.
if err := d.resources.CleanupTerminatedPods(); err != nil {
hasError = true
d.CreateEvent(k8sutil.NewErrorEvent("Pod cleanup failed", err, d.apiObject))
}
}

// Update next interval (on errors)
Expand Down
9 changes: 5 additions & 4 deletions pkg/util/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,11 @@ const (

SecretAccessPackageYaml = "accessPackage.yaml" // Key in Secret.data used to store a YAML encoded access package

FinalizerPodDrainDBServer = "dbserver.database.arangodb.com/drain" // Finalizer added to DBServers, indicating the need for draining that dbserver
FinalizerPodAgencyServing = "agent.database.arangodb.com/agency-serving" // Finalizer added to Agents, indicating the need for keeping enough agents alive
FinalizerPVCMemberExists = "pvc.database.arangodb.com/member-exists" // Finalizer added to PVCs, indicating the need to keep is as long as its member exists
FinalizerDeplReplStopSync = "replication.database.arangodb.com/stop-sync" // Finalizer added to ArangoDeploymentReplication, indicating the need to stop synchronization
FinalizerDeplRemoveChildFinalizers = "database.arangodb.com/remove-child-finalizers" // Finalizer added to ArangoDeployment, indicating the need to remove finalizers from all children
FinalizerDeplReplStopSync = "replication.database.arangodb.com/stop-sync" // Finalizer added to ArangoDeploymentReplication, indicating the need to stop synchronization
FinalizerPodAgencyServing = "agent.database.arangodb.com/agency-serving" // Finalizer added to Agents, indicating the need for keeping enough agents alive
FinalizerPodDrainDBServer = "dbserver.database.arangodb.com/drain" // Finalizer added to DBServers, indicating the need for draining that dbserver
FinalizerPVCMemberExists = "pvc.database.arangodb.com/member-exists" // Finalizer added to PVCs, indicating the need to keep is as long as its member exists

AnnotationEnforceAntiAffinity = "database.arangodb.com/enforce-anti-affinity" // Key of annotation added to PVC. Value is a boolean "true" or "false"
)

0 comments on commit d12ad72

Please sign in to comment.