From 00b78123c07dd8f43b6a038bd2335387b224cac6 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 3 Apr 2018 17:43:42 +0200 Subject: [PATCH 1/3] Quickly fail when deployment no longer exists --- pkg/deployment/deployment_inspector.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 9ce1c848e..c4c9f3063 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -28,6 +28,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // inspectDeployment inspects the entire deployment, creates @@ -44,6 +45,14 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration hasError := false ctx := context.Background() + // Check deployment still exists + if _, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) { + // Deployment is gone + log.Info().Msg("Deployment is gone") + d.Delete() + return nextInterval + } + // Is the deployment in failed state, if so, give up. if d.status.Phase == api.DeploymentPhaseFailed { log.Debug().Msg("Deployment is in Failed state.") From 58c08784717509f8fe689d268963e480c0123f63 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 5 Apr 2018 08:45:35 +0200 Subject: [PATCH 2/3] Prevent panic on duplicate Delete --- pkg/deployment/deployment.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 0f394237d..72adb4bf6 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -25,6 +25,7 @@ package deployment import ( "fmt" "reflect" + "sync/atomic" "time" "github.com/rs/zerolog" @@ -86,6 +87,7 @@ type Deployment struct { eventCh chan *deploymentEvent stopCh chan struct{} + stopped int32 eventsCli corev1.EventInterface @@ -154,7 +156,9 @@ func (d *Deployment) Update(apiObject *api.ArangoDeployment) { // Called when the deployment was deleted by the user. func (d *Deployment) Delete() { d.deps.Log.Info().Msg("deployment is deleted by user") - close(d.stopCh) + if atomic.CompareAndSwapInt32(&d.stopped, 0, 1) { + close(d.stopCh) + } } // send given event into the deployment event queue. From 8a7e1a8603522661a66996af0dda04b9c953412f Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 5 Apr 2018 08:47:59 +0200 Subject: [PATCH 3/3] Prevent panic on duplicate Delete (storage) --- pkg/storage/local_storage.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/storage/local_storage.go b/pkg/storage/local_storage.go index 9e1c2b437..2591bda6d 100644 --- a/pkg/storage/local_storage.go +++ b/pkg/storage/local_storage.go @@ -26,6 +26,7 @@ import ( "context" "fmt" "reflect" + "sync/atomic" "time" "github.com/rs/zerolog" @@ -89,6 +90,7 @@ type LocalStorage struct { eventCh chan *localStorageEvent stopCh chan struct{} + stopped int32 eventsCli corev1.EventInterface @@ -136,7 +138,9 @@ func (ls *LocalStorage) Update(apiObject *api.ArangoLocalStorage) { // Called when the local storage was deleted by the user. func (ls *LocalStorage) Delete() { ls.deps.Log.Info().Msg("local storage is deleted by user") - close(ls.stopCh) + if atomic.CompareAndSwapInt32(&ls.stopped, 0, 1) { + close(ls.stopCh) + } } // send given event into the local storage event queue.