From 616bcc40f727d0c589dd2ba8aa8337285663eae1 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Wed, 24 Nov 2021 15:46:25 -0500 Subject: [PATCH 1/3] finish the leave process if broadcasting leave timeout --- serf/serf.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/serf/serf.go b/serf/serf.go index f214a58d4..53c697878 100644 --- a/serf/serf.go +++ b/serf/serf.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/base64" "encoding/json" - "errors" "fmt" "io/ioutil" "log" @@ -727,7 +726,7 @@ func (s *Serf) Leave() error { select { case <-notifyCh: case <-time.After(s.config.BroadcastTimeout): - return errors.New("timeout while waiting for graceful leave") + s.logger.Printf("[WARN] serf: timeout while waiting for graceful leave") } } From d59a77e01fc935132eb7dc829b747e9140456215 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Mon, 10 Jan 2022 09:10:37 -0500 Subject: [PATCH 2/3] Log broadcast timeout as WARN and finish the Leave process. --- serf/serf.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/serf/serf.go b/serf/serf.go index 53c697878..7d8c278d6 100644 --- a/serf/serf.go +++ b/serf/serf.go @@ -684,6 +684,7 @@ func (s *Serf) broadcastJoin(ltime LamportTime) error { // Leave gracefully exits the cluster. It is safe to call this multiple // times. +// If the Leave broadcast timeout, Leave() will try to finish the sequence as best effort. func (s *Serf) Leave() error { // Check the current state s.stateLock.Lock() @@ -733,7 +734,7 @@ func (s *Serf) Leave() error { // Attempt the memberlist leave err := s.memberlist.Leave(s.config.BroadcastTimeout) if err != nil { - return err + s.logger.Printf("[WARN] serf: timeout waiting for leave broadcast: %s", err.Error()) } // Wait for the leave to propagate through the cluster. The broadcast From 3f940397b03a5db67bd87175a15689e8f799766f Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Wed, 12 Jan 2022 16:30:31 -0500 Subject: [PATCH 3/3] increase timeout --- serf/coalesce_member_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serf/coalesce_member_test.go b/serf/coalesce_member_test.go index ee4a2a698..ec395bfdf 100644 --- a/serf/coalesce_member_test.go +++ b/serf/coalesce_member_test.go @@ -134,7 +134,7 @@ func TestMemberEventCoalesce_TagUpdate(t *testing.T) { Members: []Member{Member{Name: "foo", Tags: map[string]string{"role": "foo"}}}, } - time.Sleep(10 * time.Millisecond) + time.Sleep(30 * time.Millisecond) select { case e := <-outCh: