Skip to content

Commit

Permalink
do not error out on leader lost
Browse files Browse the repository at this point in the history
Signed-off-by: Jakub Dyszkiewicz <[email protected]>
  • Loading branch information
jakubdyszkiewicz committed Dec 19, 2023
1 parent c5827ab commit d5b0a09
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 20 deletions.
2 changes: 1 addition & 1 deletion docs/generated/kuma-cp.md
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ multizone:
responseBackoff: 0s # ENV: KUMA_MULTIZONE_GLOBAL_KDS_RESPONSE_BACKOFF
zone:
# Kuma Zone name used to mark the zone dataplane resources
name: "" # ENV: KUMA_MULTIZONE_ZONE_NAME
name: "default" # ENV: KUMA_MULTIZONE_ZONE_NAME
# GlobalAddress URL of Global Kuma CP
globalAddress: # ENV KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS
kds:
Expand Down
2 changes: 1 addition & 1 deletion docs/generated/raw/kuma-cp.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ multizone:
responseBackoff: 0s # ENV: KUMA_MULTIZONE_GLOBAL_KDS_RESPONSE_BACKOFF
zone:
# Kuma Zone name used to mark the zone dataplane resources
name: "" # ENV: KUMA_MULTIZONE_ZONE_NAME
name: "default" # ENV: KUMA_MULTIZONE_ZONE_NAME
# GlobalAddress URL of Global Kuma CP
globalAddress: # ENV KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS
kds:
Expand Down
23 changes: 15 additions & 8 deletions pkg/defaults/envoy_admin_ca.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,24 @@ var _ component.Component = &EnvoyAdminCaDefaultComponent{}

func (e *EnvoyAdminCaDefaultComponent) Start(stop <-chan struct{}) error {
ctx, cancelFn := context.WithCancel(user.Ctx(context.Background(), user.ControlPlane))
defer cancelFn()
logger := kuma_log.AddFieldsFromCtx(log, ctx, e.Extensions)
errChan := make(chan error)
go func() {
<-stop
cancelFn()
errChan <- retry.Do(ctx, retry.WithMaxDuration(10*time.Minute, retry.NewConstant(5*time.Second)), func(ctx context.Context) error {
if err := EnsureEnvoyAdminCaExist(ctx, e.ResManager, e.Extensions); err != nil {
logger.V(1).Info("could not ensure that Envoy Admin CA exists. Retrying.", "err", err)
return retry.RetryableError(err)
}
return nil
})
}()
return retry.Do(ctx, retry.WithMaxDuration(10*time.Minute, retry.NewConstant(5*time.Second)), func(ctx context.Context) error {
if err := EnsureEnvoyAdminCaExist(ctx, e.ResManager, e.Extensions); err != nil {
log.V(1).Info("could not ensure that Envoy Admin CA exists. Retrying.", "err", err)
return retry.RetryableError(err)
}
select {
case <-stop:
return nil
})
case err := <-errChan:
return err
}
}

func (e EnvoyAdminCaDefaultComponent) NeedLeaderElection() bool {
Expand Down
24 changes: 15 additions & 9 deletions pkg/defaults/zone.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,24 @@ var _ component.Component = &ZoneDefaultComponent{}

func (e *ZoneDefaultComponent) Start(stop <-chan struct{}) error {
ctx, cancelFn := context.WithCancel(user.Ctx(context.Background(), user.ControlPlane))
defer cancelFn()
logger := kuma_log.AddFieldsFromCtx(log, ctx, e.Extensions)
errChan := make(chan error)
go func() {
<-stop
cancelFn()
errChan <- retry.Do(ctx, retry.WithMaxDuration(10*time.Minute, retry.NewConstant(5*time.Second)), func(ctx context.Context) error {
if err := EnsureOnlyOneZoneExists(ctx, e.ResManager, e.ZoneName, logger); err != nil {
log.V(1).Info("could not ensure that Zone exists. Retrying.", "err", err)
return retry.RetryableError(err)
}
return nil
})
}()
return retry.Do(ctx, retry.WithMaxDuration(10*time.Minute, retry.NewConstant(5*time.Second)), func(ctx context.Context) error {
logger := kuma_log.AddFieldsFromCtx(log, ctx, e.Extensions)
if err := EnsureOnlyOneZoneExists(ctx, e.ResManager, e.ZoneName, logger); err != nil {
log.V(1).Info("could not ensure that Zone exists. Retrying.", "err", err)
return retry.RetryableError(err)
}
select {
case <-stop:
return nil
})
case err := <-errChan:
return err
}
}

func (e ZoneDefaultComponent) NeedLeaderElection() bool {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func LeaderElectionPostgres() {
Expect(metrics).To(ContainSubstring(`leader{zone="kuma-leader2"} 0`))

// when CP 1 is killed
_, _, err = zone1.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
_, _, err = zone1.Exec("", "", AppModeCP, "pkill", "kuma-cp")
Expect(err).ToNot(HaveOccurred())

// then CP 2 is leader
Expand Down

0 comments on commit d5b0a09

Please sign in to comment.