Skip to content

Commit

Permalink
feat: add application sync timeout
Browse files Browse the repository at this point in the history
Signed-off-by: Zadkiel Aharonian <[email protected]>
  • Loading branch information
aslafy-z authored Sep 28, 2023
1 parent 08e53e7 commit 539d026
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 14 deletions.
8 changes: 7 additions & 1 deletion controller/appcontroller.go
Original file line number Diff line number Diff line change
Expand Up @@ -1180,7 +1180,7 @@ func (ctrl *ApplicationController) processRequestedAppOperation(app *appv1.Appli
if isOperationInProgress(app) {
state = app.Status.OperationState.DeepCopy()
terminating = state.Phase == synccommon.OperationTerminating
// Failed operation with retry strategy might have be in-progress and has completion time
// Failed operation with retry strategy might have be in-progress and has completion time
if state.FinishedAt != nil && !terminating {
retryAt, err := app.Status.OperationState.Operation.Retry.NextRetryAt(state.FinishedAt.Time, state.RetryCount)
if err != nil {
Expand All @@ -1202,6 +1202,12 @@ func (ctrl *ApplicationController) processRequestedAppOperation(app *appv1.Appli
// Get rid of sync results and null out previous operation completion time
state.SyncResult = nil
}
} else if state.Phase == synccommon.OperationRunning && app.Spec.SyncPolicy.Refresh && app.Status.Sync.Revision != state.Operation.Sync.Revision {
logCtx.Infof("A new revision is available, terminating app, was phase: %s, message: %s", state.Phase, state.Message)
state.Phase = synccommon.OperationTerminating
state.Message = "Operation forced to terminate (new revision available)"
ctrl.setOperationState(app, state)
return
} else {
logCtx.Infof("Resuming in-progress operation. phase: %s, message: %s", state.Phase, state.Message)
}
Expand Down
40 changes: 27 additions & 13 deletions pkg/apis/application/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -1094,19 +1094,8 @@ type SyncPolicy struct {
Retry *RetryStrategy `json:"retry,omitempty" protobuf:"bytes,3,opt,name=retry"`
// ManagedNamespaceMetadata controls metadata in the given namespace (if CreateNamespace=true)
ManagedNamespaceMetadata *ManagedNamespaceMetadata `json:"managedNamespaceMetadata,omitempty" protobuf:"bytes,4,opt,name=managedNamespaceMetadata"`
}

// IsZero returns true if the sync policy is empty
func (p *SyncPolicy) IsZero() bool {
return p == nil || (p.Automated == nil && len(p.SyncOptions) == 0 && p.Retry == nil)
}

// RetryStrategy contains information about the strategy to apply when a sync failed
type RetryStrategy struct {
// Limit is the maximum number of attempts for retrying a failed sync. If set to 0, no retries will be performed.
Limit int64 `json:"limit,omitempty" protobuf:"bytes,1,opt,name=limit"`
// Backoff controls how to backoff on subsequent retries of failed syncs
Backoff *Backoff `json:"backoff,omitempty" protobuf:"bytes,2,opt,name=backoff,casttype=Backoff"`
// Refresh indicates wether a new revision should trigger a new sync. Default to false
Refresh bool `json:"refresh,omitempty" protobuf:"bytes,5,opt,name=refresh"`
}

func parseStringToDuration(durationString string) (time.Duration, error) {
Expand All @@ -1122,6 +1111,31 @@ func parseStringToDuration(durationString string) (time.Duration, error) {
return suspendDuration, nil
}

// SyncTimeoutExceeded returns true if the sync timeout has been exceeded
func (p *SyncPolicy) SyncTimeoutExceeded(duration time.Duration) (bool, error) {
if p.SyncTimeout != "" {
syncTimeout, err := parseStringToDuration(p.SyncTimeout)
if err != nil {
return false, err
}
return duration > syncTimeout, nil
}
return false, nil
}

// IsZero returns true if the sync policy is empty
func (p *SyncPolicy) IsZero() bool {
return p == nil || (p.Automated == nil && len(p.SyncOptions) == 0 && p.Retry == nil)
}

// RetryStrategy contains information about the strategy to apply when a sync failed
type RetryStrategy struct {
// Limit is the maximum number of attempts for retrying a failed sync. If set to 0, no retries will be performed.
Limit int64 `json:"limit,omitempty" protobuf:"bytes,1,opt,name=limit"`
// Backoff controls how to backoff on subsequent retries of failed syncs
Backoff *Backoff `json:"backoff,omitempty" protobuf:"bytes,2,opt,name=backoff,casttype=Backoff"`
}

// NextRetryAt calculates the earliest time the next retry should be performed on a failing sync
func (r *RetryStrategy) NextRetryAt(lastAttempt time.Time, retryCounts int64) (time.Time, error) {
maxDuration := DefaultSyncRetryMaxDuration
Expand Down

0 comments on commit 539d026

Please sign in to comment.