Skip to content

Commit

Permalink
fix: #61 service start monitoring for new services
Browse files Browse the repository at this point in the history
  • Loading branch information
bohdan-shulha committed Oct 11, 2024
1 parent 7e8a165 commit ecb13d9
Showing 1 changed file with 39 additions and 6 deletions.
45 changes: 39 additions & 6 deletions internal/app/ptah-agent/service_monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ func (e *taskExecutor) monitorDaemonServiceLaunch(ctx context.Context, service *
// TODO: make timeout configurable
timeout := time.After(time.Duration(10) * time.Minute)

successfullChecks := 0

for {
select {
case <-ctx.Done():
Expand All @@ -71,13 +73,44 @@ func (e *taskExecutor) monitorDaemonServiceLaunch(ctx context.Context, service *
return err
}

log.Debug("service inspected", "state", service.UpdateStatus.State)
if service.UpdateStatus == nil {
log.Debug("service inspected, update status is nil, checking tasks")

tasks, err := e.docker.TaskList(ctx, types.TaskListOptions{
Filters: filters.NewArgs(
filters.Arg("service", service.ID),
),
})
if err != nil {
return err
}

if len(tasks) == 0 && service.Spec.Mode.Replicated.Replicas != nil && *service.Spec.Mode.Replicated.Replicas == 0 {
return nil
}

for _, t := range tasks {
if t.Status.Err != "" {
return errors.Errorf("task %s failed: %s", t.ID, t.Status.Err)
}
}

successfullChecks++

switch service.UpdateStatus.State {
case swarm.UpdateStateCompleted:
return nil
case swarm.UpdateStateRollbackCompleted:
return errors.Errorf("service update failed: %s", service.UpdateStatus.Message)
if successfullChecks >= 3 {
log.Debug("service launched", "service_id", service.ID, "successfull_checks", successfullChecks)

return nil
}
} else {
log.Debug("service inspected", "state", service.UpdateStatus.State)

switch service.UpdateStatus.State {
case swarm.UpdateStateCompleted:
return nil
case swarm.UpdateStateRollbackCompleted:
return errors.Errorf("service update failed: %s", service.UpdateStatus.Message)
}
}
}
}
Expand Down

0 comments on commit ecb13d9

Please sign in to comment.