From 53a0e2e8518d288498904cd6ae3740d0bb57ca52 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 15 Jun 2023 08:56:35 -0400 Subject: [PATCH] adjust prioritized client updates In #17354 we made client updates prioritized to reduce client-to-server traffic. When the client has no previously-acknowledged update we assume that the update is of typical priority; although we don't know that for sure in practice an allocation will never become healthy quickly enough that the first update we send is the update saying the alloc is healthy. But that doesn't account for allocations that quickly fail in an unrecoverable way because of allocrunner hook failures, and it'd be nice to be able to send those failure states to the server more quickly. This changeset does so and adds some extra comments on reasoning behind priority. --- client/allocrunner/alloc_runner.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index 04de68ab442..d81cedd68dc 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -1422,14 +1422,18 @@ func (ar *allocRunner) persistLastAcknowledgedState(a *state.State) { // GetUpdatePriority returns the update priority based the difference between // the current state and the state that was last acknowledged from a server -// update. This is called from the client in the same goroutine that called -// AcknowledgeState so that we can't get a TOCTOU error. +// update, returning urgent priority when the update is critical to marking +// allocations for rescheduling. This is called from the client in the same +// goroutine that called AcknowledgeState so that we can't get a TOCTOU error. func (ar *allocRunner) GetUpdatePriority(a *structs.Allocation) cstructs.AllocUpdatePriority { ar.stateLock.RLock() defer ar.stateLock.RUnlock() last := ar.lastAcknowledgedState if last == nil { + if a.ClientStatus == structs.AllocClientStatusFailed { + return cstructs.AllocUpdatePriorityUrgent + } return cstructs.AllocUpdatePriorityTypical } @@ -1439,6 +1443,9 @@ func (ar *allocRunner) GetUpdatePriority(a *structs.Allocation) cstructs.AllocUp case last.ClientDescription != a.ClientDescription: return cstructs.AllocUpdatePriorityTypical case !last.DeploymentStatus.Equal(a.DeploymentStatus): + // TODO: this field gates deployment progress, so we may consider + // returning urgent here; right now urgent updates are primarily focused + // on recovering from failure return cstructs.AllocUpdatePriorityTypical case !last.NetworkStatus.Equal(a.NetworkStatus): return cstructs.AllocUpdatePriorityTypical