From e9c7dc1286a392834388c929dbd6b6e4764ce8b2 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Fri, 21 Sep 2018 17:10:28 -0500 Subject: [PATCH 1/3] Added logging around nacked evals in the scheduler worker --- nomad/worker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nomad/worker.go b/nomad/worker.go index 911f4ee3a40..533b6d0a44d 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -114,18 +114,23 @@ func (w *Worker) run() { // Check for a shutdown if w.srv.IsShutdown() { + w.logger.Error(fmt.Sprintf("nacking eval ID %v because the server shutdown", eval.ID)) w.sendAck(eval.ID, token, false) return } // Wait for the raft log to catchup to the evaluation if err := w.waitForIndex(waitIndex, raftSyncLimit); err != nil { + wrappedErr := fmt.Sprintf("error waiting for raft index %v:%v", waitIndex, err) + w.logger.Error(wrappedErr) w.sendAck(eval.ID, token, false) continue } // Invoke the scheduler to determine placements if err := w.invokeScheduler(eval, token); err != nil { + wrappedErr := fmt.Sprintf("error invoking scheduler:%v", err) + w.logger.Error(wrappedErr) w.sendAck(eval.ID, token, false) continue } From 47e22f6b7c470887c70b53e43002d01cb1ad8448 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Fri, 21 Sep 2018 17:28:18 -0500 Subject: [PATCH 2/3] Add failed follow up to the list of allowed eval trigger reasons needs unit test --- scheduler/generic_sched.go | 3 ++- scheduler/system_sched.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index ec9a31533a1..1094bc6a717 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -130,7 +130,8 @@ func (s *GenericScheduler) Process(eval *structs.Evaluation) error { structs.EvalTriggerNodeDrain, structs.EvalTriggerNodeUpdate, structs.EvalTriggerRollingUpdate, structs.EvalTriggerQueuedAllocs, structs.EvalTriggerPeriodicJob, structs.EvalTriggerMaxPlans, - structs.EvalTriggerDeploymentWatcher, structs.EvalTriggerRetryFailedAlloc: + structs.EvalTriggerDeploymentWatcher, structs.EvalTriggerRetryFailedAlloc, + structs.EvalTriggerFailedFollowUp: default: desc := fmt.Sprintf("scheduler cannot handle '%s' evaluation reason", eval.TriggeredBy) diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index 53131e3b425..b1a243a997f 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -60,7 +60,7 @@ func (s *SystemScheduler) Process(eval *structs.Evaluation) error { // Verify the evaluation trigger reason is understood switch eval.TriggeredBy { - case structs.EvalTriggerJobRegister, structs.EvalTriggerNodeUpdate, + case structs.EvalTriggerJobRegister, structs.EvalTriggerNodeUpdate, structs.EvalTriggerFailedFollowUp, structs.EvalTriggerJobDeregister, structs.EvalTriggerRollingUpdate, structs.EvalTriggerDeploymentWatcher, structs.EvalTriggerNodeDrain: default: From b3e85557f02c32c6692b408854d437aa6b41528f Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 25 Sep 2018 10:47:45 -0700 Subject: [PATCH 3/3] fix logging --- nomad/worker.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nomad/worker.go b/nomad/worker.go index 533b6d0a44d..75b20c1c774 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -114,23 +114,21 @@ func (w *Worker) run() { // Check for a shutdown if w.srv.IsShutdown() { - w.logger.Error(fmt.Sprintf("nacking eval ID %v because the server shutdown", eval.ID)) + w.logger.Error("nacking eval because the server is shutting down", "eval", log.Fmt("%#v", eval)) w.sendAck(eval.ID, token, false) return } // Wait for the raft log to catchup to the evaluation if err := w.waitForIndex(waitIndex, raftSyncLimit); err != nil { - wrappedErr := fmt.Sprintf("error waiting for raft index %v:%v", waitIndex, err) - w.logger.Error(wrappedErr) + w.logger.Error("error waiting for Raft index", "error", err, "index", waitIndex) w.sendAck(eval.ID, token, false) continue } // Invoke the scheduler to determine placements if err := w.invokeScheduler(eval, token); err != nil { - wrappedErr := fmt.Sprintf("error invoking scheduler:%v", err) - w.logger.Error(wrappedErr) + w.logger.Error("error invoking scheduler", "error", err) w.sendAck(eval.ID, token, false) continue }