From 601e9c3bbf300a7ac5feb879b0d7cefdb01e1267 Mon Sep 17 00:00:00 2001
From: Peter Karman <peter@truss.works>
Date: Mon, 18 Mar 2019 10:36:54 -0500
Subject: [PATCH 1/2] Automatically restart timed tasks with delay in the past

---
 app/models/concerns/timeable_task.rb       |  5 ++++-
 spec/models/concerns/timeable_task_spec.rb | 24 +++++++++++++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/timeable_task.rb b/app/models/concerns/timeable_task.rb
index 00b7d816bb2..b50174d639a 100644
--- a/app/models/concerns/timeable_task.rb
+++ b/app/models/concerns/timeable_task.rb
@@ -9,7 +9,10 @@ def create!(args)
       fail Caseflow::Error::MissingTimerMethod unless method_defined?(:timer_ends_at)
 
       super(args).tap do |task|
-        TaskTimer.new(task: task).submit_for_processing!(delay: task.timer_ends_at)
+        timer = TaskTimer.new(task: task)
+        timer.submit_for_processing!(delay: task.timer_ends_at)
+        # if timer_ends_at is in the past, we automatically trigger processing now.
+        timer.restart! if timer.expired_without_processing?
       end
     end
   end
diff --git a/spec/models/concerns/timeable_task_spec.rb b/spec/models/concerns/timeable_task_spec.rb
index 4d18b2583f4..80c04bc3976 100644
--- a/spec/models/concerns/timeable_task_spec.rb
+++ b/spec/models/concerns/timeable_task_spec.rb
@@ -19,6 +19,16 @@ class AnotherTimedTask < GenericTask
     def timer_ends_at; end
   end
 
+  class OldTimedTask < GenericTask
+    include TimeableTask
+
+    def when_timer_ends; end
+
+    def timer_ends_at
+      NOW - 5.days
+    end
+  end
+
   before do
     Timecop.freeze(NOW)
   end
@@ -29,7 +39,19 @@ def timer_ends_at; end
     task = SomeTimedTask.create!(appeal: appeal, assigned_to: Bva.singleton)
     timers = TaskTimer.where(task: task)
     expect(timers.length).to eq(1)
-    expect(timers.first.last_submitted_at).to eq(Time.zone.now + 5.days - 3.hours + 1.minute)
+    delayed_start = Time.zone.now + 5.days - 3.hours + 1.minute
+    expect(timers.first.last_submitted_at).to eq(delayed_start)
+    expect(timers.first.submitted_at).to eq(delayed_start)
+  end
+
+  it "queues itself when the delay is in the past" do
+    task = OldTimedTask.create!(appeal: appeal, assigned_to: Bva.singleton)
+    timers = TaskTimer.where(task: task)
+    expect(timers.length).to eq(1)
+    expect(timers.first.submitted_and_ready?).to eq(true)
+    expect(timers.first.last_submitted_at).to eq(NOW)
+    delay_in_the_past = Time.zone.now - 5.days - 3.hours + 1.minute
+    expect(timers.first.submitted_at).to eq(delay_in_the_past)
   end
 
   context "when not correctly configured" do

From a3baba78eb23fae0a009678e2754174276f7be25 Mon Sep 17 00:00:00 2001
From: Peter Karman <peter@truss.works>
Date: Mon, 18 Mar 2019 10:51:58 -0500
Subject: [PATCH 2/2] Set attempted_at and error on TaskTimer

---
 app/jobs/task_timer_job.rb       | 2 ++
 spec/jobs/task_timer_job_spec.rb | 7 +++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/jobs/task_timer_job.rb b/app/jobs/task_timer_job.rb
index c00f2d96969..77e7f171c54 100644
--- a/app/jobs/task_timer_job.rb
+++ b/app/jobs/task_timer_job.rb
@@ -20,12 +20,14 @@ def process(task_timer)
     task_timer.with_lock do
       return if task_timer.processed?
 
+      task_timer.attempted!
       task_timer.task.when_timer_ends
       task_timer.processed!
     end
   rescue StandardError => e
     # Ensure errors are sent to Sentry, but don't block the job from continuing.
     # The next time the job runs, we'll process the unprocessed task timers again.
+    task_timer.update_error!(e.inspect)
     Raven.capture_exception(e)
   end
 end
diff --git a/spec/jobs/task_timer_job_spec.rb b/spec/jobs/task_timer_job_spec.rb
index f4e54bb7ff4..ea4c28dbba0 100644
--- a/spec/jobs/task_timer_job_spec.rb
+++ b/spec/jobs/task_timer_job_spec.rb
@@ -63,7 +63,10 @@ def timer_ends_at
 
     TaskTimerJob.perform_now
 
-    expect(timer.reload.processed_at).not_to eq(nil)
-    expect(error_timer.reload.processed_at).to eq(nil)
+    expect(timer.reload.processed_at).not_to be_nil
+    expect(timer.reload.attempted_at).not_to be_nil
+    expect(error_timer.reload.processed_at).to be_nil
+    expect(error_timer.error).to eq("RuntimeError")
+    expect(error_timer.attempted_at).to be_nil # because it was in a failed transaction
   end
 end