From 42ea6c58078666b1e7d94436b96a02284d4fe4d9 Mon Sep 17 00:00:00 2001
From: Enric Tobella <etobella@creublanca.es>
Date: Wed, 6 Apr 2022 13:29:49 +0200
Subject: [PATCH] [IMP] queue_job: Recover gracefully when the CPU limit is
 reached

---
 queue_job/jobrunner/__init__.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py
index 9de83aeeb6..eaafc480bf 100644
--- a/queue_job/jobrunner/__init__.py
+++ b/queue_job/jobrunner/__init__.py
@@ -54,6 +54,7 @@ def __init__(self, multi):
         super(WorkerJobRunner, self).__init__(multi)
         self.watchdog_timeout = None
         self.runner = QueueJobRunner.from_environ_or_config()
+        self._recover = False
 
     def sleep(self):
         pass
@@ -64,10 +65,23 @@ def signal_handler(self, sig, frame):
         self.runner.stop()
 
     def process_work(self):
+        if self._recover:
+            _logger.info("WorkerJobRunner (%s) runner is reinitialized", self.pid)
+            self.runner = QueueJobRunner.from_environ_or_config()
+            self._recover = False
         _logger.debug("WorkerJobRunner (%s) starting up", self.pid)
         time.sleep(START_DELAY)
         self.runner.run()
 
+    def signal_time_expired_handler(self, n, stack):
+        _logger.info(
+            "Worker (%d) CPU time limit (%s) reached.Stop gracefully and recover",
+            self.pid,
+            config["limit_time_cpu"],
+        )
+        self._recover = True
+        self.runner.stop()
+
 
 runner_thread = None