From 4bd112f803be43619c1840780c794169a0a9ef15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nahuel=20Unai=20Rosell=C3=B3=20Beneitez?=
 <nbeneitez@apptek.com>
Date: Mon, 19 Aug 2024 04:50:42 -0400
Subject: [PATCH] Throw CalledProcessError -> STATE_QUEUE_ERROR on bad return
 values

---
 sisyphus/aws_batch_engine.py                             | 5 ++++-
 sisyphus/load_sharing_facility_engine.py                 | 9 +++++----
 ...imple_linux_utility_for_resource_management_engine.py | 9 +++++----
 sisyphus/son_of_grid_engine.py                           | 9 +++++----
 4 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/sisyphus/aws_batch_engine.py b/sisyphus/aws_batch_engine.py
index 2ad4e90..90da492 100644
--- a/sisyphus/aws_batch_engine.py
+++ b/sisyphus/aws_batch_engine.py
@@ -222,7 +222,10 @@ def task_state(self, task, task_id):
         """Return task state:"""
         name = task.task_name()
         task_name = escape_name(name, task_id)
-        queue_state = self.queue_state()
+        try:
+            queue_state = self.queue_state()
+        except subprocess.CalledProcessError:
+            return STATE_QUEUE_ERROR
         qs = queue_state.get(task_name)
 
         # task name should be uniq
diff --git a/sisyphus/load_sharing_facility_engine.py b/sisyphus/load_sharing_facility_engine.py
index 5284159..9b13bb8 100644
--- a/sisyphus/load_sharing_facility_engine.py
+++ b/sisyphus/load_sharing_facility_engine.py
@@ -255,9 +255,7 @@ def queue_state(self):
             try:
                 out, err, retval = self.system_call(system_command)
                 if retval != 0:
-                    logging.warning(self._system_call_error_warn_msg(system_command))
-                    time.sleep(gs.WAIT_PERIOD_QSTAT_PARSING)
-                    continue
+                    raise subprocess.CalledProcessError(self._system_call_error_warn_msg(system_command))
             except subprocess.TimeoutExpired:
                 logging.warning(self._system_call_timeout_warn_msg(system_command))
                 time.sleep(gs.WAIT_PERIOD_SSH_TIMEOUT)
@@ -292,7 +290,10 @@ def task_state(self, task, task_id):
         name = task.task_name()
         name = escape_name(name).encode()
         task_name = (name, task_id)
-        queue_state = self.queue_state()
+        try:
+            queue_state = self.queue_state()
+        except subprocess.CalledProcessError:
+            return STATE_QUEUE_ERROR
         qs = queue_state[task_name]
 
         # task name should be uniq
diff --git a/sisyphus/simple_linux_utility_for_resource_management_engine.py b/sisyphus/simple_linux_utility_for_resource_management_engine.py
index ff0e71a..aadec23 100644
--- a/sisyphus/simple_linux_utility_for_resource_management_engine.py
+++ b/sisyphus/simple_linux_utility_for_resource_management_engine.py
@@ -313,9 +313,7 @@ def queue_state(self):
             try:
                 out, err, retval = self.system_call(system_command)
                 if retval != 0:
-                    logging.warning(self._system_call_error_warn_msg(system_command))
-                    time.sleep(gs.WAIT_PERIOD_QSTAT_PARSING)
-                    continue
+                    raise subprocess.CalledProcessError(self._system_call_error_warn_msg(system_command))
             except subprocess.TimeoutExpired:
                 logging.warning(self._system_call_timeout_warn_msg(system_command))
                 time.sleep(gs.WAIT_PERIOD_SSH_TIMEOUT)
@@ -350,7 +348,10 @@ def task_state(self, task, task_id):
         name = task.task_name()
         name = self.process_task_name(name)
         task_name = (name, task_id)
-        queue_state = self.queue_state()
+        try:
+            queue_state = self.queue_state()
+        except subprocess.CalledProcessError:
+            return STATE_QUEUE_ERROR
         qs = queue_state[task_name]
 
         # task name should be uniq
diff --git a/sisyphus/son_of_grid_engine.py b/sisyphus/son_of_grid_engine.py
index 4a87727..5b2dce2 100644
--- a/sisyphus/son_of_grid_engine.py
+++ b/sisyphus/son_of_grid_engine.py
@@ -324,9 +324,7 @@ def queue_state(self):
             try:
                 out, err, retval = self.system_call(system_command)
                 if retval != 0:
-                    logging.warning(self._system_call_error_warn_msg(system_command))
-                    time.sleep(gs.WAIT_PERIOD_QSTAT_PARSING)
-                    continue
+                    raise subprocess.CalledProcessError(self._system_call_error_warn_msg(system_command))
             except subprocess.TimeoutExpired:
                 logging.warning(self._system_call_timeout_warn_msg(system_command))
                 time.sleep(gs.WAIT_PERIOD_SSH_TIMEOUT)
@@ -416,7 +414,10 @@ def task_state(self, task, task_id):
         name = task.task_name()
         name = escape_name(name)
         task_name = (name, task_id)
-        queue_state = self.queue_state()
+        try:
+            queue_state = self.queue_state()
+        except subprocess.CalledProcessError:
+            return STATE_QUEUE_ERROR
         qs = queue_state[task_name]
 
         # task name should be uniq