Merge pull request #10025 from shanemcd/better-errors

Improve error handling / display when Ansible Runner errors

Before you would see unhelpful error messages like:
Traceback (most recent call last):
  File "/var/lib/awx/venv/awx/lib64/python3.8/site-packages/awx/main/tasks.py", line 1397, in run
    res = receptor_job.run()
  File "/var/lib/awx/venv/awx/lib64/python3.8/site-packages/awx/main/tasks.py", line 2957, in run
    return self._run_internal(receptor_ctl)
  File "/var/lib/awx/venv/awx/lib64/python3.8/site-packages/awx/main/tasks.py", line 3008, in _run_internal
    raise RuntimeError(detail)
RuntimeError: exit status 0

Now you will see the underlying error:
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/ansible_runner/streaming.py", line 108, in run
    utils.unstream_dir(zip_data, self.private_data_dir)
  File "/usr/local/lib/python3.8/site-packages/ansible_runner/utils.py", line 104, in unstream_dir
    data = base64.b85decode(data)
  File "/usr/lib64/python3.8/base64.py", line 463, in b85decode
    raise ValueError('bad base85 character at position %d'
ValueError: bad base85 character at position 121

Reviewed-by: Christian Adams <[email protected]>
Reviewed-by: Alan Rominger <[email protected]>
Reviewed-by: Shane McDonald <[email protected]>

awx/main/tasks.py

@@ -1279,6 +1279,10 @@ def status_handler(self, status_data, runner_config):
                 if k in job_env:
                     job_env[k] = v
             self.instance = self.update_model(self.instance.pk, job_args=json.dumps(runner_config.command), job_cwd=runner_config.cwd, job_env=job_env)
+        elif status_data['status'] == 'error':
+            result_traceback = status_data.get('result_traceback', None)
+            if result_traceback:
+                self.instance = self.update_model(self.instance.pk, result_traceback=result_traceback)
 
     def check_handler(self, config):
         """
@@ -3002,13 +3006,20 @@ def _run_internal(self, receptor_ctl):
                 # TODO: There should be a more efficient way of getting this information
                 receptor_work_list = receptor_ctl.simple_command("work list")
                 detail = receptor_work_list[self.unit_id]['Detail']
+                state_name = receptor_work_list[self.unit_id]['StateName']
+
                 if 'exceeded quota' in detail:
                     logger.warn(detail)
                     log_name = self.task.instance.log_format
                     logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.")
                     self.task.update_model(self.task.instance.pk, status='pending')
                     return
 
+                # If ansible-runner ran, but an error occured at runtime, the traceback information
+                # is saved via the status_handler passed in to the processor.
+                if state_name == 'Succeeded':
+                    return res
+
                 raise RuntimeError(detail)
 
         return res