Return failed response and request

python-sync-actions/src/component.py

@@ -5,6 +5,7 @@
 import copy
 import logging
 from io import StringIO
+from json import JSONDecodeError
 from typing import List
 
 import requests
@@ -16,7 +17,7 @@
 from actions.mapping import infer_mapping
 from configuration import Configuration, ConfigHelpers
 from http_generic.auth import AuthMethodBuilder, AuthBuilderError
-from http_generic.client import GenericHttpClient
+from http_generic.client import GenericHttpClient, HttpClientError
 from placeholders_utils import PlaceholdersUtils
 
 MAX_CHILD_CALLS = 20
@@ -248,7 +249,7 @@ def _parse_data(self, data, path) -> list:
 
         return result
 
-    def make_call(self) -> tuple[list, any, str]:
+    def make_call(self) -> tuple[list, any, str, str]:
         """
         Make call to the API
         Returns:
@@ -326,13 +327,21 @@ def recursive_call(parent_result, config_index=0):
                 else:
                     recursive_call(current_results, config_index + 1)
 
-        recursive_call({})
+        try:
+            recursive_call({})
+            error_message = ''
+        except HttpClientError as e:
+            error_message = str(e)
+            if e.response is not None:
+                self._final_response = e.response
+            else:
+                raise UserException(e) from e
 
         secrets_to_hide = self._get_values_to_hide(self._configuration.user_parameters)
         filtered_response = self._deep_copy_and_replace_words(self._final_response, secrets_to_hide)
         filtered_log = self._deep_copy_and_replace_words(self.log.getvalue(), secrets_to_hide)
 
-        return final_results, filtered_response, filtered_log
+        return final_results, filtered_response, filtered_log, error_message
 
     @sync_action('load_from_curl')
     def load_from_curl(self) -> dict:
@@ -353,7 +362,11 @@ def infer_mapping(self) -> dict:
         Load configuration from cURL command
         """
         self.init_component()
-        data, response, log = self.make_call()
+        data, response, log, error = self.make_call()
+
+        if error:
+            raise UserException(error)
+
         nesting_level = self.configuration.parameters.get('__NESTING_LEVEL', 2)
         primary_keys = self.configuration.parameters.get('__PRIMARY_KEY', [])
         parent_pkey = []
@@ -384,20 +397,31 @@ def perform_function_sync(self) -> dict:
 
     @sync_action('test_request')
     def test_request(self):
-        results, response, log = self.make_call()
+        results, response, log, error_message = self.make_call()
 
+        # TODO: UI to parse the response status code
+
+        body = None
+        if self._final_response.request.body:
+            body = self._final_response.request.body.decode('utf-8')
+
+        # get response data:
+        try:
+            response_data = self._final_response.json()
+        except JSONDecodeError:
+            response_data = self._final_response.text
         result = {
             "response": {
                 "status_code": self._final_response.status_code,
                 "reason": self._final_response.reason,
                 "headers": dict(self._final_response.headers),
-                "data": self._final_response.json()
+                "data": response_data
             },
             "request": {
                 "url": self._final_response.request.url,
                 "method": self._final_response.request.method,
                 "headers": dict(self._final_response.request.headers),
-                "data": self._final_response.request.body
+                "data": body
             },
             "records": results,
             "debug_log": log

python-sync-actions/src/http_generic/client.py

@@ -1,7 +1,6 @@
 from typing import Tuple, Dict
 
 import requests
-from keboola.component import UserException
 from keboola.http_client import HttpClient
 from requests.adapters import HTTPAdapter
 from requests.exceptions import HTTPError, InvalidJSONError, ConnectionError
@@ -10,6 +9,12 @@
 from http_generic.auth import AuthMethodBase
 
 
+class HttpClientError(Exception):
+    def __init__(self, message, response=None):
+        self.response = response
+        super().__init__(message)
+
+
 # TODO: add support for pagination methods
 class GenericHttpClient(HttpClient):
 
@@ -48,15 +53,14 @@ def send_request(self, method, endpoint_path, **kwargs):
             else:
                 message = f'Request "{method}: {endpoint_path}" failed with non-retryable error. ' \
                           f'Status Code: {e.response.status_code}. Response: {e.response.text}'
-            raise UserException(message) from e
+            raise HttpClientError(message, resp) from e
         except InvalidJSONError:
             message = f'Request "{method}: {endpoint_path}" failed. The JSON payload is invalid (more in detail). ' \
                       f'Verify the datatype conversion.'
-            data = kwargs.get('data') or kwargs.get('json')
-            raise UserException(message, data)
+            raise HttpClientError(message, resp)
         except ConnectionError as e:
             message = f'Request "{method}: {endpoint_path}" failed with the following error: {e}'
-            raise UserException(message) from e
+            raise HttpClientError(message, resp) from e
 
     def build_url(self, base_url, endpoint_path):
         self.base_url = base_url

python-sync-actions/tests/data_tests/test_005_post/config.json

@@ -0,0 +1,25 @@
+{
+  "parameters": {
+    "api": {
+      "baseUrl": "http://private-834388-extractormock.apiary-mock.com",
+      "pagination": {
+        "method": "response.url",
+        "urlKey": "next"
+      }
+    },
+    "config": {
+      "outputBucket": "getPost",
+      "jobs": [
+        {
+          "endpoint": "post",
+          "method": "POST",
+          "params": {
+            "parameter": "value"
+          }
+        }
+      ]
+    },
+    "__SELECTED_JOB": "0"
+  },
+  "action": "test_request"
+}

python-sync-actions/tests/data_tests/test_006_post_fail/config.json

@@ -0,0 +1,25 @@
+{
+  "parameters": {
+    "api": {
+      "baseUrl": "http://private-834388-extractormock.apiary-mock.com",
+      "pagination": {
+        "method": "response.url",
+        "urlKey": "next"
+      }
+    },
+    "config": {
+      "outputBucket": "getPost",
+      "jobs": [
+        {
+          "endpoint": "nonexistent",
+          "method": "POST",
+          "params": {
+            "parameter": "value"
+          }
+        }
+      ]
+    },
+    "__SELECTED_JOB": "0"
+  },
+  "action": "test_request"
+}

python-sync-actions/tests/test_component.py

@@ -51,6 +51,24 @@ def test_004_oauth_cc_post(self):
         self.assertEqual(output['response']['data'], expected_data)
         self.assertTrue(output['request']['headers']['Authorization'].startswith('Bearer '))
 
+    def test_005_post(self):
+        component = self._get_test_component(self._testMethodName)
+        output = component.test_request()
+        expected_data = [{'id': '123', 'status': 'post'}, {'id': 'potato', 'status': 'mashed'}]
+        self.assertEqual(output['response']['data'], expected_data)
+        expected_request_data = '{"parameter": "value"}'
+        self.assertEqual(output['request']['data'], expected_request_data)
+
+    def test_006_post_fail(self):
+        component = self._get_test_component(self._testMethodName)
+        output = component.test_request()
+
+        self.assertEqual(output['response']['status_code'], 404)
+        self.assertEqual(output['response']['reason'], 'Not Found')
+
+        expected_request_data = '{"parameter": "value"}'
+        self.assertEqual(output['request']['data'], expected_request_data)
+
 
 if __name__ == '__main__':
     unittest.main()