diff --git a/python-sync-actions/src/component.py b/python-sync-actions/src/component.py index 42c85fb..985d667 100644 --- a/python-sync-actions/src/component.py +++ b/python-sync-actions/src/component.py @@ -5,6 +5,7 @@ import copy import logging from io import StringIO +from json import JSONDecodeError from typing import List import requests @@ -16,7 +17,7 @@ from actions.mapping import infer_mapping from configuration import Configuration, ConfigHelpers from http_generic.auth import AuthMethodBuilder, AuthBuilderError -from http_generic.client import GenericHttpClient +from http_generic.client import GenericHttpClient, HttpClientError from placeholders_utils import PlaceholdersUtils MAX_CHILD_CALLS = 20 @@ -248,7 +249,7 @@ def _parse_data(self, data, path) -> list: return result - def make_call(self) -> tuple[list, any, str]: + def make_call(self) -> tuple[list, any, str, str]: """ Make call to the API Returns: @@ -326,13 +327,21 @@ def recursive_call(parent_result, config_index=0): else: recursive_call(current_results, config_index + 1) - recursive_call({}) + try: + recursive_call({}) + error_message = '' + except HttpClientError as e: + error_message = str(e) + if e.response is not None: + self._final_response = e.response + else: + raise UserException(e) from e secrets_to_hide = self._get_values_to_hide(self._configuration.user_parameters) filtered_response = self._deep_copy_and_replace_words(self._final_response, secrets_to_hide) filtered_log = self._deep_copy_and_replace_words(self.log.getvalue(), secrets_to_hide) - return final_results, filtered_response, filtered_log + return final_results, filtered_response, filtered_log, error_message @sync_action('load_from_curl') def load_from_curl(self) -> dict: @@ -353,7 +362,11 @@ def infer_mapping(self) -> dict: Load configuration from cURL command """ self.init_component() - data, response, log = self.make_call() + data, response, log, error = self.make_call() + + if error: + raise UserException(error) + nesting_level = self.configuration.parameters.get('__NESTING_LEVEL', 2) primary_keys = self.configuration.parameters.get('__PRIMARY_KEY', []) parent_pkey = [] @@ -384,20 +397,31 @@ def perform_function_sync(self) -> dict: @sync_action('test_request') def test_request(self): - results, response, log = self.make_call() + results, response, log, error_message = self.make_call() + # TODO: UI to parse the response status code + + body = None + if self._final_response.request.body: + body = self._final_response.request.body.decode('utf-8') + + # get response data: + try: + response_data = self._final_response.json() + except JSONDecodeError: + response_data = self._final_response.text result = { "response": { "status_code": self._final_response.status_code, "reason": self._final_response.reason, "headers": dict(self._final_response.headers), - "data": self._final_response.json() + "data": response_data }, "request": { "url": self._final_response.request.url, "method": self._final_response.request.method, "headers": dict(self._final_response.request.headers), - "data": self._final_response.request.body + "data": body }, "records": results, "debug_log": log diff --git a/python-sync-actions/src/http_generic/client.py b/python-sync-actions/src/http_generic/client.py index 205097c..e8e0e80 100644 --- a/python-sync-actions/src/http_generic/client.py +++ b/python-sync-actions/src/http_generic/client.py @@ -1,7 +1,6 @@ from typing import Tuple, Dict import requests -from keboola.component import UserException from keboola.http_client import HttpClient from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError, InvalidJSONError, ConnectionError @@ -10,6 +9,12 @@ from http_generic.auth import AuthMethodBase +class HttpClientError(Exception): + def __init__(self, message, response=None): + self.response = response + super().__init__(message) + + # TODO: add support for pagination methods class GenericHttpClient(HttpClient): @@ -48,15 +53,14 @@ def send_request(self, method, endpoint_path, **kwargs): else: message = f'Request "{method}: {endpoint_path}" failed with non-retryable error. ' \ f'Status Code: {e.response.status_code}. Response: {e.response.text}' - raise UserException(message) from e + raise HttpClientError(message, resp) from e except InvalidJSONError: message = f'Request "{method}: {endpoint_path}" failed. The JSON payload is invalid (more in detail). ' \ f'Verify the datatype conversion.' - data = kwargs.get('data') or kwargs.get('json') - raise UserException(message, data) + raise HttpClientError(message, resp) except ConnectionError as e: message = f'Request "{method}: {endpoint_path}" failed with the following error: {e}' - raise UserException(message) from e + raise HttpClientError(message, resp) from e def build_url(self, base_url, endpoint_path): self.base_url = base_url diff --git a/python-sync-actions/tests/data_tests/test_005_post/config.json b/python-sync-actions/tests/data_tests/test_005_post/config.json new file mode 100644 index 0000000..9f134db --- /dev/null +++ b/python-sync-actions/tests/data_tests/test_005_post/config.json @@ -0,0 +1,25 @@ +{ + "parameters": { + "api": { + "baseUrl": "http://private-834388-extractormock.apiary-mock.com", + "pagination": { + "method": "response.url", + "urlKey": "next" + } + }, + "config": { + "outputBucket": "getPost", + "jobs": [ + { + "endpoint": "post", + "method": "POST", + "params": { + "parameter": "value" + } + } + ] + }, + "__SELECTED_JOB": "0" + }, + "action": "test_request" +} \ No newline at end of file diff --git a/python-sync-actions/tests/data_tests/test_006_post_fail/config.json b/python-sync-actions/tests/data_tests/test_006_post_fail/config.json new file mode 100644 index 0000000..9a1371c --- /dev/null +++ b/python-sync-actions/tests/data_tests/test_006_post_fail/config.json @@ -0,0 +1,25 @@ +{ + "parameters": { + "api": { + "baseUrl": "http://private-834388-extractormock.apiary-mock.com", + "pagination": { + "method": "response.url", + "urlKey": "next" + } + }, + "config": { + "outputBucket": "getPost", + "jobs": [ + { + "endpoint": "nonexistent", + "method": "POST", + "params": { + "parameter": "value" + } + } + ] + }, + "__SELECTED_JOB": "0" + }, + "action": "test_request" +} \ No newline at end of file diff --git a/python-sync-actions/tests/test_component.py b/python-sync-actions/tests/test_component.py index 29f33bf..dd5932a 100644 --- a/python-sync-actions/tests/test_component.py +++ b/python-sync-actions/tests/test_component.py @@ -51,6 +51,24 @@ def test_004_oauth_cc_post(self): self.assertEqual(output['response']['data'], expected_data) self.assertTrue(output['request']['headers']['Authorization'].startswith('Bearer ')) + def test_005_post(self): + component = self._get_test_component(self._testMethodName) + output = component.test_request() + expected_data = [{'id': '123', 'status': 'post'}, {'id': 'potato', 'status': 'mashed'}] + self.assertEqual(output['response']['data'], expected_data) + expected_request_data = '{"parameter": "value"}' + self.assertEqual(output['request']['data'], expected_request_data) + + def test_006_post_fail(self): + component = self._get_test_component(self._testMethodName) + output = component.test_request() + + self.assertEqual(output['response']['status_code'], 404) + self.assertEqual(output['response']['reason'], 'Not Found') + + expected_request_data = '{"parameter": "value"}' + self.assertEqual(output['request']['data'], expected_request_data) + if __name__ == '__main__': unittest.main()