diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index e35d441..fea5a69 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -3,13 +3,69 @@ import singer from singer import metrics, metadata, Transformer import requests +import backoff +import simplejson DATETIME_FMT = "%Y-%m-%dT%H:%M:%SZ" - session = requests.Session() logger = singer.get_logger() +class KlaviyoError(Exception): + pass + +class KlaviyoNotFoundError(KlaviyoError): + pass + +class KlaviyoBadRequestError(KlaviyoError): + pass + +class KlaviyoUnauthorizedError(KlaviyoError): + pass + +class KlaviyoForbiddenError(KlaviyoError): + pass + +class KlaviyoInternalServiceError(KlaviyoError): + pass + +ERROR_CODE_EXCEPTION_MAPPING = { + 400: { + "raise_exception": KlaviyoBadRequestError, + "message": "Request is missing or has a bad parameter." + }, + 401: { + "raise_exception": KlaviyoUnauthorizedError, + "message": "Invalid authorization credentials." + }, + 403: { + "raise_exception": KlaviyoForbiddenError, + "message": "Invalid authorization credentials or permissions." + }, + 404: { + "raise_exception": KlaviyoNotFoundError, + "message": "The requested resource doesn't exist." + }, + 500: { + "raise_exception": KlaviyoInternalServiceError, + "message": "Internal Service Error from Klaviyo." + } +} + +def raise_for_error(response): + try: + response.raise_for_status() + except requests.HTTPError as e: + try: + json_resp = response.json() + except (ValueError, TypeError, IndexError, KeyError): + json_resp = {} + + error_code = response.status_code + message_text = json_resp.get("message", ERROR_CODE_EXCEPTION_MAPPING.get(error_code, {}).get("message", "Unknown Error")) + message = "HTTP-error-code: {}, Error: {}".format(error_code, message_text) + exc = ERROR_CODE_EXCEPTION_MAPPING.get(error_code, {}).get("raise_exception", KlaviyoError) + raise exc(message) from e def dt_to_ts(dt): return int(time.mktime(datetime.datetime.strptime( @@ -50,12 +106,16 @@ def get_starting_point(stream, state, start_date): def get_latest_event_time(events): return ts_to_dt(int(events[-1]['timestamp'])) if len(events) else None - +@backoff.on_exception(backoff.expo, KlaviyoError, max_tries=3) def authed_get(source, url, params): with metrics.http_request_timer(source) as timer: resp = session.request(method='get', url=url, params=params) - timer.tags[metrics.Tag.http_status_code] = resp.status_code - return resp + + if resp.status_code != 200: + raise_for_error(resp) + else: + timer.tags[metrics.Tag.http_status_code] = resp.status_code + return resp def get_all_using_next(stream, url, api_key, since=None): @@ -80,7 +140,7 @@ def get_all_pages(source, url, api_key): else: break - +@backoff.on_exception(backoff.expo, simplejson.scanner.JSONDecodeError, max_tries=3) def get_incremental_pull(stream, endpoint, state, api_key, start_date): latest_event_time = get_starting_point(stream, state, start_date) @@ -102,10 +162,13 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): return state - +@backoff.on_exception(backoff.expo, simplejson.scanner.JSONDecodeError, max_tries=3) def get_full_pulls(resource, endpoint, api_key): + with metrics.record_counter(resource['stream']) as counter: + for response in get_all_pages(resource['stream'], endpoint, api_key): + records = response.json().get('data') counter.increment(len(records)) transfrom_and_write_records(records, resource) diff --git a/tests/test_backoff.py b/tests/test_backoff.py new file mode 100644 index 0000000..ad3a300 --- /dev/null +++ b/tests/test_backoff.py @@ -0,0 +1,42 @@ +import tap_klaviyo.utils as utils_ +import unittest +from unittest import mock +import simplejson +import singer +from singer import metrics +import json +import requests + +class TestBackoff(unittest.TestCase): + + @mock.patch('requests.Session.request') + def test_httperror(self, mocked_session): + + mock_resp = mock.Mock() + klaviyo_error = utils_.KlaviyoError() + http_error = requests.HTTPError() + + mock_resp.raise_for_error.side_effect = klaviyo_error + mock_resp.raise_for_status.side_effect = http_error + + mocked_session.return_value = mock_resp + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError: + pass + + self.assertEquals(mocked_session.call_count, 3) + + @mock.patch("tap_klaviyo.utils.get_all_pages") + def test_jsondecode(self, mock1): + + mock1.return_value = utils_.get_all_pages("lists", "http://www.youtube.com/results?abcd", "") + + data = {'stream': 'lists'} + try: + utils_.get_full_pulls(data, "http://www.youtube.com/results?abcd", "") + except simplejson.scanner.JSONDecodeError: + pass + + self.assertEquals(mock1.call_count, 2) \ No newline at end of file diff --git a/tests/test_exception_handling.py b/tests/test_exception_handling.py new file mode 100644 index 0000000..6fa8ba6 --- /dev/null +++ b/tests/test_exception_handling.py @@ -0,0 +1,127 @@ +import tap_klaviyo.utils as utils_ +import unittest +from unittest import mock +import requests +import singer +import json + +class Mockresponse: + def __init__(self, status_code, resp=None, content=None, headers=None, raise_error=False): + self.json_data = resp + self.status_code = status_code + self.content = content + self.headers = headers + self.raise_error = raise_error + + def json(self): + return self.json_data + + def raise_for_status(self): + raise requests.HTTPError + +def successful_200_request(*args, **kwargs): + json_str = {"tap": "klaviyo", "code": 200} + + return Mockresponse(200, json_str) + +def klaviyo_400_error(*args, **kwargs): + + return Mockresponse(status_code=400, raise_error=True) + +def klaviyo_401_error(*args, **kwargs): + + return Mockresponse(status_code=401, raise_error=True) + +def klaviyo_403_error(*args, **kwargs): + + return Mockresponse(status_code=403, raise_error=True) + +def klaviyo_403_error_wrong_api_key(*args, **kwargs): + json_str = { + "status": 403, + "message": "The API key specified is invalid."} + + return Mockresponse(resp=json_str, status_code=403, raise_error=True) + +def klaviyo_403_error_missing_api_key(*args, **kwargs): + json_str = { + "status": 403, + "message": "You must specify an API key to make requests."} + + return Mockresponse(resp=json_str, status_code=403, raise_error=True) + +def klaviyo_404_error(*args, **kwargs): + + return Mockresponse(status_code=404, raise_error=True) + +def klaviyo_500_error(*args, **kwargs): + + return Mockresponse(status_code=500, raise_error=True) + + +class TestBackoff(unittest.TestCase): + + @mock.patch('requests.Session.request', side_effect=successful_200_request) + def test_200(self, successful_200_request): + test_data = {"tap": "klaviyo", "code": 200} + + actual_data = utils_.authed_get("", "", "").json() + self.assertEquals(actual_data, test_data) + + @mock.patch('requests.Session.request', side_effect=klaviyo_400_error) + def test_400_error(self, klaviyo_400_error): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + self.assertEquals(str(e), "HTTP-error-code: 400, Error: Request is missing or has a bad parameter.") + + @mock.patch('requests.Session.request', side_effect=klaviyo_401_error) + def test_401_error(self, klaviyo_401_error): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + self.assertEquals(str(e), "HTTP-error-code: 401, Error: Invalid authorization credentials.") + + @mock.patch('requests.Session.request', side_effect=klaviyo_403_error) + def test_403_error(self, klaviyo_403_error): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + with open("abc.txt", "w") as f: + f.write(str(e)) + self.assertEquals(str(e), "HTTP-error-code: 403, Error: Invalid authorization credentials or permissions.") + + @mock.patch('requests.Session.request', side_effect=klaviyo_403_error_wrong_api_key) + def test_403_error_wrong_api_key(self, klaviyo_403_error_wrong_api_key): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + self.assertEquals(str(e), "HTTP-error-code: 403, Error: The API key specified is invalid.") + + @mock.patch('requests.Session.request', side_effect=klaviyo_403_error_missing_api_key) + def test_403_error_missing_api_key(self, klaviyo_403_error_missing_api_key): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + self.assertEquals(str(e), "HTTP-error-code: 403, Error: You must specify an API key to make requests.") + + @mock.patch('requests.Session.request', side_effect=klaviyo_404_error) + def test_404_error(self, klaviyo_404_error): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + self.assertEquals(str(e), "HTTP-error-code: 404, Error: The requested resource doesn't exist.") + + @mock.patch('requests.Session.request', side_effect=klaviyo_500_error) + def test_500_error(self, klaviyo_500_error): + + try: + utils_.authed_get("", "", "") + except utils_.KlaviyoError as e: + self.assertEquals(str(e), "HTTP-error-code: 500, Error: Internal Service Error from Klaviyo.") \ No newline at end of file