From ae5113df0cc64aacba52ebfb1b077b3add669b7b Mon Sep 17 00:00:00 2001 From: jr conlin Date: Fri, 3 Nov 2017 18:00:03 -0700 Subject: [PATCH] feat: assimilate the gcm client code Closes: #1057 --- autopush/router/gcm.py | 19 ++- autopush/router/gcmclient.py | 113 +++++++++++++++ autopush/tests/test_integration.py | 11 +- autopush/tests/test_router.py | 214 ++++++++++++++++++++++++++--- 4 files changed, 325 insertions(+), 32 deletions(-) create mode 100644 autopush/router/gcmclient.py diff --git a/autopush/router/gcm.py b/autopush/router/gcm.py index 70556f62..f9241ae9 100644 --- a/autopush/router/gcm.py +++ b/autopush/router/gcm.py @@ -1,13 +1,13 @@ """GCM Router""" from typing import Any # noqa -import gcmclient from requests.exceptions import ConnectionError from twisted.internet.threads import deferToThread from twisted.logger import Logger from autopush.exceptions import RouterException from autopush.metrics import make_tags +from autopush.router import gcmclient from autopush.router.interface import RouterResponse from autopush.types import JSONDict # noqa @@ -115,12 +115,16 @@ def _route(self, notification, uaid_data): try: gcm = self.gcm[router_data['creds']['senderID']] result = gcm.send(payload) + except RouterException: + raise # pragma nocover except KeyError: self.log.critical("Missing GCM bridge credentials") - raise RouterException("Server error", status_code=500) + raise RouterException("Server error", status_code=500, + errno=900) except gcmclient.GCMAuthenticationError as e: self.log.error("GCM Authentication Error: %s" % e) - raise RouterException("Server error", status_code=500) + raise RouterException("Server error", status_code=500, + errno=901) except ConnectionError as e: self.log.warn("GCM Unavailable: %s" % e) self.metrics.increment("notification.bridge.error", @@ -128,6 +132,7 @@ def _route(self, notification, uaid_data): self._base_tags, reason="connection_unavailable")) raise RouterException("Server error", status_code=502, + errno=902, log_exception=False) except Exception as e: self.log.error("Unhandled exception in GCM Routing: %s" % e) @@ -181,7 +186,7 @@ def _process_reply(self, reply, uaid_data, ttl, notification): ) # retries: - if reply.needs_retry(): + if reply.retry_after: self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="retry")) @@ -189,7 +194,11 @@ def _process_reply(self, reply, uaid_data, ttl, notification): failed=lambda: repr(reply.failed.items())) raise RouterException("GCM failure to deliver, retry", status_code=503, - response_body="Please try request later.", + headers={"Retry-After": reply.retry_after}, + response_body="Please try request " + "in {} seconds.".format( + reply.retry_after + ), log_exception=False) self.metrics.increment("notification.bridge.sent", diff --git a/autopush/router/gcmclient.py b/autopush/router/gcmclient.py new file mode 100644 index 00000000..7c32fcfa --- /dev/null +++ b/autopush/router/gcmclient.py @@ -0,0 +1,113 @@ +import json + +import requests + +from autopush.exceptions import RouterException + + +class GCMAuthenticationError(Exception): + pass + + +class Result(object): + + def __init__(self, message, response): + self.success = {} + self.canonicals = {} + self.unavailable = [] + self.not_registered = [] + self.failed = {} + + self.message = message + self.retry_message = None + + self.retry_after = response.headers.get('Retry-After', None) + + if response.status_code != 200: + self.retry_message = message + else: + self._parse_response(message, response.content) + + def _parse_response(self, message, content): + data = json.loads(content) + if not data.get('results'): + raise RouterException("Recv'd invalid response from GCM") + reg_id = message.payload['registration_ids'][0] + for res in data['results']: + if 'message_id' in res: + self.success[reg_id] = res['message_id'] + if 'registration_id' in res: + self.canonicals[reg_id] = res['registration_id'] + else: + if res['error'] in ['Unavailable', 'InternalServerError']: + self.unavailable.append(reg_id) + elif res['error'] == 'NotRegistered': + self.not_registered.append(reg_id) + else: + self.failed[reg_id] = res['error'] + + +class JSONMessage(object): + + def __init__(self, + registration_ids, + collapse_key, + time_to_live, + dry_run, + data): + if not registration_ids: + raise RouterException("No Registration IDs specified") + if not isinstance(registration_ids, list): + registration_ids = [registration_ids] + self.registration_ids = registration_ids + self.payload = { + 'registration_ids': self.registration_ids, + 'collapse_key': collapse_key, + 'time_to_live': int(time_to_live), + 'delay_while_idle': False, + 'dry_run': bool(dry_run), + } + if data: + self.payload['data'] = data + + +class GCM(object): + + def __init__(self, + api_key=None, + logger=None, + metrics=None, + endpoint="gcm-http.googleapis.com/gcm/send", + **options): + + self._endpoint = "https://{}".format(endpoint) + self._api_key = api_key + self.metrics = metrics + self.log = logger + self._options = options + self._sender = requests.post + + def send(self, payload): + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'key={}'.format(self._api_key), + } + + response = self._sender( + url=self._endpoint, + headers=headers, + data=json.dumps(payload.payload), + **self._options + ) + + if response.status_code == 400: + raise RouterException(response.content) + + if response.status_code == 404: + raise RouterException(response.content) + + if response.status_code == 401: + raise GCMAuthenticationError("Authentication Error") + + if response.status_code == 200 or (500 <= response.status_code <= 599): + return Result(payload, response) diff --git a/autopush/tests/test_integration.py b/autopush/tests/test_integration.py index 4259dd61..98b5b667 100644 --- a/autopush/tests/test_integration.py +++ b/autopush/tests/test_integration.py @@ -1576,11 +1576,8 @@ class MockReply(object): failed_items = dict() not_registered = dict() failed = dict() - _needs_retry = False - - @classmethod - def needs_retry(cls=None): - return False + retry_after = False + retry_message = None def _add_router(self): from autopush.router.gcm import GCMRouter @@ -1645,7 +1642,7 @@ def test_registration(self): body=data ) - ca_data = self._mock_send.call_args[0][0].data + ca_data = self._mock_send.call_args[0][0].payload['data'] assert response.code == 201 # ChannelID here MUST match what we got from the registration call. # Currently, this is a lowercase, hex UUID without dashes. @@ -1704,7 +1701,7 @@ def test_registration_aes128gcm(self): body=data ) - ca_data = self._mock_send.call_args[0][0].data + ca_data = self._mock_send.call_args[0][0].payload['data'] assert response.code == 201 # ChannelID here MUST match what we got from the registration call. # Currently, this is a lowercase, hex UUID without dashes. diff --git a/autopush/tests/test_router.py b/autopush/tests/test_router.py index 9aa333e0..34d822dc 100644 --- a/autopush/tests/test_router.py +++ b/autopush/tests/test_router.py @@ -15,9 +15,9 @@ import hyper import hyper.tls -import gcmclient import pyfcm from hyper.http20.exceptions import HTTP20Error +import requests from autopush.config import AutopushConfig from autopush.db import ( @@ -30,7 +30,7 @@ GCMRouter, WebPushRouter, FCMRouter, -) + gcmclient) from autopush.router.interface import RouterResponse, IRouter from autopush.tests import MockAssist from autopush.tests.support import test_db @@ -296,9 +296,177 @@ def check_results(result): return d +class GCMClientTestCase(unittest.TestCase): + + def setUp(self): + self.gcm = gcmclient.GCM(api_key="FakeValue") + self.gcm._sender = self._m_request = Mock(spec=requests.post) + self._m_response = Mock(spec=requests.Response) + self._m_response.return_value = 200 + self._m_response.headers = dict() + self.m_payload = gcmclient.JSONMessage( + registration_ids="some_reg_id", + collapse_key="coll_key", + time_to_live=60, + dry_run=False, + data={"foo": "bar"} + ) + + def test_send(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 0, + "canonical_ids": 0, + "results": [ + { + "message_id": "0:1510011451922224%7a0e7efbaab8b7cc" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.failed) == 0 + assert len(result.canonicals) == 0 + assert (len(result.success) == 1 + and self.m_payload.registration_ids[0] in result.success) + + def test_canonical(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 0, + "canonical_ids": 0, + "results": [ + { + "message_id": "0:1510011451922224%7a0e7efbaab8b7cc", + "registration_id": "otherId", + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.failed) == 0 + assert len(result.canonicals) == 1 + assert (len(result.success) == 1 + and self.m_payload.registration_ids[0] in result.success) + + def test_bad_jsonmessage(self): + with pytest.raises(RouterException): + self.m_payload = gcmclient.JSONMessage( + registration_ids=None, + collapse_key="coll_key", + time_to_live=60, + dry_run=False, + data={"foo": "bar"} + ) + + def test_fail_invalid(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "InvalidRegistration" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.failed) == 1 + assert len(result.success) == 0 + + def test_fail_unavailable(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "Unavailable" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.unavailable) == 1 + assert len(result.success) == 0 + + def test_fail_not_registered(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "NotRegistered" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.not_registered) == 1 + assert len(result.success) == 0 + + def test_fail_bad_response(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + }) + self._m_request.return_value = self._m_response + with pytest.raises(RouterException): + self.gcm.send(self.m_payload) + + def test_fail_400(self): + self._m_response.status_code = 400 + self._m_response.content = msg = "Invalid JSON" + self._m_request.return_value = self._m_response + with pytest.raises(RouterException) as ex: + self.gcm.send(self.m_payload) + assert ex.value.status_code == 500 + assert ex.value.message == msg + + def test_fail_404(self): + self._m_response.status_code = 404 + self._m_response.content = msg = "Invalid URL" + self._m_request.return_value = self._m_response + with pytest.raises(RouterException) as ex: + self.gcm.send(self.m_payload) + assert ex.value.status_code == 500 + assert ex.value.message == msg + + def test_fail_401(self): + self._m_response.status_code = 401 + self._m_response.content = "Unauthorized" + self._m_request.return_value = self._m_response + with pytest.raises(gcmclient.GCMAuthenticationError): + self.gcm.send(self.m_payload) + + def test_fail_500(self): + self._m_response.status_code = 500 + self._m_response.content = "OMG" + self._m_response.headers['Retry-After'] = 123 + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert 'some_reg_id' in result.retry_message.registration_ids + assert result.retry_after == 123 + + class GCMRouterTestCase(unittest.TestCase): - @patch("gcmclient.gcm.GCM", spec=gcmclient.gcm.GCM) + @patch("autopush.router.gcmclient.GCM", spec=gcmclient.GCM) def setUp(self, fgcm): conf = AutopushConfig( hostname="localhost", @@ -327,17 +495,19 @@ def setUp(self, fgcm): router_data=dict( token="connect_data", creds=dict(senderID="test123", auth="12345678abcdefg"))) - mock_result = Mock(spec=gcmclient.gcm.Result) + mock_result = Mock(spec=gcmclient.Result) mock_result.canonical = dict() mock_result.failed = dict() mock_result.not_registered = dict() - mock_result.needs_retry.return_value = False + mock_result.retry_after = None self.mock_result = mock_result fgcm.send.return_value = mock_result - def _check_error_call(self, exc, code, response=None): + def _check_error_call(self, exc, code, response=None, errno=None): assert isinstance(exc, RouterException) assert exc.status_code == code + if errno is not None: + assert exc.errno == errno assert self.router.gcm['test123'].send.called if response: assert exc.response_body == response @@ -371,7 +541,7 @@ def test_register_bad(self): router_data={"token": "abcd1234"}, app_id="invalid123") - @patch("gcmclient.GCM") + @patch("autopush.router.gcmclient.GCM") def test_gcmclient_fail(self, fgcm): fgcm.side_effect = Exception conf = AutopushConfig( @@ -393,7 +563,8 @@ def check_results(result): assert isinstance(result, RouterResponse) assert self.router.gcm['test123'].send.called # Make sure the data was encoded as base64 - data = self.router.gcm['test123'].send.call_args[0][0].data + data = self.router.gcm[ + 'test123'].send.call_args[0][0].payload['data'] assert data['body'] == 'q60d6g' assert data['enc'] == 'test' assert data['chid'] == dummy_chid @@ -421,15 +592,15 @@ def check_results(result): assert "TTL" in result.headers assert self.router.gcm['test123'].send.called # Make sure the data was encoded as base64 - data = self.router.gcm['test123'].send.call_args[0][0].data - options = self.router.gcm['test123'].send.call_args[0][0].options + payload = self.router.gcm['test123'].send.call_args[0][0].payload + data = payload['data'] assert data['body'] == 'q60d6g' assert data['enc'] == 'test' assert data['chid'] == dummy_chid assert data['enckey'] == 'test' assert data['con'] == 'aesgcm' # use the defined min TTL - assert options['time_to_live'] == 60 + assert payload['time_to_live'] == 60 d.addCallback(check_results) return d @@ -449,15 +620,15 @@ def check_results(result): assert isinstance(result, RouterResponse) assert self.router.gcm['test123'].send.called # Make sure the data was encoded as base64 - data = self.router.gcm['test123'].send.call_args[0][0].data - options = self.router.gcm['test123'].send.call_args[0][0].options + payload = self.router.gcm['test123'].send.call_args[0][0].payload + data = payload['data'] assert data['body'] == 'q60d6g' assert data['enc'] == 'test' assert data['chid'] == dummy_chid assert data['enckey'] == 'test' assert data['con'] == 'aesgcm' # use the defined min TTL - assert options['time_to_live'] == 2419200 + assert payload['time_to_live'] == 2419200 d.addCallback(check_results) return d @@ -501,7 +672,7 @@ def throw_auth(arg): d = self.router.route_notification(self.notif, self.router_data) def check_results(fail): - self._check_error_call(fail.value, 500, "Server error") + self._check_error_call(fail.value, 500, "Server error", 901) d.addBoth(check_results) return d @@ -528,7 +699,7 @@ def throw_other(*args, **kwargs): d = self.router.route_notification(self.notif, self.router_data) def check_results(fail): - self._check_error_call(fail.value, 502, "Server error") + self._check_error_call(fail.value, 502, "Server error", 902) d.addBoth(check_results) return d @@ -587,7 +758,7 @@ def check_results(fail): return d def test_router_notification_gcm_needs_retry(self): - self.mock_result.needs_retry.return_value = True + self.mock_result.retry_after = 1000 self.router.gcm['test123'] = self.gcm self.router.metrics = Mock() d = self.router.route_notification(self.notif, self.router_data) @@ -609,7 +780,10 @@ def test_router_notification_gcm_no_auth(self): {"router_data": {"token": "abc"}}) def check_results(fail): - assert fail.value.status_code == 500, "Server error" + assert isinstance(fail.value, RouterException) + assert fail.value.message == "Server error" + assert fail.value.status_code == 500 + assert fail.value.errno == 900 d.addBoth(check_results) return d @@ -942,11 +1116,11 @@ def setUp(self): message_id=uuid.uuid4().hex, ) self.notif.cleanup_headers() - mock_result = Mock(spec=gcmclient.gcm.Result) + mock_result = Mock(spec=gcmclient.Result) mock_result.canonical = dict() mock_result.failed = dict() mock_result.not_registered = dict() - mock_result.needs_retry.return_value = False + mock_result.retry_after = 1000 self.router_mock = db.router self.message_mock = db.message = Mock(spec=Message) self.conf = conf