diff --git a/autopush/router/apnsrouter.py b/autopush/router/apnsrouter.py index 45dcbf0c..007c593d 100644 --- a/autopush/router/apnsrouter.py +++ b/autopush/router/apnsrouter.py @@ -1,4 +1,5 @@ """APNS Router""" +import ssl import uuid from typing import Any # noqa @@ -137,7 +138,10 @@ def _route(self, notification, router_data): payload["enckey"] = notification.headers["encryption_key"] payload['aps'] = router_data.get('aps', { "mutable-content": 1, - "alert": {"title": " ", "body": " "} + "alert": { + "loc-key": "SentTab.NoTabArrivingNotification.body", + "title-loc-key": "SentTab.NoTabArrivingNotification.title", + } }) apns_id = str(uuid.uuid4()).lower() # APNs may force close a connection on us without warning. @@ -158,6 +162,17 @@ def _route(self, notification, router_data): tags=make_tags(self._base_tags, application=rel_channel, reason="http2_error")) + except ssl.SSLError as e: + # can only str match this (for autopush#1048) + if not (e.errno == ssl.SSL_ERROR_SSL and + str(e).startswith("[SSL: BAD_WRITE_RETRY]")): + raise # pragma: nocover + self.metrics.increment( + "notification.bridge.connection.error", + tags=make_tags(self._base_tags, + application=rel_channel, + reason="bad_write_retry_error") + ) if not success: raise RouterException( "Server error", diff --git a/autopush/router/gcm.py b/autopush/router/gcm.py index 70556f62..ea768590 100644 --- a/autopush/router/gcm.py +++ b/autopush/router/gcm.py @@ -1,13 +1,13 @@ """GCM Router""" from typing import Any # noqa -import gcmclient from requests.exceptions import ConnectionError from twisted.internet.threads import deferToThread from twisted.logger import Logger from autopush.exceptions import RouterException from autopush.metrics import make_tags +from autopush.router import gcmclient from autopush.router.interface import RouterResponse from autopush.types import JSONDict # noqa @@ -35,10 +35,7 @@ def __init__(self, conf, router_conf, metrics): for sid in router_conf.get("senderIDs"): auth = router_conf.get("senderIDs").get(sid).get("auth") self.senderIDs[sid] = auth - try: - self.gcm[sid] = gcmclient.GCM(auth) - except Exception: - raise IOError("GCM Bridge not initiated in main") + self.gcm[sid] = gcmclient.GCM(auth) self._base_tags = ["platform:gcm"] self.log.debug("Starting GCM router...") @@ -115,12 +112,16 @@ def _route(self, notification, uaid_data): try: gcm = self.gcm[router_data['creds']['senderID']] result = gcm.send(payload) + except RouterException: + raise # pragma nocover except KeyError: self.log.critical("Missing GCM bridge credentials") - raise RouterException("Server error", status_code=500) + raise RouterException("Server error", status_code=500, + errno=900) except gcmclient.GCMAuthenticationError as e: self.log.error("GCM Authentication Error: %s" % e) - raise RouterException("Server error", status_code=500) + raise RouterException("Server error", status_code=500, + errno=901) except ConnectionError as e: self.log.warn("GCM Unavailable: %s" % e) self.metrics.increment("notification.bridge.error", @@ -128,6 +129,7 @@ def _route(self, notification, uaid_data): self._base_tags, reason="connection_unavailable")) raise RouterException("Server error", status_code=502, + errno=902, log_exception=False) except Exception as e: self.log.error("Unhandled exception in GCM Routing: %s" % e) @@ -146,7 +148,7 @@ def _process_reply(self, reply, uaid_data, ttl, notification): # acks: # for reg_id, msg_id in reply.success.items(): # updates - for old_id, new_id in reply.canonical.items(): + for old_id, new_id in reply.canonicals.items(): self.log.debug("GCM id changed : {old} => {new}", old=old_id, new=new_id) self.metrics.increment("notification.bridge.error", @@ -181,7 +183,7 @@ def _process_reply(self, reply, uaid_data, ttl, notification): ) # retries: - if reply.needs_retry(): + if reply.retry_after: self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="retry")) @@ -189,7 +191,11 @@ def _process_reply(self, reply, uaid_data, ttl, notification): failed=lambda: repr(reply.failed.items())) raise RouterException("GCM failure to deliver, retry", status_code=503, - response_body="Please try request later.", + headers={"Retry-After": reply.retry_after}, + response_body="Please try request " + "in {} seconds.".format( + reply.retry_after + ), log_exception=False) self.metrics.increment("notification.bridge.sent", diff --git a/autopush/router/gcmclient.py b/autopush/router/gcmclient.py new file mode 100644 index 00000000..868ce3b5 --- /dev/null +++ b/autopush/router/gcmclient.py @@ -0,0 +1,157 @@ +import json + +import requests + +from autopush.exceptions import RouterException + + +class GCMAuthenticationError(Exception): + pass + + +class Result(object): + """Abstraction object for GCM response""" + + def __init__(self, message, response): + """Process GCM message and response into abstracted object + + :param message: Message payload + :type message: JSONMessage + :param response: GCM response + :type response: requests.Response + + """ + self.success = {} + self.canonicals = {} + self.unavailable = [] + self.not_registered = [] + self.failed = {} + + self.message = message + self.retry_message = None + + self.retry_after = response.headers.get('Retry-After', None) + + if response.status_code != 200: + self.retry_message = message + else: + self._parse_response(message, response.content) + + def _parse_response(self, message, content): + data = json.loads(content) + if not data.get('results'): + raise RouterException("Recv'd invalid response from GCM") + reg_id = message.payload['registration_ids'][0] + for res in data['results']: + if 'message_id' in res: + self.success[reg_id] = res['message_id'] + if 'registration_id' in res: + self.canonicals[reg_id] = res['registration_id'] + else: + if res['error'] in ['Unavailable', 'InternalServerError']: + self.unavailable.append(reg_id) + elif res['error'] == 'NotRegistered': + self.not_registered.append(reg_id) + else: + self.failed[reg_id] = res['error'] + + +class JSONMessage(object): + """GCM formatted payload + + """ + def __init__(self, + registration_ids, + collapse_key, + time_to_live, + dry_run, + data): + """Convert data elements into a GCM payload. + + :param registration_ids: Single or list of registration ids to send to + :type registration_ids: str or list + :param collapse_key: GCM collapse key for the data. + :type collapse_key: str + :param time_to_live: Seconds to keep message alive + :type time_to_live: int + :param dry_run: GCM Dry run flag to allow remote verification + :type dry_run: bool + :param data: Data elements to send + :type data: dict + + """ + if not registration_ids: + raise RouterException("No Registration IDs specified") + if not isinstance(registration_ids, list): + registration_ids = [registration_ids] + self.registration_ids = registration_ids + self.payload = { + 'registration_ids': self.registration_ids, + 'collapse_key': collapse_key, + 'time_to_live': int(time_to_live), + 'delay_while_idle': False, + 'dry_run': bool(dry_run), + } + if data: + self.payload['data'] = data + + +class GCM(object): + """Primitive HTTP GCM service handler.""" + + def __init__(self, + api_key=None, + logger=None, + metrics=None, + endpoint="gcm-http.googleapis.com/gcm/send", + **options): + + """Initialize the GCM primitive. + + :param api_key: The GCM API key (from the Google developer console) + :type api_key: str + :param logger: Status logger + :type logger: logger + :param metrics: Metric recorder + :type metrics: autopush.metrics.IMetric + :param endpoint: GCM endpoint override + :type endpoint: str + :param options: Additional options + :type options: dict + + """ + self._endpoint = "https://{}".format(endpoint) + self._api_key = api_key + self.metrics = metrics + self.log = logger + self._options = options + self._sender = requests.post + + def send(self, payload): + """Send a payload to GCM + + :param payload: Dictionary of GCM formatted data + :type payload: JSONMessage + :return: Result + + """ + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'key={}'.format(self._api_key), + } + + response = self._sender( + url=self._endpoint, + headers=headers, + data=json.dumps(payload.payload), + **self._options + ) + + if response.status_code in (400, 404): + raise RouterException(response.content) + + if response.status_code == 401: + raise GCMAuthenticationError("Authentication Error") + + if response.status_code == 200 or (500 <= response.status_code <= 599): + return Result(payload, response) diff --git a/autopush/ssl.py b/autopush/ssl.py index 2ea70226..604a4a81 100644 --- a/autopush/ssl.py +++ b/autopush/ssl.py @@ -13,6 +13,12 @@ from OpenSSL import SSL from twisted.internet.ssl import DefaultOpenSSLContextFactory +try: + SSL_PROTO = ssl.PROTOCOL_TLS +except AttributeError: + SSL_PROTO = ssl.PROTOCOL_SSLv23 + + MOZILLA_INTERMEDIATE_CIPHERS = ( 'ECDHE-RSA-AES128-GCM-SHA256:' 'ECDHE-ECDSA-AES128-GCM-SHA256:' @@ -111,7 +117,7 @@ def ssl_wrap_socket_cached( certfile=None, # type: Optional[str] server_side=False, # type: bool cert_reqs=ssl.CERT_NONE, # type: int - ssl_version=ssl.PROTOCOL_TLS, # type: int + ssl_version=SSL_PROTO, # type: int ca_certs=None, # type: Optional[str] do_handshake_on_connect=True, # type: bool suppress_ragged_eofs=True, # type: bool diff --git a/autopush/tests/test_gcmclient.py b/autopush/tests/test_gcmclient.py new file mode 100644 index 00000000..cbb0ecc9 --- /dev/null +++ b/autopush/tests/test_gcmclient.py @@ -0,0 +1,177 @@ +import json + +import pytest +import requests +from mock import Mock +from twisted.trial import unittest + +from autopush.exceptions import RouterException +from autopush.router import gcmclient + + +class GCMClientTestCase(unittest.TestCase): + + def setUp(self): + self.gcm = gcmclient.GCM(api_key="FakeValue") + self.gcm._sender = self._m_request = Mock(spec=requests.post) + self._m_response = Mock(spec=requests.Response) + self._m_response.return_value = 200 + self._m_response.headers = dict() + self.m_payload = gcmclient.JSONMessage( + registration_ids="some_reg_id", + collapse_key="coll_key", + time_to_live=60, + dry_run=False, + data={"foo": "bar"} + ) + + def test_send(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 0, + "canonical_ids": 0, + "results": [ + { + "message_id": "0:1510011451922224%7a0e7efbaab8b7cc" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.failed) == 0 + assert len(result.canonicals) == 0 + assert (len(result.success) == 1 + and self.m_payload.registration_ids[0] in result.success) + + def test_canonical(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 0, + "canonical_ids": 1, + "results": [ + { + "message_id": "0:1510011451922224%7a0e7efbaab8b7cc", + "registration_id": "otherId", + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.failed) == 0 + assert len(result.canonicals) == 1 + assert (len(result.success) == 1 + and self.m_payload.registration_ids[0] in result.success) + + def test_bad_jsonmessage(self): + with pytest.raises(RouterException): + self.m_payload = gcmclient.JSONMessage( + registration_ids=None, + collapse_key="coll_key", + time_to_live=60, + dry_run=False, + data={"foo": "bar"} + ) + + def test_fail_invalid(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "InvalidRegistration" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.failed) == 1 + assert len(result.success) == 0 + + def test_fail_unavailable(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "Unavailable" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.unavailable) == 1 + assert len(result.success) == 0 + + def test_fail_not_registered(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "NotRegistered" + } + ] + }) + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert len(result.not_registered) == 1 + assert len(result.success) == 0 + + def test_fail_bad_response(self): + self._m_response.status_code = 200 + self._m_response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 0, + "failure": 1, + "canonical_ids": 0, + }) + self._m_request.return_value = self._m_response + with pytest.raises(RouterException): + self.gcm.send(self.m_payload) + + def test_fail_400(self): + self._m_response.status_code = 400 + self._m_response.content = msg = "Invalid JSON" + self._m_request.return_value = self._m_response + with pytest.raises(RouterException) as ex: + self.gcm.send(self.m_payload) + assert ex.value.status_code == 500 + assert ex.value.message == msg + + def test_fail_404(self): + self._m_response.status_code = 404 + self._m_response.content = msg = "Invalid URL" + self._m_request.return_value = self._m_response + with pytest.raises(RouterException) as ex: + self.gcm.send(self.m_payload) + assert ex.value.status_code == 500 + assert ex.value.message == msg + + def test_fail_401(self): + self._m_response.status_code = 401 + self._m_response.content = "Unauthorized" + self._m_request.return_value = self._m_response + with pytest.raises(gcmclient.GCMAuthenticationError): + self.gcm.send(self.m_payload) + + def test_fail_500(self): + self._m_response.status_code = 500 + self._m_response.content = "OMG" + self._m_response.headers['Retry-After'] = 123 + self._m_request.return_value = self._m_response + result = self.gcm.send(self.m_payload) + assert 'some_reg_id' in result.retry_message.registration_ids + assert result.retry_after == 123 diff --git a/autopush/tests/test_integration.py b/autopush/tests/test_integration.py index b25fa39e..01302413 100644 --- a/autopush/tests/test_integration.py +++ b/autopush/tests/test_integration.py @@ -1562,15 +1562,12 @@ class TestGCMBridgeIntegration(IntegrationBase): class MockReply(object): success = dict() - canonical = dict() + canonicals = dict() failed_items = dict() not_registered = dict() failed = dict() - _needs_retry = False - - @classmethod - def needs_retry(cls=None): - return False + retry_after = False + retry_message = None def _add_router(self): from autopush.router.gcm import GCMRouter @@ -1635,7 +1632,7 @@ def test_registration(self): body=data ) - ca_data = self._mock_send.call_args[0][0].data + ca_data = self._mock_send.call_args[0][0].payload['data'] assert response.code == 201 # ChannelID here MUST match what we got from the registration call. # Currently, this is a lowercase, hex UUID without dashes. @@ -1694,7 +1691,7 @@ def test_registration_aes128gcm(self): body=data ) - ca_data = self._mock_send.call_args[0][0].data + ca_data = self._mock_send.call_args[0][0].payload['data'] assert response.code == 201 # ChannelID here MUST match what we got from the registration call. # Currently, this is a lowercase, hex UUID without dashes. @@ -1931,8 +1928,10 @@ def test_registration(self): assert ca_data['cryptokey'] == crypto_key assert ca_data['enc'] == salt assert 'mutable-content' in ca_data['aps'] - assert ca_data['aps']['alert']['title'] == " " - assert ca_data['aps']['alert']['body'] == " " + assert ca_data["aps"]["alert"]["loc-key"] == \ + "SentTab.NoTabArrivingNotification.body" + assert ca_data["aps"]["alert"]["title-loc-key"] == \ + "SentTab.NoTabArrivingNotification.title" assert ca_data['body'] == base64url_encode(data) @inlineCallbacks diff --git a/autopush/tests/test_router.py b/autopush/tests/test_router.py index 9aa333e0..9e721585 100644 --- a/autopush/tests/test_router.py +++ b/autopush/tests/test_router.py @@ -4,6 +4,8 @@ import time import json import decimal +import requests +import ssl from autopush.utils import WebPushNotification from mock import Mock, PropertyMock, patch @@ -15,7 +17,6 @@ import hyper import hyper.tls -import gcmclient import pyfcm from hyper.http20.exceptions import HTTP20Error @@ -30,7 +31,7 @@ GCMRouter, WebPushRouter, FCMRouter, -) + gcmclient) from autopush.router.interface import RouterResponse, IRouter from autopush.tests import MockAssist from autopush.tests.support import test_db @@ -88,7 +89,8 @@ def setUp(self, mt, mc): } self.mock_connection = mc mc.return_value = mc - self.router = APNSRouter(conf, apns_config, SinkMetrics()) + self.metrics = metrics = Mock(spec=SinkMetrics) + self.router = APNSRouter(conf, apns_config, metrics) self.mock_response = Mock() self.mock_response.status = 200 mc.get_response.return_value = self.mock_response @@ -182,7 +184,10 @@ def test_route_notification(self): "ver": 10, "aps": { "mutable-content": 1, - "alert": {"title": " ", "body": " "} + "alert": { + "loc-key": "SentTab.NoTabArrivingNotification.body", + "title-loc-key": "SentTab.NoTabArrivingNotification.title", + }, }, "enckey": "test", "con": "aesgcm", @@ -250,6 +255,30 @@ def throw(*args, **kwargs): assert ex.value.message == "Server error" assert ex.value.response_body == 'APNS returned an error ' \ 'processing request' + assert self.metrics.increment.called + assert self.metrics.increment.call_args[0][0] == \ + 'notification.bridge.connection.error' + self.flushLoggedErrors() + + @inlineCallbacks + def test_fail_send_bad_write_retry(self): + def throw(*args, **kwargs): + raise ssl.SSLError( + ssl.SSL_ERROR_SSL, + "[SSL: BAD_WRITE_RETRY] bad write retry" + ) + + self.router.apns['firefox'].connections[0].request.side_effect = throw + with pytest.raises(RouterException) as ex: + yield self.router.route_notification(self.notif, self.router_data) + assert isinstance(ex.value, RouterException) + assert ex.value.status_code == 502 + assert ex.value.message == "Server error" + assert ex.value.response_body == 'APNS returned an error ' \ + 'processing request' + assert self.metrics.increment.called + assert self.metrics.increment.call_args[0][0] == \ + 'notification.bridge.connection.error' self.flushLoggedErrors() def test_too_many_connections(self): @@ -298,8 +327,7 @@ def check_results(result): class GCMRouterTestCase(unittest.TestCase): - @patch("gcmclient.gcm.GCM", spec=gcmclient.gcm.GCM) - def setUp(self, fgcm): + def setUp(self): conf = AutopushConfig( hostname="localhost", statsd_host=None, @@ -308,8 +336,24 @@ def setUp(self, fgcm): 'ttl': 60, 'senderIDs': {'test123': {"auth": "12345678abcdefg"}}} - self.gcm = fgcm + self.response = Mock(spec=requests.Response) + self.response.status_code = 200 + self.response.headers = dict() + self.response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 0, + "canonical_ids": 0, + "results": [ + { + "message_id": "0:1510011451922224%7a0e7efbaab8b7cc" + } + ] + }) + self.gcm = gcmclient.GCM(api_key="SomeKey") + self.gcm._sender = Mock(return_value=self.response) self.router = GCMRouter(conf, self.gcm_config, SinkMetrics()) + self.router.gcm['test123'] = self.gcm self.headers = {"content-encoding": "aesgcm", "encryption": "test", "encryption-key": "test"} @@ -327,18 +371,13 @@ def setUp(self, fgcm): router_data=dict( token="connect_data", creds=dict(senderID="test123", auth="12345678abcdefg"))) - mock_result = Mock(spec=gcmclient.gcm.Result) - mock_result.canonical = dict() - mock_result.failed = dict() - mock_result.not_registered = dict() - mock_result.needs_retry.return_value = False - self.mock_result = mock_result - fgcm.send.return_value = mock_result - def _check_error_call(self, exc, code, response=None): + def _check_error_call(self, exc, code, response=None, errno=None): assert isinstance(exc, RouterException) assert exc.status_code == code - assert self.router.gcm['test123'].send.called + if errno is not None: + assert exc.errno == errno + assert self.gcm._sender.called if response: assert exc.response_body == response self.flushLoggedErrors() @@ -364,36 +403,23 @@ def test_register_bad(self): with pytest.raises(RouterException): self.router.register("uaid", router_data={}, app_id="") with pytest.raises(RouterException): - self.router.register("uaid", router_data={}, app_id='') + self.router.register("uaid", router_data={}, app_id=None) with pytest.raises(RouterException): self.router.register( "uaid", router_data={"token": "abcd1234"}, app_id="invalid123") - @patch("gcmclient.GCM") - def test_gcmclient_fail(self, fgcm): - fgcm.side_effect = Exception - conf = AutopushConfig( - hostname="localhost", - statsd_host=None, - ) - with pytest.raises(IOError): - GCMRouter( - conf, - {"senderIDs": {"test123": {"auth": "abcd"}}}, - SinkMetrics() - ) - def test_route_notification(self): self.router.gcm['test123'] = self.gcm d = self.router.route_notification(self.notif, self.router_data) def check_results(result): assert isinstance(result, RouterResponse) - assert self.router.gcm['test123'].send.called + assert self.gcm._sender.called # Make sure the data was encoded as base64 - data = self.router.gcm['test123'].send.call_args[0][0].data + payload = json.loads(self.gcm._sender.call_args[1]['data']) + data = payload['data'] assert data['body'] == 'q60d6g' assert data['enc'] == 'test' assert data['chid'] == dummy_chid @@ -419,17 +445,17 @@ def check_results(result): assert result.status_code == 201 assert result.logged_status == 200 assert "TTL" in result.headers - assert self.router.gcm['test123'].send.called + assert self.gcm._sender.called # Make sure the data was encoded as base64 - data = self.router.gcm['test123'].send.call_args[0][0].data - options = self.router.gcm['test123'].send.call_args[0][0].options + payload = json.loads(self.gcm._sender.call_args[1]['data']) + data = payload['data'] assert data['body'] == 'q60d6g' assert data['enc'] == 'test' assert data['chid'] == dummy_chid assert data['enckey'] == 'test' assert data['con'] == 'aesgcm' # use the defined min TTL - assert options['time_to_live'] == 60 + assert payload['time_to_live'] == 60 d.addCallback(check_results) return d @@ -447,17 +473,17 @@ def test_ttl_high(self): def check_results(result): assert isinstance(result, RouterResponse) - assert self.router.gcm['test123'].send.called + assert self.gcm._sender.called # Make sure the data was encoded as base64 - data = self.router.gcm['test123'].send.call_args[0][0].data - options = self.router.gcm['test123'].send.call_args[0][0].options + payload = json.loads(self.gcm._sender.call_args[1]['data']) + data = payload['data'] assert data['body'] == 'q60d6g' assert data['enc'] == 'test' assert data['chid'] == dummy_chid assert data['enckey'] == 'test' assert data['con'] == 'aesgcm' # use the defined min TTL - assert options['time_to_live'] == 2419200 + assert payload['time_to_live'] == 2419200 d.addCallback(check_results) return d @@ -482,34 +508,27 @@ def check_results(result): return d def test_route_crypto_notification(self): - self.router.gcm['test123'] = self.gcm del(self.notif.headers['encryption_key']) self.notif.headers['crypto_key'] = 'crypto' d = self.router.route_notification(self.notif, self.router_data) def check_results(result): assert isinstance(result, RouterResponse) - assert self.router.gcm['test123'].send.called + assert self.gcm._sender.called d.addCallback(check_results) return d def test_router_notification_gcm_auth_error(self): - def throw_auth(arg): - raise gcmclient.GCMAuthenticationError() - self.gcm.send.side_effect = throw_auth - self.router.gcm['test123'] = self.gcm + self.response.status_code = 401 d = self.router.route_notification(self.notif, self.router_data) def check_results(fail): - self._check_error_call(fail.value, 500, "Server error") + self._check_error_call(fail.value, 500, "Server error", 901) d.addBoth(check_results) return d def test_router_notification_gcm_other_error(self): - def throw_other(arg): - raise Exception("oh my!") - self.gcm.send.side_effect = throw_other - self.router.gcm['test123'] = self.gcm + self.gcm._sender.side_effect = Exception d = self.router.route_notification(self.notif, self.router_data) def check_results(fail): @@ -523,18 +542,28 @@ def test_router_notification_connection_error(self): def throw_other(*args, **kwargs): raise ConnectionError("oh my!") - self.gcm.send.side_effect = throw_other + self.gcm._sender.side_effect = throw_other self.router.gcm['test123'] = self.gcm d = self.router.route_notification(self.notif, self.router_data) def check_results(fail): - self._check_error_call(fail.value, 502, "Server error") + self._check_error_call(fail.value, 502, "Server error", 902) d.addBoth(check_results) return d def test_router_notification_gcm_id_change(self): - self.mock_result.canonical["old"] = "new" - self.router.gcm['test123'] = self.gcm + self.response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 0, + "canonical_ids": 1, + "results": [ + { + "message_id": "0:1510011451922224%7a0e7efbaab8b7cc", + "registration_id": "new", + } + ] + }) self.router.metrics = Mock() d = self.router.route_notification(self.notif, self.router_data) @@ -546,13 +575,22 @@ def check_results(result): self.router.metrics.increment.call_args[1]['tags'].sort() assert self.router.metrics.increment.call_args[1]['tags'] == [ 'platform:gcm', 'reason:reregister'] - assert self.router.gcm['test123'].send.called + assert self.gcm._sender.called d.addCallback(check_results) return d def test_router_notification_gcm_not_regged(self): - self.mock_result.not_registered = {"connect_data": True} - self.router.gcm['test123'] = self.gcm + self.response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "NotRegistered" + } + ] + }) self.router.metrics = Mock() d = self.router.route_notification(self.notif, self.router_data) @@ -564,13 +602,22 @@ def check_results(result): self.router.metrics.increment.call_args[1]['tags'].sort() assert self.router.metrics.increment.call_args[1]['tags'] == [ 'platform:gcm', 'reason:unregistered'] - assert self.router.gcm['test123'].send.called + assert self.gcm._sender.called d.addCallback(check_results) return d def test_router_notification_gcm_failed_items(self): - self.mock_result.failed = dict(connect_data=True) - self.router.gcm['test123'] = self.gcm + self.response.content = json.dumps({ + "multicast_id": 5174939174563864884, + "success": 1, + "failure": 1, + "canonical_ids": 0, + "results": [ + { + "error": "InvalidRegistration" + } + ] + }) self.router.metrics = Mock() d = self.router.route_notification(self.notif, self.router_data) @@ -587,8 +634,9 @@ def check_results(fail): return d def test_router_notification_gcm_needs_retry(self): - self.mock_result.needs_retry.return_value = True - self.router.gcm['test123'] = self.gcm + self.response.headers['Retry-After'] = "123" + self.response.status_code = 500 + self.response.content = "" self.router.metrics = Mock() d = self.router.route_notification(self.notif, self.router_data) @@ -609,7 +657,10 @@ def test_router_notification_gcm_no_auth(self): {"router_data": {"token": "abc"}}) def check_results(fail): - assert fail.value.status_code == 500, "Server error" + assert isinstance(fail.value, RouterException) + assert fail.value.message == "Server error" + assert fail.value.status_code == 500 + assert fail.value.errno == 900 d.addBoth(check_results) return d @@ -942,11 +993,11 @@ def setUp(self): message_id=uuid.uuid4().hex, ) self.notif.cleanup_headers() - mock_result = Mock(spec=gcmclient.gcm.Result) + mock_result = Mock(spec=gcmclient.Result) mock_result.canonical = dict() mock_result.failed = dict() mock_result.not_registered = dict() - mock_result.needs_retry.return_value = False + mock_result.retry_after = 1000 self.router_mock = db.router self.message_mock = db.message = Mock(spec=Message) self.conf = conf diff --git a/autopush/tests/test_web_webpush.py b/autopush/tests/test_web_webpush.py index ac01a3cb..ca6327f5 100644 --- a/autopush/tests/test_web_webpush.py +++ b/autopush/tests/test_web_webpush.py @@ -186,6 +186,19 @@ def test_request_v2_id_no_crypt_auth(self): ) assert resp.get_status() == 401 + @inlineCallbacks + def test_request_no_router_type(self): + self.fernet_mock.decrypt.return_value = 'a' * 32 + self.db.router.get_uaid.return_value = dict( + uaid=dummy_uaid, + chid=dummy_chid, + ) + resp = yield self.client.post( + self.url(api_ver='v1', token='ignored'), + headers={'authorization': 'webpush dummy.key'} + ) + assert resp.get_status() == 410 + @inlineCallbacks def test_request_bad_v2_id_bad_pubkey(self): self.fernet_mock.decrypt.return_value = 'a' * 64 diff --git a/autopush/web/webpush.py b/autopush/web/webpush.py index 6b2cbd7a..ac8c29d2 100644 --- a/autopush/web/webpush.py +++ b/autopush/web/webpush.py @@ -54,6 +54,8 @@ # Base64 URL validation VALID_BASE64_URL = re.compile(r'^[0-9A-Za-z\-_]+=*$') +VALID_ROUTER_TYPES = ["simplepush", "webpush", "gcm", "fcm", "apns"] + class WebPushSubscriptionSchema(Schema): uaid = fields.UUID(required=True) @@ -86,13 +88,25 @@ def validate_uaid_month_and_chid(self, d): except ItemNotFound: raise InvalidRequest("UAID not found", status_code=410, errno=103) - if (result.get("router_type") in ["gcm", "fcm"] + # We must have a router_type to validate the user + router_type = result.get("router_type") + if router_type not in VALID_ROUTER_TYPES: + self.context["log"].debug(format="Dropping User", code=102, + uaid_hash=hasher(result["uaid"]), + uaid_record=dump_uaid(result)) + self.context["metrics"].increment("updates.drop_user", + tags=make_tags(errno=102)) + self.context["db"].router.drop_user(result["uaid"]) + raise InvalidRequest("No such subscription", status_code=410, + errno=106) + + if (router_type in ["gcm", "fcm"] and 'senderID' not in result.get('router_data', {}).get("creds", {})): - # Make sure we note that this record is bad. - result['critical_failure'] = \ - result.get('critical_failure', "Missing SenderID") - db.router.register_user(result) + # Make sure we note that this record is bad. + result['critical_failure'] = \ + result.get('critical_failure', "Missing SenderID") + db.router.register_user(result) if result.get("critical_failure"): raise InvalidRequest("Critical Failure: %s" % diff --git a/docs/api.rst b/docs/api.rst index f505e321..1cf48237 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -18,6 +18,7 @@ documentation is organized alphabetically by module name. api/protocol api/router/apnsrouter api/router/gcm + api/router/gcmclient api/router/fcm api/router/interface api/web/base diff --git a/docs/api/router/gcmclient.rst b/docs/api/router/gcmclient.rst new file mode 100644 index 00000000..4c5c2720 --- /dev/null +++ b/docs/api/router/gcmclient.rst @@ -0,0 +1,25 @@ +.. _router_gcmclient_module: + +:mod:`autopush.router.gcmclient` +--------------------------------- + +.. automodule:: autopush.router.gcmclient + +.. autoclass:: GCM + :members: + :special-members: __init__ + :private-members: + :member-order: bysource + +.. autoclass:: JSONMessage + :members: + :special-members: __init__ + :private-members: + :member-order: bysource + +.. autoclass:: Result + :members: + :special-members: __init__ + :private-members: + :member-order: bysource + diff --git a/docs/api/web/base.rst b/docs/api/web/base.rst index ffd895bf..785fc595 100644 --- a/docs/api/web/base.rst +++ b/docs/api/web/base.rst @@ -10,11 +10,6 @@ :private-members: :member-order: bysource -.. autoclass:: Notification - :members: - :private-members: - :member-order: bysource - .. autoclass:: BaseWebHandler :members: :private-members: diff --git a/docs/index.rst b/docs/index.rst index 9bceecb6..694c3b3f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,7 @@ autopush .. image:: https://codecov.io/github/mozilla-services/autopush/coverage.svg :target: https://codecov.io/github/mozilla-services/autopush -Mozilla Push server and Push Endpoint utilizing PyPy, twisted, and DynamoDB. +Mozilla Push server and Push Endpoint utilizing PyPy, twisted, rust, and DynamoDB. This is the third generation of Push server built in Mozilla Services, first to handle Push for FirefoxOS clients, then extended for push notifications for @@ -77,6 +77,12 @@ All source code is available on `github under autopush api +We are using `rust `_ for a number of optimizations +and speed improvements. These efforts are ongoing and may be subject to +change. Unfortunately, this also means that formal documentation is not +yet available. You are, of course, welcome to review the code located in +`./autopush_rs`. + Changelog =========