From b06c6a7519b2b5142bee57954c7bf5c4367ddc71 Mon Sep 17 00:00:00 2001 From: jrconlin Date: Wed, 20 Jul 2016 12:58:10 -0700 Subject: [PATCH] feat: Add FCM router support Adds 'fcm' as a client selectable router, to allow 'gcm' to persist or act as a fallback. closes #517 --- autopush/main.py | 33 +++++ autopush/router/__init__.py | 4 +- autopush/router/fcm.py | 165 +++++++++++++++++++++++++ autopush/tests/test_main.py | 18 +++ autopush/tests/test_router.py | 219 +++++++++++++++++++++++++++++++++- configs/autopush_shared.ini | 9 ++ docs/api.rst | 1 + docs/api/router/fcm.rst | 12 ++ docs/http.rst | 16 +-- docs/install.rst | 11 +- 10 files changed, 472 insertions(+), 16 deletions(-) create mode 100644 autopush/router/fcm.py create mode 100644 docs/api/router/fcm.rst diff --git a/autopush/main.py b/autopush/main.py index 096050f1..0fc91160 100644 --- a/autopush/main.py +++ b/autopush/main.py @@ -165,6 +165,25 @@ def add_external_router_args(parser): env_var="GCM_COLLAPSEKEY") parser.add_argument('--senderid_list', help='SenderIDs to load to S3', type=str, default="{}") + # FCM + parser.add_argument('--fcm_enabled', help="Enable FCM Bridge", + action="store_true", default=False, + env_var="FCM_ENABLED") + label = "FCM Router:" + parser.add_argument('--fcm_ttl', help="%s Time to Live" % label, + type=int, default=60, env_var="FCM_TTL") + parser.add_argument('--fcm_dryrun', + help="%s Dry run (no message sent)" % label, + action="store_true", default=False, + env_var="FCM_DRYRUN") + parser.add_argument('--fcm_collapsekey', + help="%s string to collapse messages" % label, + type=str, default="simplepush", + env_var="FCM_COLLAPSEKEY") + parser.add_argument('--fcm_auth', help='Auth Key for FCM', + type=str, default="") + parser.add_argument('--fcm_senderid', help='SenderID for FCM', + type=str, default="") # Apple Push Notification system (APNs) for iOS parser.add_argument('--apns_enabled', help="Enable APNS Bridge", action="store_true", default=False, @@ -311,6 +330,20 @@ def make_settings(args, **kwargs): "collapsekey": args.gcm_collapsekey, "senderIDs": senderIDs, "senderid_list": args.senderid_list} + if args.fcm_enabled: + # Create a common gcmclient + if not args.fcm_auth: + log.critical(format="No Authorization Key found for FCM") + return + if not args.fcm_senderid: + log.critical(format="No SenderID found for FCM") + return + router_conf["fcm"] = {"ttl": args.fcm_ttl, + "dryrun": args.fcm_dryrun, + "max_data": args.max_data, + "collapsekey": args.fcm_collapsekey, + "auth": args.fcm_auth, + "senderid": args.fcm_senderid} ami_id = None # Not a fan of double negatives, but this makes more understandable args diff --git a/autopush/router/__init__.py b/autopush/router/__init__.py index cb52e35e..e83db84c 100644 --- a/autopush/router/__init__.py +++ b/autopush/router/__init__.py @@ -8,5 +8,7 @@ from autopush.router.gcm import GCMRouter from autopush.router.simple import SimpleRouter from autopush.router.webpush import WebPushRouter +from autopush.router.fcm import FCMRouter -__all__ = ["APNSRouter", "GCMRouter", "SimpleRouter", "WebPushRouter"] +__all__ = ["APNSRouter", "FCMRouter", "GCMRouter", "SimpleRouter", + "WebPushRouter"] diff --git a/autopush/router/fcm.py b/autopush/router/fcm.py new file mode 100644 index 00000000..fd602528 --- /dev/null +++ b/autopush/router/fcm.py @@ -0,0 +1,165 @@ +"""FCM Router""" +import gcmclient +import json + +from twisted.internet.threads import deferToThread +from twisted.logger import Logger + +from autopush.router.interface import RouterException, RouterResponse + + +class FCMRouter(object): + """FCM Router Implementation + + Note: FCM is a newer branch of GCM. While there's not much change + required for the server, there is significant work required for the + client. To that end, having a separate router allows the "older" GCM + to persist and lets the client determine when they want to use the + newer FCM route. + """ + log = Logger() + gcm = None + dryRun = 0 + collapseKey = "simplepush" + + def __init__(self, ap_settings, router_conf): + """Create a new FCM router and connect to FCM""" + self.config = router_conf + self.min_ttl = router_conf.get("ttl", 60) + self.dryRun = router_conf.get("dryrun", False) + self.collapseKey = router_conf.get("collapseKey", "webpush") + self.senderID = router_conf.get("senderID") + self.auth = router_conf.get("auth") + self.metrics = ap_settings.metrics + self._base_tags = [] + try: + self.fcm = gcmclient.GCM(self.auth) + except Exception as e: + self.log.error("Could not instantiate FCM {ex}", + ex=e) + raise IOError("FCM Bridge not initiated in main") + self.log.debug("Starting FCM router...") + + def amend_msg(self, msg, data=None): + if data is not None: + msg["senderid"] = data.get('creds', {}).get('senderID') + return msg + + def register(self, uaid, router_data, router_token=None, *kwargs): + """Validate that the FCM Instance Token is in the ``router_data``""" + if "token" not in router_data: + raise self._error("connect info missing FCM Instance 'token'", + status=401) + # Assign a senderid + router_data["creds"] = {"senderID": self.senderID, "auth": self.auth} + return router_data + + def route_notification(self, notification, uaid_data): + """Start the FCM notification routing, returns a deferred""" + router_data = uaid_data["router_data"] + # Kick the entire notification routing off to a thread + return deferToThread(self._route, notification, router_data) + + def _route(self, notification, router_data): + """Blocking FCM call to route the notification""" + data = {"chid": notification.channel_id} + # Payload data is optional. The endpoint handler validates that the + # correct encryption headers are included with the data. + if notification.data: + mdata = self.config.get('max_data', 4096) + if len(notification.data) > mdata: + raise self._error("This message is intended for a " + + "constrained device and is limited " + + "to 3070 bytes. Converted buffer too " + + "long by %d bytes" % + (len(notification.data) - mdata), + 413, errno=104) + + data['body'] = notification.data + data['con'] = notification.headers['content-encoding'] + data['enc'] = notification.headers['encryption'] + + if 'crypto-key' in notification.headers: + data['cryptokey'] = notification.headers['crypto-key'] + elif 'encryption-key' in notification.headers: + data['enckey'] = notification.headers['encryption-key'] + + # registration_ids are the FCM instance tokens (specified during + # registration. + router_ttl = notification.ttl or 0 + payload = gcmclient.JSONMessage( + registration_ids=[router_data.get("token")], + collapse_key=self.collapseKey, + time_to_live=max(self.min_ttl, router_ttl), + dry_run=self.dryRun or ("dryrun" in router_data), + data=data, + ) + creds = router_data.get("creds", {"senderID": "missing id"}) + try: + self.fcm.api_key = creds["auth"] + result = self.fcm.send(payload) + except KeyError: + raise self._error("Server error, missing bridge credentials " + + "for %s" % creds.get("senderID"), 500) + except gcmclient.GCMAuthenticationError as e: + raise self._error("Authentication Error: %s" % e, 500) + except Exception as e: + raise self._error("Unhandled exception in FCM Routing: %s" % e, + 500) + self.metrics.increment("updates.client.bridge.gcm.attempted", + self._base_tags) + return self._process_reply(result) + + def _error(self, err, status, **kwargs): + """Error handler that raises the RouterException""" + self.log.debug(err, **kwargs) + return RouterException(err, status_code=status, response_body=err, + **kwargs) + + def _process_reply(self, reply): + """Process FCM send reply""" + # acks: + # for reg_id, msg_id in reply.success.items(): + # updates + for old_id, new_id in reply.canonical.items(): + self.log.info("FCM id changed : {old} => {new}", + old=old_id, new=new_id) + self.metrics.increment("updates.client.bridge.gcm.failed.rereg", + self._base_tags) + return RouterResponse(status_code=503, + response_body="Please try request again.", + router_data=dict(token=new_id)) + # naks: + # uninstall: + for reg_id in reply.not_registered: + self.metrics.increment("updates.client.bridge.gcm.failed.unreg", + self._base_tags) + self.log.info("FCM no longer registered: %s" % reg_id) + return RouterResponse( + status_code=410, + response_body="Endpoint requires client update", + router_data={}, + ) + + # for reg_id, err_code in reply.failed.items(): + if len(reply.failed.items()) > 0: + self.metrics.increment("updates.client.bridge.gcm.failed.failure", + self._base_tags) + self.log.critical("FCM failures: {failed()}", + failed=lambda: json.dumps(reply.failed.items())) + raise RouterException("FCM failure to deliver", status_code=503, + response_body="Please try request later.") + + # retries: + if reply.needs_retry(): + self.log.warn("FCM retry requested: {failed()}", + failed=lambda: json.dumps(reply.failed.items())) + self.metrics.increment("updates.client.bridge.gcm.failed.retry", + self._base_tags) + raise RouterException("FCM failure to deliver, retry", + status_code=503, + response_body="Please try request later.") + + self.metrics.increment("updates.client.bridge.gcm.succeeded", + self._base_tags) + return RouterResponse(status_code=200, response_body="Message Sent") diff --git a/autopush/tests/test_main.py b/autopush/tests/test_main.py index 4e41eb5a..190015fb 100644 --- a/autopush/tests/test_main.py +++ b/autopush/tests/test_main.py @@ -220,6 +220,12 @@ class test_arg: senderid_list = '{"12345":{"auth":"abcd"}}' key_hash = "supersikkret" no_aws = True + fcm_enabled = True + fcm_ttl = 999 + fcm_dryrun = False + fcm_collapsekey = "collapse" + fcm_senderid = '12345' + fcm_auth = 'abcde' def setUp(self): mock_s3().start() @@ -272,6 +278,18 @@ def test_bad_senders(self): eq_(ap, None) self.test_arg.senderid_list = oldList + def test_bad_fcm_senders(self): + old_auth = self.test_arg.fcm_auth + old_senderid = self.test_arg.fcm_senderid + self.test_arg.fcm_auth = "" + ap = make_settings(self.test_arg) + eq_(ap, None) + self.test_arg.fcm_auth = old_auth + self.test_arg.fcm_senderid = "" + ap = make_settings(self.test_arg) + eq_(ap, None) + self.test_arg.fcm_senderid = old_senderid + def test_gcm_start(self): endpoint_main([ "--gcm_enabled", diff --git a/autopush/tests/test_router.py b/autopush/tests/test_router.py index f9394bf8..9de2cdad 100644 --- a/autopush/tests/test_router.py +++ b/autopush/tests/test_router.py @@ -21,7 +21,9 @@ create_rotating_message_table, ) from autopush.endpoint import Notification -from autopush.router import APNSRouter, GCMRouter, SimpleRouter, WebPushRouter +from autopush.router import (APNSRouter, GCMRouter, + SimpleRouter, WebPushRouter, + FCMRouter) from autopush.router.simple import dead_cache from autopush.router.interface import RouterException, RouterResponse, IRouter from autopush.settings import AutopushSettings @@ -238,7 +240,6 @@ def _check_error_call(self, exc, code): self.flushLoggedErrors() def test_init(self): - settings = AutopushSettings( hostname="localhost", statsd_host=None, @@ -409,6 +410,220 @@ def test_ammend(self): result) +class FCMRouterTestCase(unittest.TestCase): + + @patch("gcmclient.GCM", spec=gcmclient.gcm.GCM) + def setUp(self, ffcm): + settings = AutopushSettings( + hostname="localhost", + statsd_host=None, + ) + self.fcm_config = {'s3_bucket': 'None', + 'max_data': 32, + 'ttl': 60, + 'senderID': 'test123', + "auth": "12345678abcdefg"} + self.fcm = ffcm + self.router = FCMRouter(settings, self.fcm_config) + self.headers = {"content-encoding": "aesfcm", + "encryption": "test", + "encryption-key": "test"} + # Payloads are Base64-encoded. + self.notif = Notification(10, "q60d6g", dummy_chid, self.headers, + 200) + self.router_data = dict( + router_data=dict( + token="connect_data", + creds=dict(senderID="test123", auth="12345678abcdefg"))) + mock_result = Mock(spec=gcmclient.gcm.Result) + mock_result.canonical = dict() + mock_result.failed = dict() + mock_result.not_registered = dict() + mock_result.needs_retry.return_value = False + self.mock_result = mock_result + ffcm.send.return_value = mock_result + + def _check_error_call(self, exc, code): + ok_(isinstance(exc, RouterException)) + eq_(exc.status_code, code) + assert(self.router.fcm.send.called) + self.flushLoggedErrors() + + @patch("gcmclient.GCM", spec=gcmclient.gcm.GCM) + def test_init(self, fgcm): + settings = AutopushSettings( + hostname="localhost", + statsd_host=None, + ) + + def throw_auth(arg): + raise gcmclient.GCMAuthenticationError() + fgcm.side_effect = throw_auth + self.assertRaises(IOError, FCMRouter, settings, {}) + + def test_register(self): + result = self.router.register("uaid", {"token": "connect_data"}) + # Check the information that will be recorded for this user + eq_(result, {"token": "connect_data", + "creds": {"senderID": "test123", + "auth": "12345678abcdefg"}}) + + def test_register_bad(self): + self.assertRaises(RouterException, self.router.register, "uaid", {}) + + def test_route_notification(self): + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(result): + ok_(isinstance(result, RouterResponse)) + assert(self.router.fcm.send.called) + # Make sure the data was encoded as base64 + data = self.router.fcm.send.call_args[0][0].data + eq_(data['body'], 'q60d6g') + eq_(data['enc'], 'test') + eq_(data['enckey'], 'test') + eq_(data['con'], 'aesfcm') + d.addCallback(check_results) + return d + + def test_ttl_none(self): + self.router.fcm = self.fcm + self.notif = Notification(version=10, + data="q60d6g", + channel_id=dummy_chid, + headers=self.headers, + ttl=None) + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(result): + ok_(isinstance(result, RouterResponse)) + assert(self.router.fcm.send.called) + # Make sure the data was encoded as base64 + data = self.router.fcm.send.call_args[0][0].data + options = self.router.fcm.send.call_args[0][0].options + eq_(data['body'], 'q60d6g') + eq_(data['enc'], 'test') + eq_(data['enckey'], 'test') + eq_(data['con'], 'aesfcm') + # use the defined min TTL + eq_(options['time_to_live'], 60) + d.addCallback(check_results) + return d + + def test_long_data(self): + self.router.fcm = self.fcm + badNotif = Notification( + 10, "\x01abcdefghijklmnopqrstuvwxyz0123456789", dummy_chid, + self.headers, 200) + d = self.router.route_notification(badNotif, self.router_data) + + def check_results(result): + ok_(isinstance(result.value, RouterException)) + eq_(result.value.status_code, 413) + eq_(result.value.errno, 104) + + d.addBoth(check_results) + return d + + def test_route_crypto_notification(self): + self.router.fcm = self.fcm + del(self.notif.headers['encryption-key']) + self.notif.headers['crypto-key'] = 'crypto' + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(result): + ok_(isinstance(result, RouterResponse)) + assert(self.router.fcm.send.called) + d.addCallback(check_results) + return d + + def test_router_notification_fcm_auth_error(self): + def throw_auth(arg): + raise gcmclient.GCMAuthenticationError() + self.fcm.send.side_effect = throw_auth + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(fail): + self._check_error_call(fail.value, 500) + d.addBoth(check_results) + return d + + def test_router_notification_fcm_other_error(self): + def throw_other(arg): + raise Exception("oh my!") + self.fcm.send.side_effect = throw_other + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(fail): + self._check_error_call(fail.value, 500) + d.addBoth(check_results) + return d + + def test_router_notification_fcm_id_change(self): + self.mock_result.canonical["old"] = "new" + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(result): + ok_(isinstance(result, RouterResponse)) + eq_(result.router_data, dict(token="new")) + assert(self.router.fcm.send.called) + d.addCallback(check_results) + return d + + def test_router_notification_fcm_not_regged(self): + self.mock_result.not_registered = {"connect_data": True} + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(result): + ok_(isinstance(result, RouterResponse)) + eq_(result.router_data, dict()) + assert(self.router.fcm.send.called) + d.addCallback(check_results) + return d + + def test_router_notification_fcm_failed_items(self): + self.mock_result.failed = dict(connect_data=True) + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(fail): + self._check_error_call(fail.value, 503) + d.addBoth(check_results) + return d + + def test_router_notification_fcm_needs_retry(self): + self.mock_result.needs_retry.return_value = True + self.router.fcm = self.fcm + d = self.router.route_notification(self.notif, self.router_data) + + def check_results(fail): + self._check_error_call(fail.value, 503) + d.addBoth(check_results) + return d + + def test_router_notification_fcm_no_auth(self): + d = self.router.route_notification(self.notif, + {"router_data": {"token": "abc"}}) + + def check_results(fail): + eq_(fail.value.status_code, 500) + d.addBoth(check_results) + return d + + def test_ammend(self): + self.router.register("uaid", {"token": "connect_data"}) + resp = {"key": "value"} + result = self.router.amend_msg(resp, + self.router_data.get('router_data')) + eq_({"key": "value", "senderid": "test123"}, + result) + + class SimplePushRouterTestCase(unittest.TestCase): def setUp(self): from twisted.logger import Logger diff --git a/configs/autopush_shared.ini b/configs/autopush_shared.ini index 696d4383..0c4d8e85 100644 --- a/configs/autopush_shared.ini +++ b/configs/autopush_shared.ini @@ -108,6 +108,15 @@ endpoint_port = 8082 ; {"12345": {"auth": "abcd_efg"}, "01357": {"auth": "ZYX=abc"}} #senderid_list = +; FCM is the later version of GCM +#fcm_enabled +; The FCM router only allows one senderid/auth key due to restrictions on +; the client +#fcm_collapsekey = simplepush +#fcm_senderid = +#fcm_auth = +#fcm_dryrun + ; Perform AWS specific information (like fetch the AMI ID from the meta-data ; server ; diff --git a/docs/api.rst b/docs/api.rst index b58170e7..1ebe3d06 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -17,6 +17,7 @@ documentation is organized alphabetically by module name. api/protocol api/router/apnsrouter api/router/gcm + api/router/fcm api/router/interface api/router/simple api/senderids diff --git a/docs/api/router/fcm.rst b/docs/api/router/fcm.rst new file mode 100644 index 00000000..499f4893 --- /dev/null +++ b/docs/api/router/fcm.rst @@ -0,0 +1,12 @@ +.. _router_fcm_module: + +:mod:`autopush.router.fcm` +-------------------------- + +.. automodule:: autopush.router.fcm + +.. autoclass:: FCMRouter + :members: + :special-members: __init__ + :private-members: + :member-order: bysource diff --git a/docs/http.rst b/docs/http.rst index 6d419744..9d9fc81b 100644 --- a/docs/http.rst +++ b/docs/http.rst @@ -144,7 +144,7 @@ Send a notification to the given endpoint identified by it's `token`. .. note:: Some bridged connections require data transcription and may limit the - length of data that can be sent. For instance, using a GCM bridge + length of data that can be sent. For instance, using a GCM/FCM bridge will require that the data be converted to base64. This means that data may be limited to only 2744 bytes instead of the normal 4096 bytes. @@ -242,7 +242,7 @@ Push Service Bridge HTTP Interface Push allows for remote devices to perform some functions using an HTTP interface. This is mostly used by devices that are bridging via an external protocol like -`GCM `__ or +`GCM `__/`FCM `__ or `APNs `__. All message bodies must be UTF-8 encoded. Lexicon @@ -252,13 +252,13 @@ For the following call definitions: :{type}: The bridge type. -Allowed bridges are `gcm` (Google Cloud Messaging) and `apns` (Apple -Push Notification system) +Allowed bridges are `gcm` (Google Cloud Messaging), `fcm` (Firebase Cloud +Messaging), and `apns` (Apple Push Notification system) :{token}: The bridge specific public exchange token Each protocol requires a unique token that addresses the remote application. -For GCM, this is the `SenderID` and is pre-negotiated outside of the push +For GCM/FCM, this is the `SenderID` and is pre-negotiated outside of the push service. :{instanceid}: The bridge specific private identifier token @@ -319,7 +319,7 @@ example: .. code-block:: http - > POST /v1/gcm/a1b2c3/registration + > POST /v1/fcm/a1b2c3/registration > > {"token": "1ab2c3"} @@ -371,7 +371,7 @@ example: .. code-block:: http - > PUT /v1/gcm/a1b2c3/registration/abcdef012345 + > PUT /v1/fcm/a1b2c3/registration/abcdef012345 > Authorization: Bearer 0123abcdef > > {"token": "5e6g7h8i"} @@ -415,7 +415,7 @@ example: .. code-block:: http - > POST /v1/gcm/a1b2c3/registration/abcdef012345/subscription + > POST /v1/fcm/a1b2c3/registration/abcdef012345/subscription > Authorization: Bearer 0123abcdef > > {} diff --git a/docs/install.rst b/docs/install.rst index 078e4cc5..b2463d02 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -62,13 +62,14 @@ environment variable, and add your OpenSSL library path to ``LDFLAGS`` and # `/usr/local`. export LDFLAGS="-L/usr/local/lib" CFLAGS="-I/usr/local/include" -Notes on GCM support +Notes on GCM/FCM support ==================== -autopush is capable of routing messages over Google Cloud Messaging for android -devices. You will need to set up a valid GCM `account `_. Once you have an account open the Google Developer Console: +autopush is capable of routing messages over Google Cloud Messaging/Firebase Cloud Messaging for android +devices. You will need to set up a valid `GCM `_ / `FCM `_ account. Once you have an account open the Google Developer Console: * create a new project. Record the Project Number as "SENDER_ID". You will need this value for your android application. -* create a new Auth Credential Key for your project. This is available under **APIs & Auth** >> **Credentials** of the Google Developer Console. Store this value as ``gcm_apikey`` in ``.autopush_endpoint`` server configuration file. +* create a new Auth Credential Key for your project. This is available under **APIs & Auth** >> **Credentials** of the Google Developer Console. Store this value as ``gcm_apikey`` or ``fcm_apikey`` (as appropriate) in ``.autopush_endpoint`` server configuration file. * add ``gcm_enabled`` to the ``.autopush_shared`` server configuration file to enable GCM routing. +* add ``fcm_enabled`` to the ``.autopush_shared`` server configuration file to enable FCM routing. -Additional notes on using the GCM bridge are available `on the wiki `_. +Additional notes on using the GCM/FCM bridge are available `on the wiki `_.