feat: address HTTP2 errors in APNS

Converts two errors into metrics and attempts retry on Connection or
HTTP2 errors.

Closes #1052

autopush/router/apnsrouter.py

@@ -5,7 +5,6 @@
 from hyper.http20.exceptions import ConnectionError, HTTP20Error
 from twisted.internet.threads import deferToThread
 from twisted.logger import Logger
-from twisted.python.failure import Failure
 
 from autopush.exceptions import RouterException
 from autopush.metrics import make_tags
@@ -64,6 +63,7 @@ def __init__(self, conf, router_conf, metrics, load_connections=True):
         self.metrics = metrics
         self._base_tags = ["platform:apns"]
         self.apns = dict()
+        self._max_retry = router_conf.get("max_retry", 2)
         for rel_channel in router_conf:
             self.apns[rel_channel] = self._connect(rel_channel,
                                                    load_connections)
@@ -139,31 +139,35 @@ def _route(self, notification, router_data):
                 "alert": {"title": " ", "body": " "}
             })
         apns_id = str(uuid.uuid4()).lower()
-        try:
-            apns_client.send(router_token=router_token, payload=payload,
-                             apns_id=apns_id)
-        except (ConnectionError, AttributeError) as ex:
-            self.metrics.increment("notification.bridge.error",
-                                   tags=make_tags(self._base_tags,
-                                                  application=rel_channel,
-                                                  reason="connection_error"))
-            self.log.error("Connection Error sending to APNS",
-                           log_failure=Failure(ex))
+        # APNs may force close a connection on us without warning.
+        # if that happens, retry the message.
+        success = False
+        for attempt in range(self._max_retry):
+            try:
+                apns_client.send(router_token=router_token, payload=payload,
+                                 apns_id=apns_id)
+                success = True
+                break
+            except ConnectionError:
+                self.metrics.increment("notification.bridge.connection.error",
+                                       tags=make_tags(
+                                           self._base_tags,
+                                           application=rel_channel,
+                                           attempt=attempt,
+                                           reason="connection_error"))
+            except HTTP20Error:
+                self.metrics.increment("notification.bridge.connection.error",
+                                       tags=make_tags(self._base_tags,
+                                                      application=rel_channel,
+                                                      attempt=attempt,
+                                                      reason="http2_error"))
+        if not success:
             raise RouterException(
                 "Server error",
                 status_code=502,
                 response_body="APNS returned an error processing request",
                 log_exception=False,
             )
-        except HTTP20Error as ex:
-            self.log.error("HTTP2 Error sending to APNS",
-                           log_failure=Failure(ex))
-            raise RouterException(
-                "Server error",
-                status_code=502,
-                response_body="APNS returned an error processing request",
-            )
-
         location = "%s/m/%s" % (self.conf.endpoint_url, notification.version)
         self.metrics.increment("notification.bridge.sent",
                                tags=make_tags(self._base_tags,

configs/autopush_shared.ini.sample

@@ -125,7 +125,8 @@ endpoint_port = 8082
 ;        "topic": "com.mozilla.org.Firefox",  // Bundle ID for associated app
 ;        "max_connections": 100, // Max number of connection pool entries.
 ;        "sandbox": False},  // Use the APNs sandbox feature
+         "max_retry": 2, // Max number of retries in event of an HTTP2 error
 ;     ... }
-; e.g {"firefox":{"cert":"certs/main.cert","key":"certs/main.key","topic":"com.mozilla.org.Firefox"},"beta":{"cert":"certs/beta.cert","key":"certs/beta.key","topic":"com.mozilla.org.FirefoxBeta"}}
+; e.g {"firefox":{"cert":"certs/main.cert","key":"certs/main.key","topic":"com.mozilla.org.Firefox","max_retry":2},"beta":{"cert":"certs/beta.cert","key":"certs/beta.key","topic":"com.mozilla.org.FirefoxBeta"}}
 #apns_creds =