feat: address HTTP2 errors in APNS

Converts two errors into metrics and attempts retry on Connection or
HTTP2 errors.

Closes #1052

autopush/router/apnsrouter.py

@@ -64,6 +64,7 @@ def __init__(self, conf, router_conf, metrics, load_connections=True):
         self.metrics = metrics
         self._base_tags = ["platform:apns"]
         self.apns = dict()
+        self._max_retry = router_conf.get("max_retry", 2)
         for rel_channel in router_conf:
             self.apns[rel_channel] = self._connect(rel_channel,
                                                    load_connections)
@@ -139,31 +140,39 @@ def _route(self, notification, router_data):
                 "alert": {"title": " ", "body": " "}
             })
         apns_id = str(uuid.uuid4()).lower()
-        try:
-            apns_client.send(router_token=router_token, payload=payload,
-                             apns_id=apns_id)
-        except (ConnectionError, AttributeError) as ex:
-            self.metrics.increment("notification.bridge.error",
-                                   tags=make_tags(self._base_tags,
-                                                  application=rel_channel,
-                                                  reason="connection_error"))
-            self.log.error("Connection Error sending to APNS",
-                           log_failure=Failure(ex))
+        # APNs may force close a connection on us without warning.
+        # if that happens, retry the message.
+        success = False
+        for attempt in range(0, self._max_retry):
+            try:
+                apns_client.send(router_token=router_token, payload=payload,
+                                 apns_id=apns_id)
+                success = True
+                break
+            except AttributeError as ex:
+                self.log.error("Connection Error sending to APNS",
+                               log_failure=Failure(ex))
+                break
+            except ConnectionError:
+                self.metrics.increment("notification.bridge.connection.error",
+                                       tags=make_tags(
+                                           self._base_tags,
+                                           application=rel_channel,
+                                           attempt=attempt,
+                                           reason="connection_error"))
+            except HTTP20Error:
+                self.metrics.increment("notification.bridge.connection.error",
+                                       tags=make_tags(self._base_tags,
+                                                      application=rel_channel,
+                                                      attempt=attempt,
+                                                      reason="http2_error"))
+        if not success:
             raise RouterException(
                 "Server error",
                 status_code=502,
                 response_body="APNS returned an error processing request",
                 log_exception=False,
             )
-        except HTTP20Error as ex:
-            self.log.error("HTTP2 Error sending to APNS",
-                           log_failure=Failure(ex))
-            raise RouterException(
-                "Server error",
-                status_code=502,
-                response_body="APNS returned an error processing request",
-            )
-
         location = "%s/m/%s" % (self.conf.endpoint_url, notification.version)
         self.metrics.increment("notification.bridge.sent",
                                tags=make_tags(self._base_tags,