Limit the number of upload retries in Python and iOS (#968)

* Limit the number of upload retries in iOS

* Limit the number of upload retries in Python

glean-core/ios/Glean/Net/HttpPingUploader.swift

@@ -18,6 +18,9 @@ public class HttpPingUploader {
         static let recoverableErrorStatusCode: UInt16 = 500
         // For this error, the ping data will be deleted and no retry happens
         static let unrecoverableErrorStatusCode: UInt16 = 400
+
+        // Maximum number of recoverable errors allowed before aborting the ping uploader
+        static let maxRetries = 3
     }
 
     private let logger = Logger(tag: Constants.logTag)
@@ -99,14 +102,18 @@ public class HttpPingUploader {
     /// It will report back the task status to Glean, which will take care of deleting pending ping files.
     /// It will continue upload as long as it can fetch new tasks.
     func process() {
-        while true {
+        var uploadFailures = 0
+        while uploadFailures < Constants.maxRetries {
             var incomingTask = FfiPingUploadTask()
             glean_get_upload_task(&incomingTask, config.logPings.toByte())
             let task = incomingTask.toPingUploadTask()
 
             switch task {
             case let .upload(request):
                 self.upload(path: request.path, data: request.body, headers: request.headers) { result in
+                    if case .recoverableFailure = result {
+                        uploadFailures += 1
+                    }
                     glean_process_ping_upload_response(&incomingTask, result.toFfi())
                 }
             case .wait:

glean-core/python/glean/net/ping_upload_worker.py

@@ -16,6 +16,7 @@
 from .._glean_ffi import ffi as ffi_support  # type: ignore
 from .._dispatcher import Dispatcher
 from .._process_dispatcher import ProcessDispatcher
+from .ping_uploader import RecoverableFailure
 
 
 log = logging.getLogger(__name__)
@@ -24,6 +25,9 @@
 # How many times to attempt waiting when told to by glean-core's upload API.
 MAX_WAIT_ATTEMPTS = 3
 
+# Maximum number of recoverable errors allowed before aborting the ping uploader
+MAX_RETRIES = 3
+
 
 class PingUploadWorker:
     @classmethod
@@ -112,7 +116,9 @@ def _process(data_dir: Path, configuration) -> bool:
 
     wait_attempts = 0
 
-    while True:
+    upload_failures = 0
+
+    while upload_failures < MAX_RETRIES:
         incoming_task = ffi_support.new("FfiPingUploadTask *")
         _ffi.lib.glean_get_upload_task(incoming_task, configuration.log_pings)
 
@@ -136,6 +142,9 @@ def _process(data_dir: Path, configuration) -> bool:
                 url_path, body, _parse_ping_headers(headers, doc_id), configuration
             )
 
+            if isinstance(upload_result, RecoverableFailure):
+                upload_failures = upload_failures + 1
+
             # Process the response.
             _ffi.lib.glean_process_ping_upload_response(
                 incoming_task, upload_result.to_ffi()