feat: improve firebase error handling

src/googleclouddebugger/firebase_client.py

@@ -14,6 +14,7 @@
 """Communicates with Firebase RTDB backend."""
 
 from collections import deque
+import copy
 import hashlib
 import json
 import os
@@ -22,6 +23,7 @@
 import socket
 import sys
 import threading
+import time
 import traceback
 
 import firebase_admin
@@ -114,6 +116,7 @@ def __init__(self):
 
     # Delay before retrying failed request.
     self.register_backoff = backoff.Backoff()  # Register debuggee.
+    self.subscribe_backoff = backoff.Backoff()  # Subscribe to updates.
     self.update_backoff = backoff.Backoff()  # Update breakpoint.
 
     # Maximum number of times that the message is re-transmitted before it
@@ -279,13 +282,19 @@ def _MainThreadProc(self):
     self._breakpoint_subscription.
     """
     # Note: if self._credentials is None, default app credentials will be used.
-    # TODO: Error handling.
     firebase_admin.initialize_app(self._credentials,
                                   {'databaseURL': self._database_url})
 
-    self._RegisterDebuggee()
+    registration_required, delay = True, 0
+    while registration_required:
+      time.sleep(delay)
+      registration_required, delay = self._RegisterDebuggee()
     self.registration_complete.set()
-    self._SubscribeToBreakpoints()
+
+    subscription_required, delay = True, 0
+    while subscription_required:
+      time.sleep(delay)
+      subscription_required, delay = self._SubscribeToBreakpoints()
     self.subscription_complete.set()
 
   def _TransmissionThreadProc(self):
@@ -324,10 +333,12 @@ def _RegisterDebuggee(self):
         self.register_backoff.Succeeded()
         return (False, 0)  # Proceed immediately to subscribing to breakpoints.
       except BaseException:
+        # There is no significant benefit to handing different exceptions
+        # in different ways; we will log and retry regardless.
         native.LogInfo(f'Failed to register debuggee: {traceback.format_exc()}')
     except BaseException:
-      native.LogWarning('Debuggee information not available: ' +
-                        traceback.format_exc())
+      native.LogWarning(
+          f'Debuggee information not available: {traceback.format_exc()}')
 
     return (True, self.register_backoff.Failed())
 
@@ -340,7 +351,13 @@ def _SubscribeToBreakpoints(self):
     path = f'cdbg/breakpoints/{self._debuggee_id}/active'
     native.LogInfo(f'Subscribing to breakpoint updates at {path}')
     ref = firebase_admin.db.reference(path)
-    self._breakpoint_subscription = ref.listen(self._ActiveBreakpointCallback)
+    try:
+      self._breakpoint_subscription = ref.listen(self._ActiveBreakpointCallback)
+      return (False, 0)
+    except firebase_admin.exceptions.FirebaseError:
+      native.LogInfo(
+          f'Failed to subscribe to breakpoints: {traceback.format_exc()}')
+      return (True, self.subscribe_backoff.Failed())
 
   def _ActiveBreakpointCallback(self, event):
     if event.event_type == 'put':
@@ -410,7 +427,7 @@ def _TransmitBreakpointUpdates(self):
       try:
         # Something has changed on the breakpoint.
         # It should be going from active to final, but let's make sure.
-        if not breakpoint_data['isFinalState']:
+        if not breakpoint_data.get('isFinalState'):
           raise BaseException(
               f'Unexpected breakpoint update requested: {breakpoint_data}')
 
@@ -428,6 +445,7 @@ def _TransmitBreakpointUpdates(self):
             f'cdbg/breakpoints/{self._debuggee_id}/active/{bp_id}')
         bp_ref.delete()
 
+        summary_data = breakpoint_data
         # Save snapshot data for snapshots only.
         if is_snapshot:
           # Note that there may not be snapshot data.
@@ -436,14 +454,15 @@ def _TransmitBreakpointUpdates(self):
           bp_ref.set(breakpoint_data)
 
           # Now strip potential snapshot data.
-          breakpoint_data.pop('evaluatedExpressions', None)
-          breakpoint_data.pop('stackFrames', None)
-          breakpoint_data.pop('variableTable', None)
+          summary_data = copy.deepcopy(breakpoint_data)
+          summary_data.pop('evaluatedExpressions', None)
+          summary_data.pop('stackFrames', None)
+          summary_data.pop('variableTable', None)
 
         # Then add it to the list of final breakpoints.
         bp_ref = firebase_admin.db.reference(
             f'cdbg/breakpoints/{self._debuggee_id}/final/{bp_id}')
-        bp_ref.set(breakpoint_data)
+        bp_ref.set(summary_data)
 
         native.LogInfo(f'Breakpoint {bp_id} update transmitted successfully')