Allow config override for backup constants

rqd/rqd/rqconstants.py

@@ -164,7 +164,7 @@
 
 # Backup running frames cache. Backup cache is turned off if this path is set to
 # None or ""
-BACKUP_CACHE_PATH = "/tmp/opencue/running_frames_backup.dat"
+BACKUP_CACHE_PATH = ""
 BACKUP_CACHE_TIME_TO_LIVE_SECONDS = 60
 
 try:
@@ -248,6 +248,11 @@
         if config.has_section(__host_env_var_section):
             RQD_HOST_ENV_VARS = config.options(__host_env_var_section)
 
+        if config.has_option(__override_section, "BACKUP_CACHE_PATH"):
+            BACKUP_CACHE_PATH = config.get(__override_section, "BACKUP_CACHE_PATH")
+        if config.has_option(__override_section, "BACKUP_CACHE_TIME_TO_LIVE_SECONDS"):
+            BACKUP_CACHE_TIME_TO_LIVE_SECONDS = config.getint(__override_section, "BACKUP_CACHE_TIME_TO_LIVE_SECONDS")
+
         __docker_mounts = "docker.mounts"
         __docker_config = "docker.config"
         __docker_images = "docker.images"

rqd/rqd/rqcore.py

@@ -113,6 +113,7 @@ def __init__(self, optNimbyoff=False):
                 self.backup_cache_path = rqd.rqconstants.BACKUP_CACHE_PATH
                 if not os.path.exists(os.path.dirname(self.backup_cache_path)):
                     os.makedirs(os.path.dirname(self.backup_cache_path))
+                self.recoverCache()
 
         signal.signal(signal.SIGINT, self.handleExit)
         signal.signal(signal.SIGTERM, self.handleExit)
@@ -232,6 +233,7 @@ def recoverCache(self):
                 # Ignore frames that failed to be parsed
                 try:
                     run_frame.ParseFromString(message_data)
+                    log.warning("Recovered frame %s.%s", run_frame.job_name, run_frame.frame_name)
                     running_frame = rqd.rqnetwork.RunningFrame(self, run_frame)
                     running_frame.frameAttendantThread = FrameAttendantThread(
                         self, run_frame, running_frame, recovery_mode=True)
@@ -1484,6 +1486,8 @@ def recoverDocker(self):
             raise RuntimeError("Invalid state: recovered frame does't contain a container id")
         container_id = runFrame.attributes.get("container_id")
 
+        # Recovered frame will stream back logs into a new file, therefore write a new header
+        self.__writeHeader()
         try:
             log_stream = None
             with self.rqCore.docker_lock: