feat: shut down agents gracefully

scripts/keri/cf/demo-witness-oobis.json

@@ -7,6 +7,9 @@
   "iurls": [
     "http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller?name=Wan&tag=witness",
     "http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller?name=Wes&tag=witness",
-    "http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller?name=Wil&tag=witness"
+    "http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller?name=Wil&tag=witness",
+    "http://127.0.0.1:5645/oobi/BM35JN8XeJSEfpxopjn5jr7tAHCE5749f0OobhMLCorE/controller?name=Wit&tag=witness",
+    "http://127.0.0.1:5646/oobi/BIj15u5V11bkbtAxMA7gcNJZcax-7TgaBMLsQnMHpYHP/controller?name=Wub&tag=witness",
+    "http://127.0.0.1:5647/oobi/BF2rZTW79z4IXocYRQnjjsOuvFUQv-ptCf8Yltd7PfsM/controller?name=Wyz&tag=witness"
   ]
 }

src/keria/app/agenting.py

@@ -6,13 +6,13 @@
 """
 from base64 import b64decode
 import json
-import os
 import datetime
 from dataclasses import asdict
 from urllib.parse import urlparse, urljoin
 from types import MappingProxyType
 
 import falcon
+import lmdb
 from falcon import media
 from hio.base import doing
 from hio.core import http, tcp
@@ -249,19 +249,31 @@ def delete(self, agent):
         del self.agents[agent.caid]
 
     def shut(self, agent):
-        logger.info(f"closing idle agent {agent.caid}")
+        logger.info(f"Shutting down agent {agent.caid}")
+        for doer in agent.doers:
+            if hasattr(doer, "close"):
+                doer.close()
+            elif callable(doer):  # If made from the "doify" function
+                agent.remove([doer])  # Removing doified doer
+            else:
+                agent.remove([doer])  # Removing any other doer
+        # Probably should wait for the doers to finish before closing the databases
+
         agent.remove(agent.doers)
         self.remove([agent])
         del self.agents[agent.caid]
-        agent.hby.ks.close(clear=False)
-        agent.seeker.close(clear=False)
-        agent.exnseeker.close(clear=False)
-        agent.monitor.opr.close(clear=False)
-        agent.notifier.noter.close(clear=False)
-        agent.rep.mbx.close(clear=False)
-        agent.registrar.rgy.close()
-        agent.mgr.rb.close(clear=False)
-        agent.hby.close(clear=False)
+        try:
+            agent.hby.ks.close(clear=False)
+            agent.seeker.close(clear=False)
+            agent.exnseeker.close(clear=False)
+            agent.monitor.opr.close(clear=False)
+            agent.notifier.noter.close(clear=False)
+            agent.rep.mbx.close(clear=False)
+            agent.registrar.rgy.close()
+            agent.mgr.rb.close(clear=False)
+            agent.hby.close(clear=False)
+        except lmdb.Error as ex:  # Sometimes LMDB will throw an error if the DB is already closed
+            logger.error(f"Error closing databases for agent {agent.caid}: {ex}")
 
     def get(self, caid):
         if caid in self.agents:

src/keria/app/cli/commands/sig-fix.py

@@ -6,9 +6,8 @@
 """
 import argparse
 
-from hio import help
 from hio.base import doing
-from keri import kering
+from keri import help, kering
 from keri.app import habbing
 from keri.app.cli.common import existing
 from keri.core import serdering, coring

src/keria/app/cli/commands/start.py

@@ -78,6 +78,8 @@
                     dest="bootUsername",
                     default=os.getenv("KERIA_EXPERIMENTAL_BOOT_USERNAME"))
 
+logger = help.ogler.getLogger()
+
 def getListVariable(name):
     value = os.getenv(name)
     return value.split(";") if value else None
@@ -88,12 +90,10 @@ def launch(args):
         help.ogler.headDirPath = args.logfile
         help.ogler.reopen(name=args.name, temp=False, clear=True)
 
-    logger = help.ogler.getLogger()
-
     logger.info("Starting Agent for %s listening: admin/%s, http/%s, boot/%s", args.name, args.admin, args.http, args.boot)
     logger.info("PID: %s", os.getpid())
 
-    agency = agenting.setup(name=args.name or "ahab",
+    doers = agenting.setup(name=args.name or "ahab",
                             base=args.base or "",
                             bran=args.bran,
                             adminPort=args.admin,
@@ -111,16 +111,27 @@ def launch(args):
                             durls=getListVariable("KERIA_DURLS"),
                             bootPassword=args.bootPassword,
                             bootUsername=args.bootUsername)
+    agency = None
+    for doer in doers:
+        if isinstance(doer, agenting.Agency):
+            agency = doer
+            break
 
     tock = 0.03125
     doist = doing.Doist(limit=0.0, tock=tock, real=True)
 
-    def handleSignal(sig, frame):
-        logger.info("Received signal %s", signal.strsignal(sig))
-        doist.exit()
-
-    signal.signal(signal.SIGTERM, handleSignal)
+    signal.signal(signal.SIGTERM, shutdown_handler_fn(agency, doist))
 
-    doist.do(doers=agency)
+    doist.do(doers=doers)
 
     logger.info("Agent %s gracefully stopped", args.name)
+
+def shutdown_handler_fn(agency, doist):
+    """Construct a shutdown handler function for a given agency and doist"""
+    def handle_sigterm(sig, frame):
+        agents = list(agency.agents.keys())
+        logger.info("Shutting down due to %s | stopping %s agents", signal.strsignal(sig), len(agents))
+        for caid in agents:
+            agency.shut(agency.agents[caid])
+        doist.exit()
+    return handle_sigterm