Merge pull request #7935 from sfayer/fix_cloudcleanup

[9.0] Run CE cleanup step at correct point

src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -750,6 +750,11 @@ def _monitorPilotsPerQueue(self, queue: str):
         ceType = self.queueDict[queue]["CEType"]
         siteName = self.queueDict[queue]["Site"]
 
+        # Call cleanup before checking pilot statuses (so cleanup always runs)
+        # This is needed to delete things like old cloud instances after the pilots are done
+        if callable(getattr(ce, "cleanupPilots", None)):
+            ce.cleanupPilots()
+
         # Select pilots in a transient states
         result = self.pilotAgentsDB.selectPilots(
             {
@@ -799,10 +804,6 @@ def _monitorPilotsPerQueue(self, queue: str):
         # Update the status of the pilots in the DB
         self._updatePilotsInDB(updatedPilots)
 
-        # FIXME: seems like it is only used by the CloudCE? Couldn't it be called from CloudCE.getJobStatus()?
-        if callable(getattr(ce, "cleanupPilots", None)):
-            ce.cleanupPilots()
-
         # Check if the accounting is to be sent
         if self.sendAccounting:
             result = self.pilotAgentsDB.selectPilots(