Merge pull request #7439 from fstagni/90_sb_removeExt

[9.0] SandboxStore: remove external SE feature

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Services/SandboxStore/index.rst

@@ -8,15 +8,8 @@ Some extra options are required to configure this service:
 +---------------------------+----------------------------------------------+-----------------------------------------+
 | **Name**                  | **Description**                              | **Example**                             |
 +---------------------------+----------------------------------------------+-----------------------------------------+
-| *Backend*                 |                                              | Backend = local                         |
-+---------------------------+----------------------------------------------+-----------------------------------------+
 | *BasePath*                | Base path where the files are stored         | BasePath = /opt/dirac/storage/sandboxes |
 |                           | task queues in the system                    |                                         |
 +---------------------------+----------------------------------------------+-----------------------------------------+
-| *DelayedExternalDeletion* | Boolean used to define if the external       | DelayedExternalDeletion = True          |
-|                           | deletion must be done                        |                                         |
-+---------------------------+----------------------------------------------+-----------------------------------------+
 | *MaxSandboxSize*          | Maximum size of sanbox files expressed in MB | MaxSandboxSize = 10                     |
 +---------------------------+----------------------------------------------+-----------------------------------------+
-| *SandboxPrefix*           | Path prefix where sandbox are stored         | SandboxPrefix = Sandbox                 |
-+---------------------------+----------------------------------------------+-----------------------------------------+

src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py

@@ -65,14 +65,6 @@ def __getTransferClient(self):
             return self.__transferClient
         return TransferClient(self.__serviceName, **self.__kwargs)
 
-    # Upload sandbox to jobs and pilots
-
-    def uploadFilesAsSandboxForJob(self, fileList, jobId, sbType, sizeLimit=0):
-        """Upload SB for a job"""
-        if sbType not in self.__validSandboxTypes:
-            return S_ERROR(f"Invalid Sandbox type {sbType}")
-        return self.uploadFilesAsSandbox(fileList, sizeLimit, assignTo={f"Job:{jobId}": sbType})
-
     # Upload generic sandbox
 
     def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None):

src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg

@@ -136,12 +136,8 @@ Services
     Port = 9196
     LocalSE = ProductionSandboxSE
     MaxThreads = 200
-    toClientMaxThreads = 100
-    Backend = local
     MaxSandboxSizeMiB = 10
-    SandboxPrefix = Sandbox
     BasePath = /opt/dirac/storage/sandboxes
-    DelayedExternalDeletion = True
     # If true, uploads the sandbox via diracx on an S3 storage
     UseDiracXBackend = False
     Authorization
@@ -160,12 +156,8 @@ Services
     Protocol = https
     LocalSE = ProductionSandboxSE
     MaxThreads = 200
-    toClientMaxThreads = 100
-    Backend = local
     MaxSandboxSizeMiB = 10
-    SandboxPrefix = Sandbox
     BasePath = /opt/dirac/storage/sandboxes
-    DelayedExternalDeletion = True
     Authorization
     {
       Default = authenticated

src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py

@@ -128,7 +128,10 @@ def registerAndGetSandbox(self, owner, ownerGroup, sbSE, sbPFN, size=0):
             if not result["Value"]:
                 return S_ERROR("SandBox already exists but doesn't belong to the user")
             sbId = result["Value"][0][0]
-            self.accessedSandboxById(sbId)
+            if not (
+                ret := self._update(f"UPDATE `sb_SandBoxes` SET LastAccessTime=UTC_TIMESTAMP() WHERE SBId = {sbId}")
+            )["OK"]:
+                return ret
             return S_OK((sbId, False))
         # Inserted, time to get the id
         if "lastRowId" in result:
@@ -142,11 +145,7 @@ def accessedSandboxById(self, sbId):
         """
         Update last access time for sb id
         """
-        return self.__accessedSandboxByCond({"SBId": sbId})
-
-    def __accessedSandboxByCond(self, condDict):
-        sqlCond = [f"{key}={condDict[key]}" for key in condDict]
-        return self._update(f"UPDATE `sb_SandBoxes` SET LastAccessTime=UTC_TIMESTAMP() WHERE {' AND '.join(sqlCond)}")
+        return self._update(f"UPDATE `sb_SandBoxes` SET LastAccessTime=UTC_TIMESTAMP() WHERE SBId = {sbId}")
 
     def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, ownerName="", ownerGroup=""):
         """
@@ -184,8 +183,8 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, owner
         sbIds = []
         assigned = 0
         for entityId, SBType, SEName, SEPFN in entitiesToSandboxList:
-            result = self.getSandboxId(SEName, SEPFN, requesterName, requesterGroup)
             insertValues = []
+            result = self.getSandboxId(SEName, SEPFN, requesterName, requesterGroup)
             if not result["OK"]:
                 self.log.warn(
                     f"Cannot find id for {SEName}:",
@@ -205,8 +204,7 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, owner
 
             if not insertValues:
                 return S_ERROR(
-                    "Sandbox does not exist or you're not authorized to assign it being %s@%s"
-                    % (requesterName, requesterGroup)
+                    f"Sandbox does not exist or you're not authorized to assign it being {requesterName}@{requesterGroup}"
                 )
             sqlCmd = f"INSERT INTO `sb_EntityMapping` ( entityId, Type, SBId ) VALUES {', '.join(insertValues)}"
             result = self._update(sqlCmd)
@@ -215,8 +213,7 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, owner
                     return result
             assigned += 1
         sqlCmd = f"UPDATE `sb_SandBoxes` SET Assigned=1 WHERE SBId in ( {', '.join(sbIds)} )"
-        result = self._update(sqlCmd)
-        if not result["OK"]:
+        if not (result := self._update(sqlCmd))["OK"]:
             return result
         return S_OK(assigned)
 
@@ -380,12 +377,12 @@ def getSandboxOwner(self, SEName, SEPFN, requesterDN, requesterGroup):
 
         :returns: S_OK with tuple (owner, ownerGroup)
         """
-        res = self.getSandboxId(SEName, SEPFN, None, requesterGroup, "OwnerId", requesterDN=requesterDN)
-        if not res["OK"]:
+        if not (res := self.getSandboxId(SEName, SEPFN, None, requesterGroup, "OwnerId", requesterDN=requesterDN))[
+            "OK"
+        ]:
             return res
 
         sqlCmd = "SELECT `Owner`, `OwnerGroup` FROM `sb_Owners` WHERE `OwnerId` = %d" % res["Value"]
-        res = self._query(sqlCmd)
-        if not res["OK"]:
+        if not (res := self._query(sqlCmd))["OK"]:
             return res
         return S_OK(res["Value"][0])

src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py

@@ -1,5 +1,5 @@
 """ The Job Sanity executor assigns sandboxes to the job """
-from DIRAC import S_OK, S_ERROR
+from DIRAC import S_OK
 from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient
 from DIRAC.WorkloadManagementSystem.Executor.Base.OptimizerExecutor import OptimizerExecutor
 
@@ -59,7 +59,7 @@ def checkInputSandbox(self, jobState, manifest):
         result = self.sandboxClient.assignSandboxesToJob(jobState.jid, sbsToAssign, ownerName, ownerGroup)
         if not result["OK"]:
             self.jobLog.error("Could not assign sandboxes in the SandboxStore")
-            return S_ERROR("Cannot assign sandbox to job")
+            return result
         assigned = result["Value"]
         if assigned != numSBsToAssign:
             self.jobLog.error("Could not assign all sandboxes", f"({numSBsToAssign}). Only assigned {assigned}")