From 5a459a5c1b3500783e4f66643b50206bc10c428b Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 22 Aug 2024 11:35:12 +0200 Subject: [PATCH 1/3] feat: added a DIRAC to GOCDB conversion --- .../Command/DowntimeCommand.py | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py index d810de83bcd..78024052a44 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py @@ -24,6 +24,27 @@ from DIRAC.ResourceStatusSystem.Command.Command import Command +# conversion from DIRAC resource type to GOCDB service type +diracToGOC_conversion = { + # Computing elements + "CREAM": "CREAM-CE", # deprecated + "HTCondorCE" :"org.opensciencegrid.htcondorce", + "ARC" : "ARC-CE", + "ARC6" : "ARC-CE", + "AREX" : "ARC-CE", + # FTS + "FTS3": "FTS", + "FTS": "FTS", + # Storage elements + "disk_srm": "srm", + "tape_srm": "srm.nearline", + "disk_root": "xrootd", + "tape_root": "wlcg.xrootd.tape", + "disk_https": "webdav", + "tape_https": "wlcg.webdav.tape", +} + + class DowntimeCommand(Command): """ Downtime "master" Command or removed DTs. @@ -135,26 +156,24 @@ def _prepareCommand(self): else: elementName = gocSite["Value"] - # The DIRAC se names mean nothing on the grid, but their hosts do mean. + # The DIRAC se names mean nothing on the grid, but their hosts and service types do mean. elif elementType == "StorageElement": - # for SRM and SRM only, we need to distinguish if it's tape or disk - # if it's not SRM, then gOCDBServiceType will be None (and we'll use them all) try: se = StorageElement(elementName) - seOptions = se.options - seProtocols = set(se.localAccessProtocolList) | set(se.localWriteProtocolList) + se_protocols = list(se.localAccessProtocolList) + se_protocols.extend(x for x in se.localWriteProtocolList if x not in se_protocols) except AttributeError: # Sometimes the SE can't be instantiated properly self.log.error("Failure instantiating StorageElement object", elementName) return S_ERROR("Failure instantiating StorageElement") - if "SEType" in seOptions and "srm" in seProtocols: + if "SEType" in se.options: # Type should follow the convention TXDY - seType = seOptions["SEType"] + seType = se.options["SEType"] diskSE = re.search("D[1-9]", seType) is not None tapeSE = re.search("T[1-9]", seType) is not None if tapeSE: - gOCDBServiceType = "srm.nearline" + gOCDBServiceType = diracToGOC_conversion[f"tape_{se_protocols[0]}"] # Just take the first, for simplicity elif diskSE: - gOCDBServiceType = "srm" + gOCDBServiceType = diracToGOC_conversion[f"disk_{se_protocols[0]}"] # Just take the first, for simplicity res = getSEHosts(elementName) if not res["OK"]: @@ -166,7 +185,7 @@ def _prepareCommand(self): elementName = seHosts # in this case it will return a list, because there might be more than one host only elif elementType in ["FTS", "FTS3"]: - gOCDBServiceType = "FTS" + gOCDBServiceType = diracToGOC_conversion[elementType] # WARNING: this method presupposes that the server is an FTS3 type gocSite = getGOCFTSName(elementName) if not gocSite["OK"]: @@ -182,10 +201,7 @@ def _prepareCommand(self): ceType = gConfig.getValue( cfgPath("Resources", "Sites", siteName.split(".")[0], siteName, "CEs", elementName, "CEType") ) - if ceType == "HTCondorCE": - gOCDBServiceType = "org.opensciencegrid.htcondorce" - elif ceType in ["ARC", "ARC6", "AREX"]: - gOCDBServiceType = "ARC-CE" + gOCDBServiceType = diracToGOC_conversion[ceType] return S_OK((element, elementName, hours, gOCDBServiceType)) From 47841f6671f50a5cf3bb0be47d17563ef939c3f1 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 22 Aug 2024 12:08:40 +0200 Subject: [PATCH 2/3] style: more compact code (Codeium) --- .../Command/DowntimeCommand.py | 49 ++++++++----------- .../Test_RSS_Command_GOCDBStatusCommand.py | 2 + 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py index 78024052a44..a4a360f6fd4 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py @@ -28,10 +28,10 @@ diracToGOC_conversion = { # Computing elements "CREAM": "CREAM-CE", # deprecated - "HTCondorCE" :"org.opensciencegrid.htcondorce", - "ARC" : "ARC-CE", - "ARC6" : "ARC-CE", - "AREX" : "ARC-CE", + "HTCondorCE": "org.opensciencegrid.htcondorce", + "ARC": "ARC-CE", + "ARC6": "ARC-CE", + "AREX": "ARC-CE", # FTS "FTS3": "FTS", "FTS": "FTS", @@ -53,15 +53,8 @@ class DowntimeCommand(Command): def __init__(self, args=None, clients=None): super().__init__(args, clients) - if "GOCDBClient" in self.apis: - self.gClient = self.apis["GOCDBClient"] - else: - self.gClient = GOCDBClient() - - if "ResourceManagementClient" in self.apis: - self.rmClient = self.apis["ResourceManagementClient"] - else: - self.rmClient = ResourceManagementClient() + self.gClient = self.apis.get("GOCDBClient", GOCDBClient()) + self.rmClient = self.apis.get("ResourceManagementClient", ResourceManagementClient()) def _storeCommand(self, result): """ @@ -158,23 +151,23 @@ def _prepareCommand(self): # The DIRAC se names mean nothing on the grid, but their hosts and service types do mean. elif elementType == "StorageElement": + # Get the SE object and its protocols try: se = StorageElement(elementName) se_protocols = list(se.localAccessProtocolList) se_protocols.extend(x for x in se.localWriteProtocolList if x not in se_protocols) - except AttributeError: # Sometimes the SE can't be instantiated properly + except AttributeError: self.log.error("Failure instantiating StorageElement object", elementName) return S_ERROR("Failure instantiating StorageElement") - if "SEType" in se.options: - # Type should follow the convention TXDY - seType = se.options["SEType"] - diskSE = re.search("D[1-9]", seType) is not None - tapeSE = re.search("T[1-9]", seType) is not None - if tapeSE: - gOCDBServiceType = diracToGOC_conversion[f"tape_{se_protocols[0]}"] # Just take the first, for simplicity - elif diskSE: - gOCDBServiceType = diracToGOC_conversion[f"disk_{se_protocols[0]}"] # Just take the first, for simplicity + # Determine the SE type and update gOCDBServiceType accordingly + se_type = se.options.get("SEType", "") + if re.search(r"D[1-9]", se_type): + gOCDBServiceType = diracToGOC_conversion[f"disk_{se_protocols[0]}"] + elif re.search(r"T[1-9]", se_type): + gOCDBServiceType = diracToGOC_conversion[f"tape_{se_protocols[0]}"] + + # Get the SE hosts and return an error if none are found res = getSEHosts(elementName) if not res["OK"]: return res @@ -252,9 +245,8 @@ def doNew(self, masterParams=None): # cleaning the Cache if elementNames: - cleanRes = self._cleanCommand(element, elementNames) - if not cleanRes["OK"]: - return cleanRes + if not (res := self._cleanCommand(element, elementNames))["OK"]: + return res uniformResult = [] @@ -286,9 +278,8 @@ def doNew(self, masterParams=None): uniformResult.append(dt) - storeRes = self._storeCommand(uniformResult) - if not storeRes["OK"]: - return storeRes + if not (res := self._storeCommand(uniformResult))["OK"]: + return res return S_OK() diff --git a/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py b/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py index 9b6dd171152..35665e3029e 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py @@ -16,6 +16,8 @@ # Mock external libraries / modules not interesting for the unit test seMock = mock.MagicMock() seMock.options = {"SEType": "T0D1"} +seMock.localAccessProtocolList = ["https", "root"] +seMock.localWriteProtocolList = ["https"] mock_GOCDBClient = mock.MagicMock() mock_RMClient = mock.MagicMock() From 7d0d26636e114e761c35534c625b88286a8b5034 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 22 Aug 2024 14:20:23 +0200 Subject: [PATCH 3/3] fix: do not get double entries --- src/DIRAC/Core/LCG/GOCDBClient.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/Core/LCG/GOCDBClient.py b/src/DIRAC/Core/LCG/GOCDBClient.py index 2b735a1709d..8bb9be47698 100644 --- a/src/DIRAC/Core/LCG/GOCDBClient.py +++ b/src/DIRAC/Core/LCG/GOCDBClient.py @@ -356,10 +356,11 @@ def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None): try: dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]] = elements - dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][ - "URL" - ] = urls[0] - except Exception: + if urls: + dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][ + "URL" + ] = urls[0] + except Exception as e: try: dtDict[ str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["HOSTNAME"]