From 5027310d524f4fcfd86f994c15a64e1cf84e38d0 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 11 Sep 2024 14:53:19 +0200 Subject: [PATCH 1/2] sweep: #7759 RSS: add a DIRAC to GOCDB service type conversion --- src/DIRAC/Core/LCG/GOCDBClient.py | 9 ++- .../Command/DowntimeCommand.py | 74 ++++++++++--------- .../Test_RSS_Command_GOCDBStatusCommand.py | 2 + 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/DIRAC/Core/LCG/GOCDBClient.py b/src/DIRAC/Core/LCG/GOCDBClient.py index 915d3652c11..233a77f1973 100644 --- a/src/DIRAC/Core/LCG/GOCDBClient.py +++ b/src/DIRAC/Core/LCG/GOCDBClient.py @@ -356,10 +356,11 @@ def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None): try: dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]] = elements - dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][ - "URL" - ] = urls[0] - except Exception: + if urls: + dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][ + "URL" + ] = urls[0] + except Exception as e: try: dtDict[ str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["HOSTNAME"] diff --git a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py index 8c7fbce396e..6b5571ec473 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py @@ -24,6 +24,24 @@ from DIRAC.ResourceStatusSystem.Command.Command import Command +# conversion from DIRAC resource type to GOCDB service type +diracToGOC_conversion = { + # Computing elements + "HTCondorCE": "org.opensciencegrid.htcondorce", + "AREX": "ARC-CE", + # FTS + "FTS3": "FTS", + "FTS": "FTS", + # Storage elements + "disk_srm": "srm", + "tape_srm": "srm.nearline", + "disk_root": "xrootd", + "tape_root": "wlcg.xrootd.tape", + "disk_https": "webdav", + "tape_https": "wlcg.webdav.tape", +} + + class DowntimeCommand(Command): """ Downtime "master" Command or removed DTs. @@ -32,15 +50,8 @@ class DowntimeCommand(Command): def __init__(self, args=None, clients=None): super().__init__(args, clients) - if "GOCDBClient" in self.apis: - self.gClient = self.apis["GOCDBClient"] - else: - self.gClient = GOCDBClient() - - if "ResourceManagementClient" in self.apis: - self.rmClient = self.apis["ResourceManagementClient"] - else: - self.rmClient = ResourceManagementClient() + self.gClient = self.apis.get("GOCDBClient", GOCDBClient()) + self.rmClient = self.apis.get("ResourceManagementClient", ResourceManagementClient()) def _storeCommand(self, result): """ @@ -135,27 +146,25 @@ def _prepareCommand(self): else: elementName = gocSite["Value"] - # The DIRAC se names mean nothing on the grid, but their hosts do mean. + # The DIRAC se names mean nothing on the grid, but their hosts and service types do mean. elif elementType == "StorageElement": - # for SRM and SRM only, we need to distinguish if it's tape or disk - # if it's not SRM, then gOCDBServiceType will be None (and we'll use them all) + # Get the SE object and its protocols try: se = StorageElement(elementName) - seOptions = se.options - seProtocols = set(se.localAccessProtocolList) | set(se.localWriteProtocolList) - except AttributeError: # Sometimes the SE can't be instantiated properly + se_protocols = list(se.localAccessProtocolList) + se_protocols.extend(x for x in se.localWriteProtocolList if x not in se_protocols) + except AttributeError: self.log.error("Failure instantiating StorageElement object", elementName) return S_ERROR("Failure instantiating StorageElement") - if "SEType" in seOptions and "srm" in seProtocols: - # Type should follow the convention TXDY - seType = seOptions["SEType"] - diskSE = re.search("D[1-9]", seType) is not None - tapeSE = re.search("T[1-9]", seType) is not None - if tapeSE: - gOCDBServiceType = "srm.nearline" - elif diskSE: - gOCDBServiceType = "srm" + # Determine the SE type and update gOCDBServiceType accordingly + se_type = se.options.get("SEType", "") + if re.search(r"D[1-9]", se_type): + gOCDBServiceType = diracToGOC_conversion[f"disk_{se_protocols[0]}"] + elif re.search(r"T[1-9]", se_type): + gOCDBServiceType = diracToGOC_conversion[f"tape_{se_protocols[0]}"] + + # Get the SE hosts and return an error if none are found res = getSEHosts(elementName) if not res["OK"]: return res @@ -166,7 +175,7 @@ def _prepareCommand(self): elementName = seHosts # in this case it will return a list, because there might be more than one host only elif elementType in ["FTS", "FTS3"]: - gOCDBServiceType = "FTS" + gOCDBServiceType = diracToGOC_conversion[elementType] # WARNING: this method presupposes that the server is an FTS3 type gocSite = getGOCFTSName(elementName) if not gocSite["OK"]: @@ -182,10 +191,7 @@ def _prepareCommand(self): ceType = gConfig.getValue( cfgPath("Resources", "Sites", siteName.split(".")[0], siteName, "CEs", elementName, "CEType") ) - if ceType == "HTCondorCE": - gOCDBServiceType = "org.opensciencegrid.htcondorce" - elif ceType == "AREX": - gOCDBServiceType = "ARC-CE" + gOCDBServiceType = diracToGOC_conversion[ceType] return S_OK((element, elementName, hours, gOCDBServiceType)) @@ -236,9 +242,8 @@ def doNew(self, masterParams=None): # cleaning the Cache if elementNames: - cleanRes = self._cleanCommand(element, elementNames) - if not cleanRes["OK"]: - return cleanRes + if not (res := self._cleanCommand(element, elementNames))["OK"]: + return res uniformResult = [] @@ -270,9 +275,8 @@ def doNew(self, masterParams=None): uniformResult.append(dt) - storeRes = self._storeCommand(uniformResult) - if not storeRes["OK"]: - return storeRes + if not (res := self._storeCommand(uniformResult))["OK"]: + return res return S_OK() diff --git a/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py b/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py index 9b6dd171152..35665e3029e 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py @@ -16,6 +16,8 @@ # Mock external libraries / modules not interesting for the unit test seMock = mock.MagicMock() seMock.options = {"SEType": "T0D1"} +seMock.localAccessProtocolList = ["https", "root"] +seMock.localWriteProtocolList = ["https"] mock_GOCDBClient = mock.MagicMock() mock_RMClient = mock.MagicMock() From 02f94a2b25aa5ff934136340c9e17ab8760240a1 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 11 Sep 2024 14:57:07 +0200 Subject: [PATCH 2/2] style: imports ordering --- src/DIRAC/Core/LCG/GOCDBClient.py | 1 + .../Command/DowntimeCommand.py | 15 +++++++-------- .../test/Test_RSS_Command_GOCDBStatusCommand.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/DIRAC/Core/LCG/GOCDBClient.py b/src/DIRAC/Core/LCG/GOCDBClient.py index 233a77f1973..1293b490101 100644 --- a/src/DIRAC/Core/LCG/GOCDBClient.py +++ b/src/DIRAC/Core/LCG/GOCDBClient.py @@ -10,6 +10,7 @@ from xml.dom import minidom import requests + from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Security.Locations import getCAsLocation diff --git a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py index 6b5571ec473..3f4f54e7f6d 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py @@ -4,26 +4,25 @@ GOCDB downtimes that are modified or deleted are also synced. """ import re -from urllib.error import URLError from datetime import datetime, timedelta from operator import itemgetter +from urllib.error import URLError -from DIRAC import S_OK, S_ERROR, gConfig -from DIRAC.Core.LCG.GOCDBClient import GOCDBClient -from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts +from DIRAC import S_ERROR, S_OK, gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath from DIRAC.ConfigurationSystem.Client.Helpers.Resources import ( + getCESiteMapping, getFTS3Servers, + getGOCFTSName, getGOCSiteName, getGOCSites, - getGOCFTSName, - getCESiteMapping, ) +from DIRAC.Core.LCG.GOCDBClient import GOCDBClient +from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient from DIRAC.ResourceStatusSystem.Command.Command import Command - # conversion from DIRAC resource type to GOCDB service type diracToGOC_conversion = { # Computing elements @@ -146,7 +145,7 @@ def _prepareCommand(self): else: elementName = gocSite["Value"] - # The DIRAC se names mean nothing on the grid, but their hosts and service types do mean. + # The DIRAC SE names mean nothing on the grid, but their hosts and service types do mean. elif elementType == "StorageElement": # Get the SE object and its protocols try: diff --git a/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py b/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py index 35665e3029e..c64da8eeed8 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py @@ -1,11 +1,11 @@ """ Test_RSS_Command_GOCDBStatusCommand """ from datetime import datetime, timedelta - from unittest import mock + import pytest -from DIRAC import gLogger, S_OK +from DIRAC import S_OK, gLogger from DIRAC.ResourceStatusSystem.Command.DowntimeCommand import DowntimeCommand """