diff --git a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py index 69e8e757dd1..5b7c0086120 100755 --- a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +++ b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py @@ -15,7 +15,6 @@ getServiceSection, getAgentSection, getExecutorSection, - getSystemSection, ) from DIRAC.Core.Utilities.Devloader import Devloader @@ -507,7 +506,7 @@ def __addUserDataToConfiguration(self): if self.componentType == "service": self.__setDefaultSection(getServiceSection(self.componentName)) elif self.componentType == "tornado": - self.__setDefaultSection(getSystemSection("Tornado")) + self.__setDefaultSection("/Systems/Tornado") elif self.componentType == "agent": self.__setDefaultSection(getAgentSection(self.componentName)) elif self.componentType == "executor": diff --git a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py index f5470659cca..1946cee118d 100755 --- a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py @@ -7,14 +7,6 @@ from DIRAC.ConfigurationSystem.Client.Helpers import Path -def getDIRACSetup(): - """Get DIRAC default setup name - - :return: str - """ - return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup") - - def divideFullName(entityName, componentName=None): """Convert component full name to tuple @@ -32,45 +24,12 @@ def divideFullName(entityName, componentName=None): raise RuntimeError(f"Service ({entityName}) name must be with the form system/service") -def getSystemInstance(system, setup=False): - """Find system instance name - - :param str system: system name - :param str setup: setup name - - :return: str - """ - optionPath = Path.cfgPath("/DIRAC/Setups", setup or getDIRACSetup(), system) - instance = gConfigurationData.extractOptionFromCFG(optionPath) - if not instance: - raise RuntimeError(f"Option {optionPath} is not defined") - return instance - - -def getSystemSection(system, instance=False, setup=False): - """Get system section - - :param str system: system name - :param str instance: instance name - :param str setup: setup name - - :return: str -- system section path - """ - system, _ = divideFullName(system, "_") # for backward compatibility - return Path.cfgPath( - "/Systems", - system, - instance or getSystemInstance(system, setup=setup), - ) - - -def getComponentSection(system, component=False, setup=False, componentCategory="Services"): +def getComponentSection(system, component=False, componentCategory="Services"): """Function returns the path to the component. :param str system: system name or component name prefixed by the system in which it is placed. e.g. 'WorkloadManagement/SandboxStoreHandler' :param str component: component name, e.g. 'SandboxStoreHandler' - :param str setup: Name of the setup. :param str componentCategory: Category of the component, it can be: 'Agents', 'Services', 'Executors' or 'Databases'. @@ -80,14 +39,14 @@ def getComponentSection(system, component=False, setup=False, componentCategory= :raise RuntimeException: If in the system - the system part does not correspond to any known system in DIRAC. Examples: - getComponentSection('WorkloadManagement/SandboxStoreHandler', setup='Production', componentCategory='Services') - getComponentSection('WorkloadManagement', 'SandboxStoreHandler', 'Production') + getComponentSection('WorkloadManagement/SandboxStoreHandler', componentCategory='Services') + getComponentSection('WorkloadManagement', 'SandboxStoreHandler') """ system, component = divideFullName(system, component) - return Path.cfgPath(getSystemSection(system, setup=setup), componentCategory, component) + return Path.cfgPath(f"/Systems/{system}", componentCategory, component) -def getAPISection(system, endpointName=False, setup=False): +def getAPISection(system, endpointName=False): """Get API section in a system :param str system: system name @@ -95,66 +54,61 @@ def getAPISection(system, endpointName=False, setup=False): :return: str """ - return getComponentSection(system, component=endpointName, setup=setup, componentCategory="APIs") + return getComponentSection(system, component=endpointName, componentCategory="APIs") -def getServiceSection(system, serviceName=False, setup=False): +def getServiceSection(system, serviceName=False): """Get service section in a system :param str system: system name :param str serviceName: service name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=serviceName, setup=setup) + return getComponentSection(system, component=serviceName) -def getAgentSection(system, agentName=False, setup=False): +def getAgentSection(system, agentName=False): """Get agent section in a system :param str system: system name :param str agentName: agent name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=agentName, setup=setup, componentCategory="Agents") + return getComponentSection(system, component=agentName, componentCategory="Agents") -def getExecutorSection(system, executorName=None, component=False, setup=False): +def getExecutorSection(system, executorName=None): """Get executor section in a system :param str system: system name :param str executorName: executor name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=executorName, setup=setup, componentCategory="Executors") + return getComponentSection(system, component=executorName, componentCategory="Executors") -def getDatabaseSection(system, dbName=False, setup=False): +def getDatabaseSection(system, dbName=False): """Get DB section in a system :param str system: system name :param str dbName: DB name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=dbName, setup=setup, componentCategory="Databases") + return getComponentSection(system, component=dbName, componentCategory="Databases") -def getSystemURLSection(system, setup=False): +def getSystemURLSection(system): """Get URLs section in a system :param str system: system name - :param str setup: setup name :return: str """ - return Path.cfgPath(getSystemSection(system, setup=setup), "URLs") + return Path.cfgPath(f"/Systems/{system}", "URLs") def checkComponentURL(componentURL, system=None, component=None, pathMandatory=False): @@ -183,27 +137,25 @@ def checkComponentURL(componentURL, system=None, component=None, pathMandatory=F return url.geturl() -def getSystemURLs(system, setup=False, failover=False): +def getSystemURLs(system, failover=False): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager - :param str setup: DIRAC setup name, can be defined in dirac.cfg :param bool failover: to add failover URLs to end of result list :return: dict -- complete urls. e.g. [dips://some-domain:3424/Framework/Service] """ urlDict = {} - for service in gConfigurationData.getOptionsFromCFG(f"{getSystemSection(system, setup=setup)}/URLs") or []: - urlDict[service] = getServiceURLs(system, service, setup=setup, failover=failover) + for service in gConfigurationData.getOptionsFromCFG(f"/Systems/{system}/URLs") or []: + urlDict[service] = getServiceURLs(system, service, failover=failover) return urlDict -def getServiceURLs(system, service=None, setup=False, failover=False): +def getServiceURLs(system, service=None, failover=False): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager :param str service: service name, like 'ProxyManager'. - :param str setup: DIRAC setup name, can be defined in dirac.cfg :param bool failover: to add failover URLs to end of result list :return: list -- complete urls. e.g. [dips://some-domain:3424/Framework/Service] @@ -211,7 +163,7 @@ def getServiceURLs(system, service=None, setup=False, failover=False): system, service = divideFullName(system, service) resList = [] mainServers = None - systemSection = getSystemSection(system, setup=setup) + systemSection = f"/Systems/{system}" # Add failover URLs at the end of the list failover = "Failover" if failover else "" @@ -228,7 +180,7 @@ def getServiceURLs(system, service=None, setup=False, failover=False): # Operations cannot be imported at the beginning because of a bootstrap problem from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations - mainServers = Operations(setup=setup).getValue("MainServers", []) + mainServers = Operations().getValue("MainServers", []) if not mainServers: raise Exception("No Main servers defined") @@ -261,31 +213,29 @@ def useLegacyAdapter(system, service=None) -> bool: return (value or "no").lower() in ("y", "yes", "true", "1") -def getServiceURL(system, service=None, setup=False): +def getServiceURL(system, service=None): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager :param str service: service name, like 'ProxyManager'. - :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: str -- complete list of urls. e.g. dips://some-domain:3424/Framework/Service, dips://.. """ system, service = divideFullName(system, service) - urls = getServiceURLs(system, service=service, setup=setup) + urls = getServiceURLs(system, service=service) return ",".join(urls) if urls else "" -def getServiceFailoverURL(system, service=None, setup=False): +def getServiceFailoverURL(system, service=None): """Get failover URLs for service :param str system: system name or full name, like 'Framework/Service'. :param str service: service name, like 'ProxyManager'. - :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: str -- complete list of urls """ system, service = divideFullName(system, service) - systemSection = getSystemSection(system, setup=setup) + systemSection = f"/Systems/{system}" failovers = gConfigurationData.extractOptionFromCFG(f"{systemSection}/FailoverURLs/{service}") if not failovers: return "" diff --git a/src/DIRAC/ConfigurationSystem/Client/Utilities.py b/src/DIRAC/ConfigurationSystem/Client/Utilities.py index e7572b6ff6d..b783a7087aa 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Utilities.py +++ b/src/DIRAC/ConfigurationSystem/Client/Utilities.py @@ -10,7 +10,6 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACSiteName from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection from DIRAC.Core.Utilities.Glue2 import getGlue2CEInfo -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance def getGridVOs(): @@ -612,7 +611,7 @@ def getAuthAPI(): :return: str """ - return gConfig.getValue(f"/Systems/Framework/{getSystemInstance('Framework')}/URLs/AuthAPI") + return gConfig.getValue(f"/Systems/Framework/URLs/AuthAPI") def getAuthorizationServerMetadata(issuer=None, ignoreErrors=False): @@ -651,5 +650,5 @@ def isDownloadProxyAllowed(): :return: S_OK(bool)/S_ERROR() """ - cs_path = f"/Systems/Framework/{getSystemInstance('Framework')}/APIs/Auth" + cs_path = f"/Systems/Framework/APIs/Auth" return gConfig.getValue(cs_path + "/allowProxyDownload", True) diff --git a/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py index 0025eb2a1ac..3348140e438 100644 --- a/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py @@ -12,33 +12,19 @@ mergedCFG = CFG() mergedCFG.loadFromBuffer( """ -DIRAC -{ - Setup=TestSetup - Setups - { - TestSetup - { - WorkloadManagement=MyWM - } - } -} Systems { WorkloadManagement { - MyWM + URLs + { + Service1 = dips://server1:1234/WorkloadManagement/Service1 + Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2 + } + FailoverURLs { - URLs - { - Service1 = dips://server1:1234/WorkloadManagement/Service1 - Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2 - } - FailoverURLs - { - Service2 = dips://failover1:5678/WorkloadManagement/Service2 - Service2 += dips://failover2:5678/WorkloadManagement/Service2 - } + Service2 = dips://failover1:5678/WorkloadManagement/Service2 + Service2 += dips://failover2:5678/WorkloadManagement/Service2 } } } @@ -63,53 +49,44 @@ def pathFinder(monkeypatch): return PathFinder -def test_getDIRACSetup(pathFinder): - """Test getDIRACSetup""" - assert pathFinder.getDIRACSetup() == "TestSetup" - - @pytest.mark.parametrize( - "system, componentName, setup, componentType, result", + "system, componentName, componentType, result", [ ( "WorkloadManagement/SandboxStoreHandler", False, - False, "Services", - "/Systems/WorkloadManagement/MyWM/Services/SandboxStoreHandler", + "/Systems/WorkloadManagement/Services/SandboxStoreHandler", ), ( "WorkloadManagement", "SandboxStoreHandler", - False, "Services", - "/Systems/WorkloadManagement/MyWM/Services/SandboxStoreHandler", + "/Systems/WorkloadManagement/Services/SandboxStoreHandler", ), # tricky case one could expect that if entity string is wrong # than some kind of error will be returned, but it is not the case ( "WorkloadManagement/SimpleLogConsumer", False, - False, "NonRonsumersNon", - "/Systems/WorkloadManagement/MyWM/NonRonsumersNon/SimpleLogConsumer", + "/Systems/WorkloadManagement/NonRonsumersNon/SimpleLogConsumer", ), ], ) -def test_getComponentSection(pathFinder, system, componentName, setup, componentType, result): +def test_getComponentSection(pathFinder, system, componentName, componentType, result): """Test getComponentSection""" - assert pathFinder.getComponentSection(system, componentName, setup, componentType) == result + assert pathFinder.getComponentSection(system, componentName, componentType) == result @pytest.mark.parametrize( - "system, setup, result", + "system, result", [ - ("WorkloadManagement", False, "/Systems/WorkloadManagement/MyWM/URLs"), - ("WorkloadManagement", "TestSetup", "/Systems/WorkloadManagement/MyWM/URLs"), + ("WorkloadManagement", "/Systems/WorkloadManagement/MyWM/URLs"), ], ) -def test_getSystemURLSection(pathFinder, system, setup, result): - assert pathFinder.getSystemURLs(system, setup) +def test_getSystemURLSection(pathFinder, system, result): + assert pathFinder.getSystemURLs(system) @pytest.mark.parametrize( @@ -205,11 +182,10 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result): @pytest.mark.parametrize( - "system, setup, failover, result", + "system, failover, result", [ ( "WorkloadManagement", - None, False, { "Service1": {"dips://server1:1234/WorkloadManagement/Service1"}, @@ -221,7 +197,6 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result): ), ( "WorkloadManagement", - None, True, { "Service1": {"dips://server1:1234/WorkloadManagement/Service1"}, @@ -235,8 +210,8 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result): ), ], ) -def test_getSystemURLs(pathFinder, system, setup, failover, result): - sysDict = pathFinder.getSystemURLs(system, setup=setup, failover=failover) +def test_getSystemURLs(pathFinder, system, failover, result): + sysDict = pathFinder.getSystemURLs(system, failover=failover) for service in sysDict: assert set(sysDict[service]) == result[service] diff --git a/src/DIRAC/Core/Base/private/ModuleLoader.py b/src/DIRAC/Core/Base/private/ModuleLoader.py index 18f0be10877..c1139c6ee38 100644 --- a/src/DIRAC/Core/Base/private/ModuleLoader.py +++ b/src/DIRAC/Core/Base/private/ModuleLoader.py @@ -3,7 +3,6 @@ import os from DIRAC.Core.Utilities import List from DIRAC import gConfig, S_ERROR, S_OK, gLogger -from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.Utilities.Extensions import extensionsByPriority, recurseImport @@ -44,8 +43,7 @@ def loadModules(self, modulesList, hideExceptions=False): # Check if it's a system name # Look in the CS system = modName - # Can this be generated with sectionFinder? - csPath = f"{PathFinder.getSystemSection(system)}/Executors" + csPath = f"/Systems/{system}/Executors" gLogger.verbose(f"Exploring {csPath} to discover modules") result = gConfig.getSections(csPath) if result["OK"]: diff --git a/src/DIRAC/Core/DISET/private/BaseClient.py b/src/DIRAC/Core/DISET/private/BaseClient.py index b51d176071a..c1f2da0147d 100755 --- a/src/DIRAC/Core/DISET/private/BaseClient.py +++ b/src/DIRAC/Core/DISET/private/BaseClient.py @@ -84,7 +84,6 @@ def __init__(self, serviceName, **kwargs): self.__retryCounter = 1 self.__bannedUrls = [] for initFunc in ( - self.__discoverSetup, self.__discoverVO, self.__discoverTimeout, self.__discoverURL, @@ -117,24 +116,6 @@ def getServiceName(self): """ return self._serviceName - def __discoverSetup(self): - """Discover which setup to use and stores it in self.setup - The setup is looked for: - * kwargs of the constructor (see KW_SETUP) - * the ThreadConfig - * in the CS /DIRAC/Setup - * default to 'Test' - - :return: S_OK()/S_ERROR() - """ - if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]: - self.setup = str(self.kwargs[self.KW_SETUP]) - else: - self.setup = self.__threadConfig.getSetup() - if not self.setup: - self.setup = gConfig.getValue("/DIRAC/Setup", "Test") - return S_OK() - def __discoverVO(self): """Discover which VO to use and stores it in self.vo The VO is looked for: @@ -326,16 +307,16 @@ def __findServiceURL(self): # We extract the list of URLs from the CS (System/URLs/Component) try: - urls = getServiceURL(self._destinationSrv, setup=self.setup) + urls = getServiceURL(self._destinationSrv) except Exception as e: - return S_ERROR(f"Cannot get URL for {self._destinationSrv} in setup {self.setup}: {repr(e)}") + return S_ERROR(f"Cannot get URL for {self._destinationSrv} : {repr(e)}") if not urls: return S_ERROR(f"URL for service {self._destinationSrv} not found") failoverUrls = [] # Try if there are some failover URLs to use as last resort try: - failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) + failoverUrlsStr = getServiceFailoverURL(self._destinationSrv) if failoverUrlsStr: failoverUrls = failoverUrlsStr.split(",") except Exception: @@ -564,7 +545,7 @@ def _proposeAction(self, transport, action): """ if not self.__initStatus["OK"]: return self.__initStatus - stConnectionInfo = ((self.__URLTuple[3], self.setup, self.vo), action, self.__extraCredentials, DIRAC.version) + stConnectionInfo = ((self.__URLTuple[3], "None", self.vo), action, self.__extraCredentials, DIRAC.version) # Send the connection info and get the answer back retVal = transport.sendData(S_OK(BaseClient._serializeStConnectionInfo(stConnectionInfo))) diff --git a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py index 3d57b999fda..8b0580e5def 100644 --- a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py @@ -116,13 +116,11 @@ def __init__(self, serviceName, **kwargs): self.__bannedUrls = [] # For pylint... - self.setup = None self.vo = None self.serviceURL = None for initFunc in ( self.__discoverTimeout, - self.__discoverSetup, self.__discoverVO, self.__discoverCredentialsToUse, self.__discoverExtraCredentials, @@ -132,21 +130,6 @@ def __init__(self, serviceName, **kwargs): if not result["OK"] and self.__initStatus["OK"]: self.__initStatus = result - def __discoverSetup(self): - """Discover which setup to use and stores it in self.setup - The setup is looked for: - * kwargs of the constructor (see KW_SETUP) - * in the CS /DIRAC/Setup - * default to 'Test' - """ - if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]: - self.setup = str(self.kwargs[self.KW_SETUP]) - else: - self.setup = self.__threadConfig.getSetup() - if not self.setup: - self.setup = gConfig.getValue("/DIRAC/Setup", "Test") - return S_OK() - def __discoverURL(self): """Calculate the final URL. It is called at initialization and in connect in case of issue @@ -375,9 +358,9 @@ def __findServiceURL(self): # If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component) try: # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) - urlsList = getServiceURLs(self._destinationSrv, setup=self.setup, failover=True) + urlsList = getServiceURLs(self._destinationSrv, failover=True) except Exception as e: - return S_ERROR(f"Cannot get URL for {self._destinationSrv} in setup {self.setup}: {repr(e)}") + return S_ERROR(f"Cannot get URL for {self._destinationSrv} : {repr(e)}") if not urlsList: return S_ERROR(f"URL for service {self._destinationSrv} not found") @@ -500,7 +483,8 @@ def _request(self, retry=0, outputFile=None, **kwargs): if self.__extraCredentials: kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) kwargs["clientVO"] = self.vo - kwargs["clientSetup"] = self.setup + # No more Setups + kwargs["clientSetup"] = "None" # Getting URL url = self.__findServiceURL() diff --git a/src/DIRAC/Core/Tornado/Server/HandlerManager.py b/src/DIRAC/Core/Tornado/Server/HandlerManager.py index 9c48fb2f6dd..5cb56df7335 100644 --- a/src/DIRAC/Core/Tornado/Server/HandlerManager.py +++ b/src/DIRAC/Core/Tornado/Server/HandlerManager.py @@ -54,22 +54,17 @@ def discoverHandlers(self, handlerInstance): if diracSystems["OK"]: for system in diracSystems["Value"]: try: - sysInstance = PathFinder.getSystemInstance(system) - result = gConfig.getSections(f"/Systems/{system}/{sysInstance}/{handlerInstance}") + result = gConfig.getSections(f"/Systems/{system}/{handlerInstance}") if result["OK"]: for instName in result["Value"]: newInst = f"{system}/{instName}" - port = gConfig.getValue( - f"/Systems/{system}/{sysInstance}/{handlerInstance}/{instName}/Port" - ) + port = gConfig.getValue(f"/Systems/{system}/{handlerInstance}/{instName}/Port") if port: newInst += f":{port}" if handlerInstance == "Services": # We search in the CS all handlers which used HTTPS as protocol - isHTTPS = gConfig.getValue( - f"/Systems/{system}/{sysInstance}/{handlerInstance}/{instName}/Protocol" - ) + isHTTPS = gConfig.getValue(f"/Systems/{system}/{handlerInstance}/{instName}/Protocol") if isHTTPS and isHTTPS.lower() == "https": urls.append(newInst) else: diff --git a/src/DIRAC/Core/Tornado/Server/TornadoServer.py b/src/DIRAC/Core/Tornado/Server/TornadoServer.py index 509974f92a2..f252855c986 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoServer.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoServer.py @@ -25,7 +25,6 @@ from DIRAC.Core.Security import Locations from DIRAC.Core.Utilities import Network, TimeUtilities from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager -from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations sLog = gLogger.getSubLogger(__name__) @@ -91,7 +90,7 @@ def __init__(self, services=True, endpoints=False, port=None): self.__appsSettings = {} # Default port, if enother is not discover if port is None: - port = gConfig.getValue(f"/Systems/Tornado/{PathFinder.getSystemInstance('Tornado')}/Port", 8443) + port = gConfig.getValue(f"/Systems/Tornado/Port", 8443) self.port = port # Handler manager initialization with default settings diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py index 9ca623ed99e..6a9157dee04 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py @@ -47,7 +47,7 @@ def main(): # We check if there is no configuration server started as master # If you want to start a master CS you should use Configuration_Server.cfg and # use tornado-start-CS.py - key = f"/Systems/Configuration/{PathFinder.getSystemInstance('Configuration')}/Services/Server/Protocol" + key = f"/Systems/Configuration/Services/Server/Protocol" if gConfigurationData.isMaster() and gConfig.getValue(key, "dips").lower() == "https": gLogger.fatal("You can't run the CS and services in the same server!") sys.exit(0) diff --git a/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py b/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py index 36f17e12ad4..83eb1bde83b 100644 --- a/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py @@ -525,9 +525,8 @@ def checkURLs(self): # get port used for https based services try: - tornadoSystemInstance = PathFinder.getSystemInstance(system="Tornado") self._tornadoPort = gConfig.getValue( - Path.cfgPath("/System/Tornado/", tornadoSystemInstance, "Port"), + Path.cfgPath("/System/Tornado/", "Port"), self._tornadoPort, ) except RuntimeError: diff --git a/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py b/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py index 3dfb6d74d40..4e5d6744f3a 100644 --- a/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py @@ -22,19 +22,6 @@ def clientMock(ret): return clientModuleMock -def mockComponentSection(*_args, **kwargs): - """Mock the PathFinder.getComponentSection to return individual componentSections.""" - system = kwargs.get("system") - component = kwargs.get("component") - return f"/Systems/{system}/Production/Services/{component}" - - -def mockURLSection(*_args, **kwargs): - """Mock the PathFinder.getSystemURLSection to return individual componentSections.""" - system = kwargs.get("system") - return f"/Systems/{system}/Production/URLs/" - - class TestComponentSupervisionAgent(unittest.TestCase): """TestComponentSupervisionAgent class.""" @@ -680,22 +667,13 @@ def gVal(*args, **_kwargs): with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=gConfigMock), patch( "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.socket.gethostname", return_value=host - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemInstance", - return_value="Decertification", - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", - side_effect=mockComponentSection, - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemURLSection", - side_effect=mockURLSection, ): res = self.restartAgent.checkURLs() self.assertTrue(res["OK"]) self.restartAgent.csAPI.modifyValue.assert_has_calls( [ - call("/Systems/Sys/Production/URLs/Serv", ",".join(tempurls)), - call("/Systems/Sys/Production/URLs/Serv", ",".join(newurls)), + call("/Systems/Sys/URLs/Serv", ",".join(tempurls)), + call("/Systems/Sys/URLs/Serv", ",".join(newurls)), ], any_order=False, ) @@ -710,26 +688,14 @@ def test_checkURLs_2(self): self.restartAgent.sysAdminClient.getOverallStatus.return_value = S_OK(dict(Services={})) self.restartAgent.csAPI.commit = MagicMock(return_value=S_ERROR("Nope")) - with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemInstance", - return_value="Decertification", - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", - side_effect=mockComponentSection, - ): + with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()): res = self.restartAgent.checkURLs() self.assertFalse(res["OK"]) self.assertIn("Failed to commit", res["Message"]) self.assertIn("Commit to CS failed", self.restartAgent.errors[0]) self.restartAgent.csAPI.commit = MagicMock(return_value=S_OK()) - with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemInstance", - return_value="Decertification", - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", - side_effect=mockComponentSection, - ): + with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()): res = self.restartAgent.checkURLs() self.assertTrue(res["OK"]) diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index e6217033d6e..05cd12eef8a 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -142,7 +142,7 @@ def _makeComponentDict(component, setupDict, installedDict, compType, system, ru def _getSectionName(compType): """ Returns the section name for a component in the CS - For self.instance, the section for service is Services, + The section for service is Services, whereas the section for agent is Agents """ return f"{compType.title()}s" @@ -208,8 +208,6 @@ def loadDiracCfg(self): gLogger.always("Can't load ", self.cfgFile) gLogger.always("Might be OK if setting up the site") - self.setup = self.localCfg.getOption(cfgPath("DIRAC", "Setup"), "") - self.instance = self.localCfg.getOption(cfgInstallPath("InstanceName"), self.setup) self.logLevel = self.localCfg.getOption(cfgInstallPath("LogLevel"), "INFO") self.linkedRootPath = rootPath @@ -315,7 +313,6 @@ def getInfo(self): if not result["OK"]: return result rDict = result["Value"] - rDict["Setup"] = self.setup or "Unknown" return S_OK(rDict) def _addCfgToDiracCfg(self, cfg): @@ -405,6 +402,9 @@ def _getCentralCfg(self, installCfg): if extensions: centralCfg["DIRAC"].addKey("Extensions", ",".join(extensions), "") # pylint: disable=no-member + # No Setups will be used + centralCfg["DIRAC"].addKey("NoSetup", "True", "") # pylint: disable=no-member + vo = self.localCfg.getOption(cfgInstallPath("VirtualOrganization"), "") if vo: centralCfg["DIRAC"].addKey("VirtualOrganization", vo, "") # pylint: disable=no-member @@ -567,12 +567,10 @@ def addOptionToDiracCfg(self, option, value): return S_ERROR(f"Could not merge {option}={value} with local configuration") - def removeComponentOptionsFromCS(self, system, component, mySetup=None): + def removeComponentOptionsFromCS(self, system, component): """ Remove the section with Component options from the CS, if possible """ - if mySetup is None: - mySetup = self.setup result = self.monitoringClient.getInstallations( {"UnInstallationTime": None, "Instance": component}, {"DIRACSystem": system}, {}, True @@ -581,12 +579,6 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): return result installations = result["Value"] - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig: - compInstance = gConfig.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if len(installations) == 1: remove = True removeMain = False @@ -617,18 +609,16 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if remove: result = self._removeSectionFromCS( - cfgPath("Systems", system, compInstance, installation["Component"]["Type"].title() + "s", component) + cfgPath("Systems", system, installation["Component"]["Type"].title() + "s", component) ) if not result["OK"]: return result if not isRenamed and cType == "service": - result = self._removeOptionFromCS(cfgPath("Systems", system, compInstance, "URLs", component)) + result = self._removeOptionFromCS(cfgPath("Systems", system, "URLs", component)) if not result["OK"]: # It is maybe in the FailoverURLs ? - result = self._removeOptionFromCS( - cfgPath("Systems", system, compInstance, "FailoverURLs", component) - ) + result = self._removeOptionFromCS(cfgPath("Systems", system, "FailoverURLs", component)) if not result["OK"]: return result @@ -637,7 +627,6 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): cfgPath( "Systems", system, - compInstance, installation["Component"]["Type"].title() + "s", installation["Component"]["Module"], ) @@ -647,14 +636,12 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if cType == "service": result = self._removeOptionFromCS( - cfgPath("Systems", system, compInstance, "URLs", installation["Component"]["Module"]) + cfgPath("Systems", system, "URLs", installation["Component"]["Module"]) ) if not result["OK"]: # it is maybe in the FailoverURLs ? result = self._removeOptionFromCS( - cfgPath( - "Systems", system, compInstance, "FailoverURLs", installation["Component"]["Module"] - ) + cfgPath("Systems", system, "FailoverURLs", installation["Component"]["Module"]) ) if not result["OK"]: return result @@ -669,7 +656,6 @@ def addDefaultOptionsToCS( systemName, component, extensions, - mySetup=None, specialOptions={}, overwrite=False, addDefaultOptions=True, @@ -677,26 +663,16 @@ def addDefaultOptionsToCS( """ Add the section with the component options to the CS """ - if not mySetup: - mySetup = self.setup if gConfig_o: gConfig_o.forceRefresh() system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") - sectionName = _getSectionName(componentType) # Check if the component CS options exist addOptions = True - componentSection = cfgPath("Systems", system, compInstance, sectionName, component) + componentSection = cfgPath("Systems", system, sectionName, component) if not overwrite: if gConfig_o: result = gConfig_o.getOptions(componentSection) @@ -707,9 +683,7 @@ def addDefaultOptionsToCS( return S_OK("Component options already exist") # Add the component options now - result = self.getComponentCfg( - componentType, system, component, compInstance, extensions, specialOptions, addDefaultOptions - ) + result = self.getComponentCfg(componentType, system, component, extensions, specialOptions, addDefaultOptions) if not result["OK"]: return result compCfg = result["Value"] @@ -727,7 +701,7 @@ def addDefaultOptionsToCS( execList = compCfg.getOption(f"{componentSection}/Load", []) for element in execList: result = self.addDefaultOptionsToCS( - gConfig_o, componentType, systemName, element, extensions, self.setup, {}, overwrite + gConfig_o, componentType, systemName, element, extensions, {}, overwrite ) if not result["OK"]: gLogger.warn("Can't add to default CS", result["Message"]) @@ -741,13 +715,9 @@ def addDefaultOptionsToComponentCfg(self, componentType, systemName, component, Add default component options local component cfg """ system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", self.setup, system) - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") # Add the component options now - result = self.getComponentCfg(componentType, system, component, compInstance, extensions) + result = self.getComponentCfg(componentType, system, component, extensions) if not result["OK"]: return result compCfg = result["Value"] @@ -766,16 +736,12 @@ def addCfgToComponentCfg(self, componentType, systemName, component, cfg): if not cfg: return S_OK() system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", self.setup, system) - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") compCfgFile = os.path.join(rootPath, "etc", f"{system}_{component}.cfg") compCfg = CFG() if os.path.exists(compCfgFile): compCfg.loadFromFile(compCfgFile) - sectionPath = cfgPath("Systems", system, compInstance, sectionName) + sectionPath = cfgPath("Systems", system, sectionName) newCfg = self.__getCfg(sectionPath) newCfg.createNewSection(cfgPath(sectionPath, component), "Added by ComponentInstaller", cfg) @@ -785,9 +751,7 @@ def addCfgToComponentCfg(self, componentType, systemName, component, cfg): gLogger.error(error) return S_ERROR(error) - def getComponentCfg( - self, componentType, system, component, compInstance, extensions, specialOptions={}, addDefaultOptions=True - ): + def getComponentCfg(self, componentType, system, component, extensions, specialOptions={}, addDefaultOptions=True): """ Get the CFG object of the component configuration """ @@ -822,7 +786,7 @@ def getComponentCfg( # Delete Dependencies section if any compCfg.deleteKey("Dependencies") - sectionPath = cfgPath("Systems", system, compInstance, sectionName) + sectionPath = cfgPath("Systems", system, sectionName) cfg = self.__getCfg(sectionPath) cfg.createNewSection(cfgPath(sectionPath, component), "", compCfg) @@ -837,15 +801,13 @@ def getComponentCfg( port = compCfg.getOption("Port", 0) protocol = compCfg.getOption("Protocol", "dips") if (port or protocol == "https") and self.host: - urlsPath = cfgPath("Systems", system, compInstance, "URLs") + urlsPath = cfgPath("Systems", system, "URLs") cfg.createNewSection(urlsPath) - failoverUrlsPath = cfgPath("Systems", system, compInstance, "FailoverURLs") + failoverUrlsPath = cfgPath("Systems", system, "FailoverURLs") cfg.createNewSection(failoverUrlsPath) if protocol == "https": if not port: - port = gConfig.getValue( - f"/Systems/Tornado/{PathFinder.getSystemInstance('Tornado')}/Port", 8443 - ) + port = gConfig.getValue(f"/Systems/Tornado/Port", 8443) cfg.setOption( # Strip "Tornado" from the beginning of component name if present cfgPath(urlsPath, component[len("Tornado") if component.startswith("Tornado") else 0 :]), @@ -858,29 +820,19 @@ def getComponentCfg( return S_OK(cfg) - def addDatabaseOptionsToCS(self, gConfig_o, systemName, dbName, mySetup=None, overwrite=False): + def addDatabaseOptionsToCS(self, gConfig_o, systemName, dbName, overwrite=False): """ Add the section with the database options to the CS """ - if mySetup is None: - mySetup = self.setup - if gConfig_o: gConfig_o.forceRefresh() system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") # Check if the component CS options exist addOptions = True if not overwrite: - databasePath = cfgPath("Systems", system, compInstance, "Databases", dbName) + databasePath = cfgPath("Systems", system, "Databases", dbName) result = gConfig_o.getOptions(databasePath) if result["OK"]: addOptions = False @@ -888,20 +840,17 @@ def addDatabaseOptionsToCS(self, gConfig_o, systemName, dbName, mySetup=None, ov return S_OK("Database options already exist") # Add the component options now - result = self.getDatabaseCfg(system, dbName, compInstance) + result = self.getDatabaseCfg(system, dbName) if not result["OK"]: return result databaseCfg = result["Value"] gLogger.notice("Adding to CS", f"{system}/{dbName}") return self._addCfgToCS(databaseCfg) - def removeDatabaseOptionsFromCS(self, gConfig_o, system, dbName, mySetup=None): + def removeDatabaseOptionsFromCS(self, system, dbName): """ Remove the section with database options from the CS, if possible """ - if mySetup is None: - mySetup = self.setup - result = self.monitoringClient.installationExists( {"UnInstallationTime": None}, {"DIRACSystem": system, "Type": "DB", "DIRACModule": dbName}, {} ) @@ -909,49 +858,24 @@ def removeDatabaseOptionsFromCS(self, gConfig_o, system, dbName, mySetup=None): return result exists = result["Value"] - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not exists: - result = self._removeSectionFromCS(cfgPath("Systems", system, compInstance, "Databases", dbName)) + result = self._removeSectionFromCS(cfgPath("Systems", system, "Databases", dbName)) if not result["OK"]: return result return S_OK("Successfully removed entries from CS") - def getDatabaseCfg(self, system, dbName, compInstance): + def getDatabaseCfg(self, system, dbName): """ Get the CFG object of the database configuration """ - databasePath = cfgPath("Systems", system, compInstance, "Databases", dbName) + databasePath = cfgPath("Systems", system, "Databases", dbName) cfg = self.__getCfg(databasePath, "DBName", dbName) cfg.setOption(cfgPath(databasePath, "Host"), self.mysqlHost) cfg.setOption(cfgPath(databasePath, "Port"), self.mysqlPort) return S_OK(cfg) - def addSystemInstance(self, systemName, compInstance, mySetup=None, myCfg=False): - """ - Add a new system self.instance to dirac.cfg and CS - """ - if mySetup is None: - mySetup = self.setup - - system = systemName.replace("System", "") - gLogger.notice( - f"Adding {system} system as {compInstance} self.instance for {mySetup} self.setup to dirac.cfg and CS" - ) - - cfg = self.__getCfg(cfgPath("DIRAC", "Setups", mySetup), system, compInstance) - if myCfg: - if not self._addCfgToDiracCfg(cfg): - return S_ERROR("Failed to add system self.instance to dirac.cfg") - - return self._addCfgToCS(cfg) - def printStartupStatus(self, rDict): """ Print in nice format the return dictionary from self.getStartupComponentStatus @@ -1188,11 +1112,7 @@ def getComponentModule(self, system, component, compType): """ Get the component software module """ - self.setup = CSGlobals.getSetup() - self.instance = gConfig.getValue(cfgPath("DIRAC", "Setups", self.setup, system), "") - if not self.instance: - return S_OK(component) - module = gConfig.getValue(cfgPath("Systems", system, self.instance, compType, component, "Module"), "") + module = gConfig.getValue(cfgPath("Systems", system, compType, component, "Module"), "") if not module: module = component return S_OK(module) @@ -1382,7 +1302,7 @@ def setupSite(self, scriptCfg, cfg=None): setupWeb = self.localCfg.getOption(cfgInstallPath("WebPortal"), False) setupConfigurationMaster = self.localCfg.getOption(cfgInstallPath("ConfigurationMaster"), False) setupPrivateConfiguration = self.localCfg.getOption(cfgInstallPath("PrivateConfiguration"), False) - setupConfigurationName = self.localCfg.getOption(cfgInstallPath("ConfigurationName"), self.setup) + setupConfigurationName = self.localCfg.getOption(cfgInstallPath("ConfigurationName"), "DIRAC-Prod") setupAddConfiguration = self.localCfg.getOption(cfgInstallPath("AddConfiguration"), True) for serviceTuple in setupServices: @@ -1437,7 +1357,7 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return S_ERROR(error) - # if any server or agent needs to be install we need the startup directory and runsvdir running + # if any server or agent needs to be installed we need the startup directory and runsvdir running if setupServices or setupAgents or setupExecutors or setupWeb: if not os.path.exists(self.startDir): mkDir(self.startDir) @@ -1469,17 +1389,13 @@ def setupSite(self, scriptCfg, cfg=None): gLogger.notice("Installing Master Configuration Server (Tornado-based)") # Add some needed bootstrapping configuration - cfg = self.__getCfg(cfgPath("DIRAC", "Setups", self.setup), "Tornado", self.instance) - self._addCfgToDiracCfg(cfg) cfg = self.__getCfg( - cfgPath("Systems", "Configuration", self.instance, "Services", "Server"), + cfgPath("Systems", "Configuration", "Services", "Server"), "HandlerPath", "DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py", ) self._addCfgToDiracCfg(cfg) - cfg = self.__getCfg( - cfgPath("Systems", "Configuration", self.instance, "Services", "Server"), "Port", "9135" - ) + cfg = self.__getCfg(cfgPath("Systems", "Configuration", "Services", "Server"), "Port", "9135") self._addCfgToDiracCfg(cfg) cfg = self.__getCfg(cfgPath("DIRAC", "Configuration"), "Master", "yes") cfg.setOption(cfgPath("DIRAC", "Configuration", "Name"), setupConfigurationName) @@ -1489,14 +1405,9 @@ def setupSite(self, scriptCfg, cfg=None): serverUrl = f"https://{self.host}:9135/Configuration/Server" cfg.setOption(serversCfgPath, serverUrl) gConfigurationData.setOptionInCFG(serversCfgPath, serverUrl) - instanceOptionPath = cfgPath("DIRAC", "Setups", self.setup) - instanceCfg = self.__getCfg(instanceOptionPath, "Configuration", self.instance) - cfg = cfg.mergeWith(instanceCfg) self._addCfgToDiracCfg(cfg) - result = self.getComponentCfg( - "service", "Configuration", "Server", self.instance, extensions, addDefaultOptions=True - ) + result = self.getComponentCfg("service", "Configuration", "Server", extensions, addDefaultOptions=True) if not result["OK"]: if self.exitOnError: DIRAC.exit(-1) @@ -1542,10 +1453,6 @@ def setupSite(self, scriptCfg, cfg=None): # If the Configuration Server used is not the Master, it can take some time for this # info to be propagated, this may cause the later self.setup to fail if setupAddConfiguration: - gLogger.notice("Registering System instances") - for system in setupSystems: # pylint: disable=not-an-iterable - self.addSystemInstance(system, self.instance, self.setup, True) - for system, service in setupServices: if not self.addDefaultOptionsToCS(None, "service", system, service, extensions, overwrite=True)["OK"]: # If we are not allowed to write to the central CS, add the configuration to the local file @@ -1733,18 +1640,10 @@ def installComponent(self, componentType, system, component, extensions, compone gLogger.notice(f"Installing {componentType} {system}/{component}") - # Retrieve bash variables to be set - result = gConfig.getOption(f"DIRAC/Setups/{CSGlobals.getSetup()}/{system}") - if not result["OK"]: - return result - self.instance = result["Value"] - specialOptions = {} if componentModule: specialOptions["Module"] = componentModule - result = self.getComponentCfg( - componentType, system, component, self.instance, extensions, specialOptions=specialOptions - ) + result = self.getComponentCfg(componentType, system, component, extensions, specialOptions=specialOptions) if not result["OK"]: return result compCfg = result["Value"] @@ -1752,9 +1651,9 @@ def installComponent(self, componentType, system, component, extensions, compone section = _getSectionName(componentType) bashVars = "" - if compCfg.isSection(f"Systems/{system}/{self.instance}/{section}/{component}/Environment"): + if compCfg.isSection(f"Systems/{system}/{section}/{component}/Environment"): dictionary = compCfg.getAsDict() - bashSection = dictionary["Systems"][system][self.instance][section][component]["BashVariables"] + bashSection = dictionary["Systems"][system][section][component]["BashVariables"] for var in bashSection: bashVars = f"{bashVars}\nexport {var}={bashSection[var]}" @@ -1827,7 +1726,7 @@ def installComponent(self, componentType, system, component, extensions, compone os.chmod(stopFile, self.gDefaultPerms) except Exception: - error = f"Failed to prepare self.setup for {componentType} {system}/{component}" + error = f"Failed to prepare setup for {componentType} {system}/{component}" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -1996,7 +1895,7 @@ def installPortal(self): os.chmod(runFile, self.gDefaultPerms) except Exception: - error = "Failed to prepare self.setup for Web Portal" + error = "Failed to prepare setup for Web Portal" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -2266,7 +2165,7 @@ def uninstallDatabase(self, gConfig_o, dbName): dbSystem = result["Value"][dbName]["System"] - result = self.removeDatabaseOptionsFromCS(gConfig_o, dbSystem, dbName) + result = self.removeDatabaseOptionsFromCS(dbSystem, dbName) if not result["OK"]: return result @@ -2385,12 +2284,6 @@ def installTornado(self): gLogger.notice("Tornado_Tornado already installed") return S_OK(runitCompDir) - # Check the setup for the given system - result = gConfig.getOption(f"DIRAC/Setups/{CSGlobals.getSetup()}/Tornado") - if not result["OK"]: - return result - self.instance = result["Value"] - # Now do the actual installation try: self._createRunitLog(runitCompDir) @@ -2416,7 +2309,7 @@ def installTornado(self): os.chmod(runFile, self.gDefaultPerms) except Exception: - error = "Failed to prepare self.setup for Tornado" + error = "Failed to prepare setup for Tornado" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -2435,15 +2328,7 @@ def addTornadoOptionsToCS(self, gConfig_o): if gConfig_o: gConfig_o.forceRefresh() - instanceOption = cfgPath("DIRAC", "Setups", self.setup, "Tornado") - - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") - tornadoSection = cfgPath("Systems", "Tornado", compInstance) + tornadoSection = cfgPath("Systems", "Tornado") if gConfig_o: tornadoPort = gConfig_o.getValue(cfgPath(tornadoSection, "Port"), "8443") diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 82a62070c19..e1303de123c 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -649,12 +649,6 @@ def do_install(self, args): return system = result["Value"][database]["System"] dbType = result["Value"][database]["Type"] - setup = CSGlobals.getSetup() - instance = gConfig.getValue(f"/DIRAC/Setups/{setup}/{system}", "") - if not instance: - self._errMsg(f"No instance defined for system {system}") - self._errMsg(f"\tAdd new instance with 'add instance {system} '") - return if dbType == "MySQL": if not gComponentInstaller.mysqlPassword: @@ -1083,7 +1077,7 @@ def do_update(self, args): else: gLogger.notice("Software successfully updated.") gLogger.notice("You should restart the services to use the new software version.") - gLogger.notice("Think of updating /Operations///Pilot/Versions section in the CS") + gLogger.notice("Think of updating /Operations//Pilot/Versions section in the CS") def do_revert(self, args): """ @@ -1100,44 +1094,6 @@ def do_revert(self, args): else: gLogger.notice("Software reverted to", result["Value"]) - def do_add(self, args): - """ - Add new entity to the Configuration Service - - usage: - - add system - """ - if not args: - gLogger.notice(self.do_add.__doc__) - return - - argss = args.split() - option = argss[0] - del argss[0] - if option == "instance" or option == "system": - system = argss[0] - instance = argss[1] - client = SystemAdministratorClient(self.host, self.port) - result = client.getInfo() - if not result["OK"]: - self._errMsg(result["Message"]) - hostSetup = CSGlobals.getSetup() - instanceName = gConfig.getValue(f"/DIRAC/Setups/{hostSetup}/{system}", "") - if instanceName: - if instanceName == instance: - gLogger.notice(f"System {system} already has instance {instance}") - else: - self._errMsg(f"System {system} already has instance {instance}") - return - result = gComponentInstaller.addSystemInstance(system, instance, hostSetup) - if not result["OK"]: - self._errMsg(result["Message"]) - else: - gLogger.notice(f"{system} system instance {instance} added successfully") - else: - gLogger.notice("Unknown option:", option) - def do_exec(self, args): """ Execute a shell command on the remote host and get back the output diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index 4227267a005..f204a7290ec 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -29,7 +29,7 @@ import DIRAC from DIRAC import S_ERROR, S_OK, gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getSystemSection +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL from DIRAC.Core.Base.API import API from DIRAC.Core.Base.Client import Client from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd @@ -1210,7 +1210,7 @@ def getPhysicalFileMetadata(self, pfn, storageElement, printOutput=False): Example Usage: - >>> print(dirac.getPhysicalFileMetadata('srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data) + >>> print(dirac.getPhysicalFileMetadata('srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data') /lhcb/data/CCRC08/RAW/LHCb/CCRC/23341/023341_0000039571.raw','NIKHEF-RAW') {'OK': True, 'Value': {'Successful': {'srm://...': {'SRM2': 'rfio://...'}}, 'Failed': {}}} @@ -2207,7 +2207,7 @@ def pingService(self, system=None, service=None, printOutput=False, url=None): result = S_ERROR() try: if not url: - systemSection = getSystemSection(system + "/") + systemSection = f"/Systems/{system}" self.log.verbose(f"System section is: {systemSection}") section = f"{systemSection}/{service}" self.log.verbose(f"Requested service should have CS path: {section}") diff --git a/src/DIRAC/RequestManagementSystem/private/test/Test_RequestValidator.py b/src/DIRAC/RequestManagementSystem/private/test/Test_RequestValidator.py index 2cd6c3ad1f0..763b228a0b6 100644 --- a/src/DIRAC/RequestManagementSystem/private/test/Test_RequestValidator.py +++ b/src/DIRAC/RequestManagementSystem/private/test/Test_RequestValidator.py @@ -32,7 +32,6 @@ def setup(): def test_Validator(mocker, setup): """validator test""" - mocker.patch("DIRAC.ConfigurationSystem.Client.PathFinder.getSystemInstance", new=MagicMock()) request = Request() operation = Operation() @@ -117,7 +116,6 @@ def test_Validator(mocker, setup): def test_Validator_backward_compatibility(mocker, setup): """validator test""" - mocker.patch("DIRAC.ConfigurationSystem.Client.PathFinder.getSystemInstance", new=MagicMock()) request = Request() operation = Operation() diff --git a/src/DIRAC/TransformationSystem/test/Test_DRA.py b/src/DIRAC/TransformationSystem/test/Test_DRA.py index 510343641dd..bd6b1b90e9e 100644 --- a/src/DIRAC/TransformationSystem/test/Test_DRA.py +++ b/src/DIRAC/TransformationSystem/test/Test_DRA.py @@ -22,7 +22,6 @@ class TestDRA(unittest.TestCase): dra = None @patch("DIRAC.Core.Base.AgentModule.PathFinder", new=Mock()) - @patch("DIRAC.ConfigurationSystem.Client.PathFinder.getSystemInstance", new=Mock()) @patch(f"{MODULE_NAME}.ReqClient", new=Mock()) def setUp(self): self.dra = DataRecoveryAgent(agentName="ILCTransformationSystem/DataRecoveryAgent", loadName="TestDRA") @@ -66,7 +65,6 @@ def getTestMock(self, nameID=0, jobID=1234567): return testJob @patch("DIRAC.Core.Base.AgentModule.PathFinder", new=Mock()) - @patch("DIRAC.ConfigurationSystem.Client.PathFinder.getSystemInstance", new=Mock()) @patch(f"{MODULE_NAME}.ReqClient", new=Mock()) def test_init(self): """test for DataRecoveryAgent initialisation....................................................""" diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py index e65c7f1cbcd..8d3122efee5 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py @@ -15,7 +15,6 @@ from DIRAC.AccountingSystem.Client.Types.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd @@ -60,9 +59,6 @@ def initialize(self): if not self.am_getOption("Enable", True): self.log.info("Stalled Job Agent running in disabled mode") - wmsInstance = getSystemInstance("WorkloadManagement") - if not wmsInstance: - return S_ERROR("Can not get the WorkloadManagement system instance") self.stalledJobsTolerantSites = self.am_getOption("StalledJobsTolerantSites", self.stalledJobsTolerantSites) self.stalledJobsToleranceTime = self.am_getOption("StalledJobsToleranceTime", self.stalledJobsToleranceTime) @@ -74,7 +70,7 @@ def initialize(self): self.matchedTime = self.am_getOption("MatchedTime", self.matchedTime) self.rescheduledTime = self.am_getOption("RescheduledTime", self.rescheduledTime) - wrapperSection = cfgPath("Systems", "WorkloadManagement", wmsInstance, "JobWrapper") + wrapperSection = cfgPath("Systems", "WorkloadManagement", "JobWrapper") failedTime = self.am_getOption("FailedTimeHours", 6) watchdogCycle = gConfig.getValue(cfgPath(wrapperSection, "CheckingTime"), 30 * 60) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py index 43010883cf8..438caa3c775 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py @@ -186,7 +186,6 @@ def test_submitJobWrapper(mocker, jobID): side_effect=lambda x, y=None: y, create=True, ) - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", return_value="Value") # Initialize PJA jobAgent = PushJobAgent("Test", "Test1") diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py index 22a24de8fd0..94111b9e224 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py @@ -25,12 +25,10 @@ def sja(mocker): ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB") mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getSystemInstance", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobMonitoringClient", return_value=MagicMock()) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobManagerClient", return_value=MagicMock()) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotManagerClient", return_value=MagicMock()) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.WMSClient", return_value=MagicMock()) - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getSystemInstance", return_value="/bof/bih") mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getDNForUsername", return_value=MagicMock()) stalledJobAgent = StalledJobAgent() diff --git a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py index eae626de7df..889e03d1693 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py @@ -6,7 +6,6 @@ import tempfile from DIRAC import S_ERROR, S_OK, gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemSection from DIRAC.Core.Utilities.Os import getDiskSpace from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers @@ -229,7 +228,7 @@ def __checkDiskSpace(self, totalSize): diskSpace = getDiskSpace(self.__getDownloadDir(False)) # MB availableBytes = diskSpace * 1024 * 1024 # bytes bufferGBs = gConfig.getValue( - os.path.join(getSystemSection("WorkloadManagement/JobWrapper"), "JobWrapper", "MinOutputDataBufferGB"), 5.0 + os.path.join("/Systems/WorkloadManagement/JobWrapper", "JobWrapper", "MinOutputDataBufferGB"), 5.0 ) data = bufferGBs * 1024 * 1024 * 1024 # bufferGBs in bytes if (data + totalSize) < availableBytes: diff --git a/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py b/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py index d92428f8df4..1e7ea5fcca5 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py @@ -114,7 +114,6 @@ def test_DLIDownloadFromBestSE_Fail(dli, mockSE, osPathExists): def test_DLI_execute(mocker, dli, mockSE): - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) dli._downloadFromSE = MagicMock(return_value=S_OK({"path": "/local/path/1.txt"})) res = dli.execute(dataToResolve=["/a/lfn/1.txt"]) @@ -125,7 +124,6 @@ def test_DLI_execute(mocker, dli, mockSE): def test_DLI_execute_getFileMetadata_Fails(mocker, dli, mockSE): """When getFileMetadata fails for the first SE, we should fall back to the second.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) mockObjectSE = mockSE.return_value mockObjectSE.getFileMetadata.return_value = S_OK( @@ -143,7 +141,6 @@ def test_DLI_execute_getFileMetadata_Fails(mocker, dli, mockSE): def test_DLI_execute_getFileMetadata_Lost(mocker, dli, mockSE): """When getFileMetadata fails for the first SE, we should fall back to the second.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) mockObjectSE = mockSE.return_value mockObjectSE.getFileMetadata.return_value = S_OK( @@ -169,7 +166,6 @@ def test_DLI_execute_getFileMetadata_Lost(mocker, dli, mockSE): def test_DLI_execute_getFileMetadata_Unavailable(mocker, dli, mockSE): """When getFileMetadata fails for the first SE, we should fall back to the second.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) mockObjectSE = mockSE.return_value mockObjectSE.getFileMetadata.return_value = S_OK( @@ -195,7 +191,6 @@ def test_DLI_execute_getFileMetadata_Unavailable(mocker, dli, mockSE): def test_DLI_execute_getFileMetadata_Cached(mocker, dli, mockSE): """When getFileMetadata fails for the first SE, we should fall back to the second.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) mockObjectSE = mockSE.return_value mockObjectSE.getFileMetadata.return_value = S_OK( @@ -221,7 +216,6 @@ def test_DLI_execute_getFileMetadata_Cached(mocker, dli, mockSE): def test_DLI_execute_FirstDownFailed(mocker, dli, mockSE): """When getFileMetadata fails for the first SE, we should fall back to the second.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) mockObjectSE = mockSE.return_value mockObjectSE.getFileMetadata.return_value = S_OK( @@ -238,7 +232,6 @@ def test_DLI_execute_FirstDownFailed(mocker, dli, mockSE): def test_DLI_execute_AllDownFailed(mocker, dli, mockSE): """When getFileMetadata fails for the first SE, we should fall back to the second.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) mockObjectSE = mockSE.return_value mockObjectSE.getFileMetadata.return_value = S_OK( @@ -255,7 +248,6 @@ def test_DLI_execute_AllDownFailed(mocker, dli, mockSE): def test_DLI_execute_NoLocal(mocker, dli, mockSE): """Data only at the remote SE.""" - mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.getSystemSection", return_value="pippo") mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2) dli = DownloadInputData( { diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index bfd8801a475..b9247bba477 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -31,7 +31,6 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemSection from DIRAC.Core.Utilities import DEncode, DErrno, List from DIRAC.Core.Utilities.Adler import fileAdler from DIRAC.Core.Utilities.File import getGlobbedFiles, getGlobbedTotalSize @@ -63,7 +62,7 @@ class JobWrapper: def __init__(self, jobID: int | None = None, jobReport: JobReport | None = None): """Standard constructor""" self.initialTiming = os.times() - self.section = os.path.join(getSystemSection("WorkloadManagement/JobWrapper"), "JobWrapper") + self.section = "/Systems/WorkloadManagement/JobWrapper" # Create the accounting report self.accountingReport = AccountingJob() # Initialize for accounting diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py index bc5e8488d11..ec177dabe1f 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py @@ -25,7 +25,6 @@ from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.ConfigurationSystem.Client.Config import gConfig -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance from DIRAC.Core.Utilities.Os import getDiskSpace from DIRAC.Core.Utilities.Profiler import Profiler from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft @@ -124,10 +123,7 @@ def initialize(self): else: self.initialized = True - wms_instance = getSystemInstance("WorkloadManagement") - if not wms_instance: - return S_ERROR("Can not get the WorkloadManagement system instance") - self.section = f"/Systems/WorkloadManagement/{wms_instance}/JobWrapper" + self.section = f"/Systems/WorkloadManagement/JobWrapper" self.log.verbose("Watchdog initialization") # Test control flags diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py index 82316bbd286..ff2bb185152 100644 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py @@ -19,8 +19,6 @@ from DIRAC.WorkloadManagementSystem.JobWrapper.JobExecutionCoordinator import JobExecutionCoordinator from DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper import JobWrapper -getSystemSectionMock = MagicMock() -getSystemSectionMock.return_value = "aValue" gLogger.setLevel("DEBUG") @@ -30,9 +28,6 @@ @pytest.fixture def setup_job_wrapper(mocker): """Fixture to create a JobWrapper instance with a JobExecutionCoordinator.""" - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) def _setup(jobArgs=None, ceArgs=None): jw = JobWrapper() @@ -196,10 +191,6 @@ def test_preProcess_dirac_jobexec_with_args(setup_job_wrapper): @pytest.mark.slow def test_processSuccessfulCommand(mocker): """Test the process method of the JobWrapper class: most common scenario.""" - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper() jw.jobArgs = {"CPUTime": 100, "Memory": 1} @@ -227,10 +218,6 @@ def test_processSuccessfulCommand(mocker): @pytest.mark.slow def test_processSuccessfulDiracJobExec(mocker): """Test the process method of the JobWrapper class: most common scenario with dirac-jobexec.""" - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper() jw.jobArgs = {"CPUTime": 100, "Memory": 1} @@ -255,10 +242,6 @@ def test_processSuccessfulDiracJobExec(mocker): @pytest.mark.slow def test_processFailedCommand(mocker): """Test the process method of the JobWrapper class: the command fails.""" - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper() jw.jobArgs = {"CPUTime": 100, "Memory": 1} @@ -289,10 +272,6 @@ def test_processFailedCommand(mocker): @pytest.mark.slow def test_processFailedSubprocess(mocker): """Test the process method of the JobWrapper class: the subprocess fails.""" - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) mock_system_call = mocker.patch("DIRAC.Core.Utilities.Subprocess.Subprocess.systemCall") mock_system_call.return_value = S_ERROR("Any problem") mock_system_call = mocker.patch("DIRAC.Core.Utilities.Subprocess.Subprocess.getChildPID") @@ -321,10 +300,6 @@ def test_processFailedSubprocess(mocker): @pytest.mark.slow def test_processQuickExecutionNoWatchdog(mocker): """Test the process method of the JobWrapper class: the payload is too fast to start the watchdog.""" - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper() jw.jobArgs = {"CPUTime": 100, "Memory": 1} @@ -348,10 +323,6 @@ def test_processQuickExecutionNoWatchdog(mocker): @pytest.mark.slow def test_processSubprocessFailureNoPid(mocker): """Test the process method of the JobWrapper class: the subprocess fails and no PID is returned.""" - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) # Test failure in starting the payload process jw = JobWrapper() jw.jobArgs = {} @@ -612,10 +583,6 @@ def test_execute(mocker, executable, args, src, expectedResult): """Test the status of the job after JobWrapper.execute(). The returned value of JobWrapper.execute() is not checked as it can apparently be wrong depending on the shell used. """ - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance", return_value="Value") - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) if src: shutil.copy(os.path.join(src, executable), executable) @@ -684,9 +651,6 @@ def jobIDPath(): @pytest.fixture def setup_another_job_wrapper(mocker, jobIDPath): """Fixture to create a JobWrapper instance with the jobIDPath.""" - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper(jobIDPath) jw.jobIDPath = Path(str(jobIDPath)) jw.failedFlag = False @@ -887,9 +851,6 @@ def test_processJobOutputs_output_data_upload(mocker, setup_another_job_wrapper) def test_resolveInputData(mocker): mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ObjectLoader", side_effect=MagicMock()) - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper() jw.jobArgs["InputData"] = "" @@ -975,9 +936,6 @@ def test_transferInputSandbox(mocker, setup_another_job_wrapper): ) def test_finalize(mocker, failedFlag, expectedRes, finalStates): mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ObjectLoader", side_effect=MagicMock()) - mocker.patch( - "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock - ) jw = JobWrapper() jw.jobArgs = {"Executable": "/bin/ls"} diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index 472da21809a..d25387abc4c 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -31,7 +31,7 @@ services: interval: 5s timeout: 2s retries: 15 - start_period: 60s + start_period: 150s pull_policy: always iam-login-service: diff --git a/tests/Jenkins/dirac-cfg-update-dbs.py b/tests/Jenkins/dirac-cfg-update-dbs.py index aa459d486b0..72001f337c1 100644 --- a/tests/Jenkins/dirac-cfg-update-dbs.py +++ b/tests/Jenkins/dirac-cfg-update-dbs.py @@ -42,10 +42,9 @@ for sct in [ "Systems/DataManagement", - "Systems/DataManagement/Production", - "Systems/DataManagement/Production/Databases", - "Systems/DataManagement/Production/Databases/FileCatalogDB", - "Systems/DataManagement/Production/Databases/MultiVOFileCatalogDB", + "Systems/DataManagement/Databases", + "Systems/DataManagement/Databases/FileCatalogDB", + "Systems/DataManagement/Databases/MultiVOFileCatalogDB", ]: res = csAPI.createSection(sct) if not res["OK"]: @@ -55,13 +54,13 @@ dbHost = os.environ["DB_HOST"] dbPort = os.environ["DB_PORT"] -csAPI.setOption("Systems/DataManagement/Production/Databases/FileCatalogDB/DBName", "FileCatalogDB") -csAPI.setOption("Systems/DataManagement/Production/Databases/FileCatalogDB/Host", dbHost) -csAPI.setOption("Systems/DataManagement/Production/Databases/FileCatalogDB/Port", dbPort) +csAPI.setOption("Systems/DataManagement/Databases/FileCatalogDB/DBName", "FileCatalogDB") +csAPI.setOption("Systems/DataManagement/Databases/FileCatalogDB/Host", dbHost) +csAPI.setOption("Systems/DataManagement/Databases/FileCatalogDB/Port", dbPort) -csAPI.setOption("Systems/DataManagement/Production/Databases/MultiVOFileCatalogDB/DBName", "MultiVOFileCatalogDB") -csAPI.setOption("Systems/DataManagement/Production/Databases/MultiVOFileCatalogDB/Host", dbHost) -csAPI.setOption("Systems/DataManagement/Production/Databases/MultiVOFileCatalogDB/Port", dbPort) +csAPI.setOption("Systems/DataManagement/Databases/MultiVOFileCatalogDB/DBName", "MultiVOFileCatalogDB") +csAPI.setOption("Systems/DataManagement/Databases/MultiVOFileCatalogDB/Host", dbHost) +csAPI.setOption("Systems/DataManagement/Databases/MultiVOFileCatalogDB/Port", dbPort) # Setup other DBs (this is for LHCb - innocuous!) # @@ -84,19 +83,18 @@ for sct in [ "Systems/Bookkeeping", - "Systems/Bookkeeping/Production", - "Systems/Bookkeeping/Production/Databases", - "Systems/Bookkeeping/Production/Databases/BookkeepingDB", + "Systems/Bookkeeping/Databases", + "Systems/Bookkeeping/Databases/BookkeepingDB", ]: res = csAPI.createSection(sct) if not res["OK"]: print(res["Message"]) exit(1) -csAPI.setOption("Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingTNS", "FILL_ME") -csAPI.setOption("Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingUser", "FILL_ME") -csAPI.setOption("Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingPassword", "FILL_ME") -csAPI.setOption("Systems/Bookkeeping/Production/Databases/BookkeepingDB/LHCbDIRACBookkeepingServer", "FILL_ME") +csAPI.setOption("Systems/Bookkeeping/Databases/BookkeepingDB/LHCbDIRACBookkeepingTNS", "FILL_ME") +csAPI.setOption("Systems/Bookkeeping/Databases/BookkeepingDB/LHCbDIRACBookkeepingUser", "FILL_ME") +csAPI.setOption("Systems/Bookkeeping/Databases/BookkeepingDB/LHCbDIRACBookkeepingPassword", "FILL_ME") +csAPI.setOption("Systems/Bookkeeping/Databases/BookkeepingDB/LHCbDIRACBookkeepingServer", "FILL_ME") # Commit csAPI.commit() diff --git a/tests/Jenkins/dirac-cfg-update-server.py b/tests/Jenkins/dirac-cfg-update-server.py index 15b3a4263e0..d7ebd375b91 100644 --- a/tests/Jenkins/dirac-cfg-update-server.py +++ b/tests/Jenkins/dirac-cfg-update-server.py @@ -23,10 +23,10 @@ csAPI = CSAPI() -csAPI.setOption("Systems/WorkloadManagement/Production/Services/SandboxStore/BasePath", f"{setupName}/sandboxes") -csAPI.setOption("Systems/WorkloadManagement/Production/Services/SandboxStore/LogLevel", "DEBUG") -csAPI.setOption("Systems/WorkloadManagement/Production/Services/TornadoSandboxStore/BasePath", f"{setupName}/sandboxes") -csAPI.setOption("Systems/WorkloadManagement/Production/Services/TornadoSandboxStore/LogLevel", "DEBUG") +csAPI.setOption("Systems/WorkloadManagement/Services/SandboxStore/BasePath", f"{setupName}/sandboxes") +csAPI.setOption("Systems/WorkloadManagement/Services/SandboxStore/LogLevel", "DEBUG") +csAPI.setOption("Systems/WorkloadManagement/Services/TornadoSandboxStore/BasePath", f"{setupName}/sandboxes") +csAPI.setOption("Systems/WorkloadManagement/Services/TornadoSandboxStore/LogLevel", "DEBUG") # Now setting a SandboxSE as the following: # ProductionSandboxSE @@ -552,9 +552,9 @@ "TrustedHost,CSAdministrator,JobAdministrator,FullDelegation,ProxyManagement,Operator,ProductionManagement,GenericPilot", ) -# Setting Systems/WorkloadManagement/Production/Executors/Optimizers/JobScheduling/RescheduleDelays +# Setting Systems/WorkloadManagement/Executors/Optimizers/JobScheduling/RescheduleDelays # to avoid having to wait while testing rescheduling -csAPI.setOption("Systems/WorkloadManagement/Production/Executors/Optimizers/JobScheduling/RescheduleDelays", "0") +csAPI.setOption("Systems/WorkloadManagement/Executors/Optimizers/JobScheduling/RescheduleDelays", "0") csAPI.createSection("Systems/WorkloadManagement/Production/JobWrapper/") csAPI.setOption("Systems/WorkloadManagement/Production/JobWrapper/MinOutputDataBufferGB", 1) diff --git a/tests/Jenkins/dirac-cfg-update-services.py b/tests/Jenkins/dirac-cfg-update-services.py index 85dd49251a7..3d618d095c5 100644 --- a/tests/Jenkins/dirac-cfg-update-services.py +++ b/tests/Jenkins/dirac-cfg-update-services.py @@ -59,33 +59,31 @@ multiFC = f"Tornado{multiFC}" for sct in [ - "Systems/DataManagement/Production/Services", - f"Systems/DataManagement/Production/Services/{fc}", - f"Systems/DataManagement/Production/Services/{multiFC}", + "Systems/DataManagement/Services", + f"Systems/DataManagement/Services/{fc}", + f"Systems/DataManagement/Services/{multiFC}", ]: res = csAPI.createSection(sct) if not res["OK"]: print(res["Message"]) sys.exit(1) -csAPI.setOption(f"Systems/DataManagement/Production/Services/{fc}/DirectoryManager", "DirectoryClosure") -csAPI.setOption(f"Systems/DataManagement/Production/Services/{fc}/FileManager", "FileManagerPs") -csAPI.setOption(f"Systems/DataManagement/Production/Services/{fc}/SecurityManager", "VOMSSecurityManager") -csAPI.setOption(f"Systems/DataManagement/Production/Services/{fc}/UniqueGUID", True) +csAPI.setOption(f"Systems/DataManagement/Services/{fc}/DirectoryManager", "DirectoryClosure") +csAPI.setOption(f"Systems/DataManagement/Services/{fc}/FileManager", "FileManagerPs") +csAPI.setOption(f"Systems/DataManagement/Services/{fc}/SecurityManager", "VOMSSecurityManager") +csAPI.setOption(f"Systems/DataManagement/Services/{fc}/UniqueGUID", True) -csAPI.setOption(f"Systems/DataManagement/Production/Services/{multiFC}/DirectoryManager", "DirectoryClosure") -csAPI.setOption(f"Systems/DataManagement/Production/Services/{multiFC}/FileManager", "FileManagerPs") -csAPI.setOption(f"Systems/DataManagement/Production/Services/{multiFC}/SecurityManager", "NoSecurityManager") -csAPI.setOption(f"Systems/DataManagement/Production/Services/{multiFC}/UniqueGUID", True) +csAPI.setOption(f"Systems/DataManagement/Services/{multiFC}/DirectoryManager", "DirectoryClosure") +csAPI.setOption(f"Systems/DataManagement/Services/{multiFC}/FileManager", "FileManagerPs") +csAPI.setOption(f"Systems/DataManagement/Services/{multiFC}/SecurityManager", "NoSecurityManager") +csAPI.setOption(f"Systems/DataManagement/Services/{multiFC}/UniqueGUID", True) # configure MultiVO metadata related options: -res = csAPI.setOption(f"Systems/DataManagement/Production/Services/{multiFC}/FileMetadata", "MultiVOFileMetadata") +res = csAPI.setOption(f"Systems/DataManagement/Services/{multiFC}/FileMetadata", "MultiVOFileMetadata") if not res["OK"]: print(res["Message"]) sys.exit(1) -res = csAPI.setOption( - f"Systems/DataManagement/Production/Services/{multiFC}/DirectoryMetadata", "MultiVODirectoryMetadata" -) +res = csAPI.setOption(f"Systems/DataManagement/Services/{multiFC}/DirectoryMetadata", "MultiVODirectoryMetadata") if not res["OK"]: print(res["Message"]) sys.exit(1)