From 3565f21fe08d754dbbded792d28ea3f73cd15dd2 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 31 Oct 2023 12:36:17 +0100 Subject: [PATCH 1/3] fix: SiteDirector acts for a VO --- .../Agent/SiteDirector.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 68f528f67a8..bf20077ae78 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -19,7 +19,9 @@ from DIRAC import S_ERROR, S_OK, gConfig from DIRAC.AccountingSystem.Client.DataStoreClient import gDataStoreClient from DIRAC.AccountingSystem.Client.Types.Pilot import Pilot as PilotAccounting -from DIRAC.AccountingSystem.Client.Types.PilotSubmission import PilotSubmission as PilotSubmissionAccounting +from DIRAC.AccountingSystem.Client.Types.PilotSubmission import ( + PilotSubmission as PilotSubmissionAccounting, +) from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals, Registry from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping @@ -29,15 +31,16 @@ from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter +from DIRAC.Resources.Computing.ComputingElement import ComputingElement from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus -from DIRAC.Resources.Computing.ComputingElement import ComputingElement from DIRAC.WorkloadManagementSystem.Client import PilotStatus -from DIRAC.WorkloadManagementSystem.Client.PilotScopes import PILOT_SCOPES - from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient +from DIRAC.WorkloadManagementSystem.Client.PilotScopes import PILOT_SCOPES from DIRAC.WorkloadManagementSystem.Client.ServerUtils import getPilotAgentsDB -from DIRAC.WorkloadManagementSystem.private.ConfigHelper import findGenericPilotCredentials +from DIRAC.WorkloadManagementSystem.private.ConfigHelper import ( + findGenericPilotCredentials, +) from DIRAC.WorkloadManagementSystem.Service.WMSUtilities import getGridEnv from DIRAC.WorkloadManagementSystem.Utilities.PilotWrapper import ( _writePilotWrapperFile, @@ -126,25 +129,24 @@ def initialize(self): self.gridEnv = getGridEnv() # The SiteDirector is for a particular user community - self.vo = self.am_getOption("VO", "") - if not self.vo: - self.vo = self.am_getOption("Community", "") + self.vo = self.am_getOption("VO", self.am_getOption("Community", "")) if not self.vo: self.vo = CSGlobals.getVO() + if not self.vo: + return S_ERROR("Need a VO") # The SiteDirector is for a particular user group self.group = self.am_getOption("Group", "") # Choose the group for which pilots will be submitted. This is a hack until # we will be able to match pilots to VOs. if not self.group: - if self.vo: - result = Registry.getGroupsForVO(self.vo) - if not result["OK"]: - return result - self.voGroups = [] - for group in result["Value"]: - if "NormalUser" in Registry.getPropertiesForGroup(group): - self.voGroups.append(group) + result = Registry.getGroupsForVO(self.vo) + if not result["OK"]: + return result + self.voGroups = [] + for group in result["Value"]: + if "NormalUser" in Registry.getPropertiesForGroup(group): + self.voGroups.append(group) else: self.voGroups = [self.group] @@ -570,8 +572,7 @@ def _getTQDictForMatching(self): :returns dict: tqDict of task queue descriptions """ tqDict = {"Setup": CSGlobals.getSetup(), "CPUTime": 9999999} - if self.vo: - tqDict["Community"] = self.vo + tqDict["Community"] = self.vo if self.voGroups: tqDict["OwnerGroup"] = self.voGroups @@ -667,8 +668,7 @@ def _getCE(self, queue): if self.queueDict[queue]["Site"] not in self.siteMaskList: ceDict["JobType"] = "Test" - if self.vo: - ceDict["Community"] = self.vo + ceDict["Community"] = self.vo if self.voGroups: ceDict["OwnerGroup"] = self.voGroups @@ -1049,8 +1049,7 @@ def _getPilotOptions(self, queue, **kwargs): # SiteName pilotOptions.append(f"-n {queueDict['Site']}") # VO - if self.vo: - pilotOptions.append(f"--wnVO={self.vo}") + pilotOptions.append(f"--wnVO={self.vo}") # Generic Options if "GenericOptions" in queueDict: From e8816af20eba49265e9c3aa994816cd117bb24d0 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 7 Nov 2023 11:51:43 +0100 Subject: [PATCH 2/3] feat: remove voGroups --- .../Agent/SiteDirector.py | 23 ------------------- .../ConfigTemplate.cfg | 2 -- 2 files changed, 25 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index bf20077ae78..c161147d401 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -82,8 +82,6 @@ def __init__(self, *args, **kwargs): self.gridEnv = "" self.vo = "" self.group = "" - # self.voGroups contain all the eligible user groups for pilots submitted by this SiteDirector - self.voGroups = [] self.pilotDN = "" self.pilotGroup = "" self.platforms = [] @@ -134,21 +132,6 @@ def initialize(self): self.vo = CSGlobals.getVO() if not self.vo: return S_ERROR("Need a VO") - # The SiteDirector is for a particular user group - self.group = self.am_getOption("Group", "") - - # Choose the group for which pilots will be submitted. This is a hack until - # we will be able to match pilots to VOs. - if not self.group: - result = Registry.getGroupsForVO(self.vo) - if not result["OK"]: - return result - self.voGroups = [] - for group in result["Value"]: - if "NormalUser" in Registry.getPropertiesForGroup(group): - self.voGroups.append(group) - else: - self.voGroups = [self.group] # Get the clients self.siteClient = SiteStatus() @@ -227,8 +210,6 @@ def beginExecution(self): tags = None self.log.always("VO:", self.vo) - if self.voGroups: - self.log.always("Group(s):", self.voGroups) self.log.always("Sites:", siteNames) self.log.always("CETypes:", ceTypes) self.log.always("CEs:", ces) @@ -573,8 +554,6 @@ def _getTQDictForMatching(self): """ tqDict = {"Setup": CSGlobals.getSetup(), "CPUTime": 9999999} tqDict["Community"] = self.vo - if self.voGroups: - tqDict["OwnerGroup"] = self.voGroups if self.checkPlatform: platforms = self._getPlatforms() @@ -669,8 +648,6 @@ def _getCE(self, queue): if self.queueDict[queue]["Site"] not in self.siteMaskList: ceDict["JobType"] = "Test" ceDict["Community"] = self.vo - if self.voGroups: - ceDict["OwnerGroup"] = self.voGroups if self.checkPlatform: platform = self.queueDict[queue]["ParametersDict"].get("Platform") diff --git a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg index 131aec67efa..fbbfb60f4bf 100644 --- a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +++ b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg @@ -282,8 +282,6 @@ Agents VO = # VO treated (leave empty for auto-discovery) Community = - # Group treated (leave empty for auto-discovery) - Group = # Grid Environment (leave empty for auto-discovery) GridEnv = # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used From eb9d9ab15404de1fa2e535e9d861ecbacf9617da Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 7 Nov 2023 11:52:21 +0100 Subject: [PATCH 3/3] fix: removed self.group --- src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index c161147d401..55f042973a3 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -81,7 +81,6 @@ def __init__(self, *args, **kwargs): self.gridEnv = "" self.vo = "" - self.group = "" self.pilotDN = "" self.pilotGroup = "" self.platforms = [] @@ -1046,9 +1045,6 @@ def _getPilotOptions(self, queue, **kwargs): if "PipInstallOptions" in queueDict: pilotOptions.append(f"--pipInstallOptions={queueDict['PipInstallOptions']}") - if self.group: - pilotOptions.append(f"-G {self.group}") - return pilotOptions ####################################################################################