Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.0] FIX Multi VO metadata double VO suffix issue fix (#7697) #7708

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -158,17 +158,17 @@ def setMetaParameter(self, dPath, metaName, metaValue, credDict):

def getDirectoryMetadata(self, path, credDict, inherited=True, ownData=True):
"""
Get metadata for the given directory aggregating metadata for the directory itself
and for all the parent directories if inherited flag is True. Get also the non-indexed
metadata parameters.
:param str path: directory path
:param dict credDict: client credential dictionary
:param bool inherited: include parent directories if True
:param bool ownData:
:return: standard Dirac result object + additional MetadataOwner \
and MetadataType dict entries if the operation is successful.
"""
Get metadata for the given directory aggregating metadata for the directory itself
and for all the parent directories if inherited flag is True. Get also the non-indexed
metadata parameters.
:param str path: directory path
:param dict credDict: client credential dictionary
:param bool inherited: include parent directories if True
:param bool ownData:
:return: standard Dirac result object + additional MetadataOwner \
and MetadataType dict entries if the operation is successful.
"""

result = super().getDirectoryMetadata(path, credDict, inherited, ownData)
if not result["OK"]:
Expand All @@ -181,9 +181,16 @@ def getDirectoryMetadata(self, path, credDict, inherited=True, ownData=True):

return result

def findDirIDsByMetadata(self, metaDict, dPath, credDict):
"""Find Directories satisfying the given metadata and being subdirectories of
the given path
def findDirectoriesByMetadata(self, queryDict, path, credDict):
"""
fMetaDict = _getMetaNameDict(metaDict, credDict)
return super().findDirIDsByMetadata(fMetaDict, dPath, credDict)
Find Directory names satisfying the given metadata and being subdirectories of
the given path.
:param dict queryDict: dictionary containing query data
:param str path: starting directory path
:param dict credDict: client credential dictionary
:return: S_OK/S_ERROR, Value list of selected directory paths
"""

fMetaDict = _getMetaNameDict(queryDict, credDict)
return super().findDirectoriesByMetadata(fMetaDict, path, credDict)
11 changes: 10 additions & 1 deletion src/DIRAC/TransformationSystem/Agent/InputDataAgent.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def __init__(self, *args, **kwargs):
self.transClient = TransformationClient()
self.metadataClient = FileCatalogClient()
self.transformationTypes = None
self.multiVO = False

#############################################################################
def initialize(self):
Expand All @@ -64,6 +65,8 @@ def initialize(self):
self.transformationTypes.remove(extendable)
# This is because the Extendables do not use this Agent (have no Input data query)

self.multiVO = self.am_getOption("MultiVO", self.multiVO)

return S_OK()

##############################################################################
Expand Down Expand Up @@ -116,7 +119,13 @@ def execute(self):
# Perform the query to the metadata catalog
self.log.verbose("Using input data query for transformation", "%d: %s" % (transID, str(inputDataQuery)))
start = time.time()
result = self.metadataClient.findFilesByMetadata(inputDataQuery)
mdc = self.metadataClient
if self.multiVO:
ownerDN = transDict["AuthorDN"]
ownerGroup = transDict["AuthorGroup"]
self.log.debug(f"Querying file catalog as {ownerDN}, {ownerGroup}")
mdc = FileCatalogClient(useCertificates=True, delegatedDN=ownerDN, delegatedGroup=ownerGroup)
result = mdc.findFilesByMetadata(inputDataQuery)
rtime = time.time() - start
self.log.verbose("Metadata catalog query time", f": {rtime:.2f} seconds.")
if not result["OK"]:
Expand Down
2 changes: 2 additions & 0 deletions src/DIRAC/TransformationSystem/ConfigTemplate.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ Agents
PollingTime = 120
FullUpdatePeriod = 86400
RefreshOnly = False
# If True, query the FileCatalog as the owner of the transformation, needed for MultiVO*MetaData filecatalogs
MultiVO = False
}
##END
##BEGIN MCExtensionAgent
Expand Down
Loading