From b080e3d3bbcae9c33a0c087129ffc66a94ec60a5 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Fri, 21 Jun 2024 10:28:05 +0200 Subject: [PATCH 1/3] feat(InputDataAgent): allow using transformation author credentials to query filecatalog. This is needed for MultiVO*Metadata file catalog configurations --- .../TransformationSystem/Agent/InputDataAgent.py | 11 ++++++++++- src/DIRAC/TransformationSystem/ConfigTemplate.cfg | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/TransformationSystem/Agent/InputDataAgent.py b/src/DIRAC/TransformationSystem/Agent/InputDataAgent.py index f767cebf61c..9267da1b6a1 100644 --- a/src/DIRAC/TransformationSystem/Agent/InputDataAgent.py +++ b/src/DIRAC/TransformationSystem/Agent/InputDataAgent.py @@ -46,6 +46,7 @@ def __init__(self, *args, **kwargs): self.transClient = TransformationClient() self.metadataClient = FileCatalogClient() self.transformationTypes = None + self.multiVO = False ############################################################################# def initialize(self): @@ -64,6 +65,8 @@ def initialize(self): self.transformationTypes.remove(extendable) # This is because the Extendables do not use this Agent (have no Input data query) + self.multiVO = self.am_getOption("MultiVO", self.multiVO) + return S_OK() ############################################################################## @@ -116,7 +119,13 @@ def execute(self): # Perform the query to the metadata catalog self.log.verbose("Using input data query for transformation", "%d: %s" % (transID, str(inputDataQuery))) start = time.time() - result = self.metadataClient.findFilesByMetadata(inputDataQuery) + mdc = self.metadataClient + if self.multiVO: + ownerDN = transDict["AuthorDN"] + ownerGroup = transDict["AuthorGroup"] + self.log.debug(f"Querying file catalog as {ownerDN}, {ownerGroup}") + mdc = FileCatalogClient(useCertificates=True, delegatedDN=ownerDN, delegatedGroup=ownerGroup) + result = mdc.findFilesByMetadata(inputDataQuery) rtime = time.time() - start self.log.verbose("Metadata catalog query time", f": {rtime:.2f} seconds.") if not result["OK"]: diff --git a/src/DIRAC/TransformationSystem/ConfigTemplate.cfg b/src/DIRAC/TransformationSystem/ConfigTemplate.cfg index f3dbbce2d72..21e21638564 100644 --- a/src/DIRAC/TransformationSystem/ConfigTemplate.cfg +++ b/src/DIRAC/TransformationSystem/ConfigTemplate.cfg @@ -25,6 +25,8 @@ Agents PollingTime = 120 FullUpdatePeriod = 86400 RefreshOnly = False + # If True, query the FileCatalog as the owner of the transformation, needed for MultiVO*MetaData filecatalogs + MultiVO = False } ##END ##BEGIN MCExtensionAgent From 27bb1c24c411a50e57fe3516b219443d28dee58a Mon Sep 17 00:00:00 2001 From: martynia Date: Thu, 27 Jun 2024 17:45:30 +0200 Subject: [PATCH 2/3] fix: add findDirectoriesByMetadata to the derived class, remove findDirIDsByMetadata from derived class) --- .../MultiVODirectoryMetadata.py | 39 +++++++++++-------- .../FileMetadata/MultiVOFileMetadata.py | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py index e0aa1d625d7..879bae5d536 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py @@ -158,17 +158,17 @@ def setMetaParameter(self, dPath, metaName, metaValue, credDict): def getDirectoryMetadata(self, path, credDict, inherited=True, ownData=True): """ - Get metadata for the given directory aggregating metadata for the directory itself - and for all the parent directories if inherited flag is True. Get also the non-indexed - metadata parameters. - - :param str path: directory path - :param dict credDict: client credential dictionary - :param bool inherited: include parent directories if True - :param bool ownData: - :return: standard Dirac result object + additional MetadataOwner \ - and MetadataType dict entries if the operation is successful. - """ + Get metadata for the given directory aggregating metadata for the directory itself + and for all the parent directories if inherited flag is True. Get also the non-indexed + metadata parameters. + + :param str path: directory path + :param dict credDict: client credential dictionary + :param bool inherited: include parent directories if True + :param bool ownData: + :return: standard Dirac result object + additional MetadataOwner \ + and MetadataType dict entries if the operation is successful. + """ result = super().getDirectoryMetadata(path, credDict, inherited, ownData) if not result["OK"]: @@ -181,9 +181,16 @@ def getDirectoryMetadata(self, path, credDict, inherited=True, ownData=True): return result - def findDirIDsByMetadata(self, metaDict, dPath, credDict): - """Find Directories satisfying the given metadata and being subdirectories of - the given path + def findDirectoriesByMetadata(self, queryDict, path, credDict): """ - fMetaDict = _getMetaNameDict(metaDict, credDict) - return super().findDirIDsByMetadata(fMetaDict, dPath, credDict) + Find Directory names satisfying the given metadata and being subdirectories of + the given path. + + :param dict queryDict: dictionary containing query data + :param str path: starting directory path + :param dict credDict: client credential dictionary + :return: S_OK/S_ERROR, Value list of selected directory paths + """ + + fMetaDict = _getMetaNameDict(queryDict, credDict) + return super().findDirectoriesByMetadata(fMetaDict, path, credDict) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py index 67261768d65..02892073b43 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py @@ -120,7 +120,7 @@ def getFileUserMetadata(self, path, credDict): def findFilesByMetadata(self, metaDict, path, credDict): """Find Files satisfying the given metadata - :param dict metaDict: dictionary with the metaquery parameters + :param dict metaDict: dictionary with the metaquery parameters :param dict metaDict: dictionary with the metaquery parameters :param str path: Path to search into :param dict credDict: Dictionary with the user credentials From 621ec601d016cd563a987518447e00e43c6af666 Mon Sep 17 00:00:00 2001 From: martynia Date: Fri, 28 Jun 2024 08:21:48 +0200 Subject: [PATCH 3/3] fix: reformat docs strings. --- .../FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py index 02892073b43..67261768d65 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/MultiVOFileMetadata.py @@ -120,7 +120,7 @@ def getFileUserMetadata(self, path, credDict): def findFilesByMetadata(self, metaDict, path, credDict): """Find Files satisfying the given metadata - :param dict metaDict: dictionary with the metaquery parameters :param dict metaDict: dictionary with the metaquery parameters + :param dict metaDict: dictionary with the metaquery parameters :param str path: Path to search into :param dict credDict: Dictionary with the user credentials