Merge pull request #7639 from fstagni/80_logHeaders

[8.0] added log headers to InputDataResolution modules

src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py

@@ -1,22 +1,16 @@
-########################################################################
-# File :    DownloadInputData.py
-# Author :  Stuart Paterson
-########################################################################
-
 """ The Download Input Data module wraps around the Replica Management
-    components to provide access to datasets by available site protocols as
-    defined in the CS for the VO.
+    components to provide access to datasets by downloading locally
 """
 import os
-import tempfile
 import random
+import tempfile
 
-from DIRAC import S_OK, S_ERROR, gLogger
-from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
-from DIRAC.Resources.Storage.StorageElement import StorageElement
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Core.Utilities.Os import getDiskSpace
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
+from DIRAC.Resources.Storage.StorageElement import StorageElement
+from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
 
 COMPONENT_NAME = "DownloadInputData"
 
@@ -37,15 +31,17 @@ class DownloadInputData:
     #############################################################################
     def __init__(self, argumentsDict):
         """Standard constructor"""
-        self.name = COMPONENT_NAME
-        self.log = gLogger.getSubLogger(self.name)
         self.inputData = argumentsDict["InputData"]
         self.configuration = argumentsDict["Configuration"]
+        self.jobID = self.configuration.get("JobID")
         # Warning: this contains not only the SEs but also the file metadata
         self.fileCatalogResult = argumentsDict["FileCatalog"]
         # By default put each input data file into a separate directory
         self.inputDataDirectory = argumentsDict.get("InputDataDirectory", "PerFile")
-        self.jobID = None
+
+        self.log = gLogger.getSubLogger(f"[{self.jobID}]{self.__class__.__name__}")
+        self.log.showHeaders(True)
+
         self.counter = 1
         self.availableSEs = DMSHelpers().getStorageElements()
 
@@ -59,8 +55,6 @@ def execute(self, dataToResolve=None):
         # Define local configuration options present at every site
         localSESet = set(self.configuration["LocalSEList"])
 
-        self.jobID = self.configuration.get("JobID")
-
         if dataToResolve:
             self.log.verbose("Data to resolve passed directly to DownloadInputData module")
             self.inputData = dataToResolve  # e.g. list supplied by another module
@@ -173,10 +167,12 @@ def execute(self, dataToResolve=None):
                     self.log.error(error, lfn)
                     result = {"OK": False}
                 else:
-                    self.log.info("Preliminary checks OK", f"download {lfn} from {seName}:")
+                    self.log.info("Preliminary checks OK", f": now downloading {lfn} from {seName}")
                     result = self._downloadFromSE(lfn, seName, reps, guid)
                     if not result["OK"]:
                         self.log.error("Download failed", f"Tried downloading from SE {seName}: {result['Message']}")
+                    else:
+                        self.log.info(f"Download of {lfn} from {seName} finalized")
             else:
                 result = {"OK": False}
 

src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py

@@ -1,13 +1,8 @@
-########################################################################
-# File :    InputDataByProtocol.py
-# Author :  Stuart Paterson
-########################################################################
-
 """ The Input Data By Protocol module wraps around the Replica Management
     components to provide access to datasets by available site protocols as
     defined in the CS for the VO.
 """
-from DIRAC import S_OK, S_ERROR, gLogger
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
 
@@ -18,12 +13,12 @@ class InputDataByProtocol:
     #############################################################################
     def __init__(self, argumentsDict):
         """Standard constructor"""
-        self.name = COMPONENT_NAME
-        self.log = gLogger.getSubLogger(self.name)
         self.inputData = argumentsDict["InputData"]
         self.configuration = argumentsDict["Configuration"]
+        self.jobID = self.configuration.get("JobID")
         self.fileCatalogResult = argumentsDict["FileCatalog"]
-        self.jobID = None
+        self.log = gLogger.getSubLogger(f"[{self.jobID}]{self.__class__.__name__}")
+        self.log.showHeaders(True)
         # This is because  replicas contain SEs and metadata keys!
         # FIXME: the structure of the dictionary must be fixed to avoid this mess
         self.metaKeys = {
@@ -52,7 +47,6 @@ def execute(self, dataToResolve=None):
 
         # Define local configuration options present at every site
         localSEList = self.configuration["LocalSEList"]
-        self.jobID = self.configuration.get("JobID")
         allReplicas = self.configuration.get("AllReplicas", False)
         if allReplicas:
             self.log.info("All replicas will be used in the resolution")