From ec5b25b63cbb4df384d5719586d9514d5bba02dd Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 24 May 2024 16:21:15 +0200 Subject: [PATCH 1/3] test: added a workflow test with input data --- .../Client/DownloadInputData.py | 5 ++++ .../Client/InputDataByProtocol.py | 4 +-- tests/.dirac-ci-config.yaml | 2 +- tests/CI/install_client.sh | 25 +++++++++++++--- .../WorkloadManagementSystem/Test_JobDB.py | 1 + tests/Jenkins/dirac-cfg-update-server.py | 28 +++++++++++++++++- tests/Workflow/Integration/Test_UserJobs.py | 29 ++++++++++++++++--- 7 files changed, 81 insertions(+), 13 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py index 3ded1d0eb8e..bdf522d25cd 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py @@ -52,6 +52,11 @@ def execute(self, dataToResolve=None): to leave room for any produced files. """ + print("in execute") + gLogger.always("in execute, gLogger") + print(self.log) + self.log.always("in execute, self.log") + # Define local configuration options present at every site localSESet = set(self.configuration["LocalSEList"]) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py b/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py index b08eeae1bf9..5f7a6c5e65e 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py @@ -1,4 +1,4 @@ -""" The Input Data By Protocol module wraps around the Replica Management +""" The Input Data By Protocol module wraps around the StorageElement components to provide access to datasets by available site protocols as defined in the CS for the VO. """ @@ -6,8 +6,6 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient -COMPONENT_NAME = "InputDataByProtocol" - class InputDataByProtocol: ############################################################################# diff --git a/tests/.dirac-ci-config.yaml b/tests/.dirac-ci-config.yaml index 126e8a12a65..42c53e67037 100644 --- a/tests/.dirac-ci-config.yaml +++ b/tests/.dirac-ci-config.yaml @@ -1,6 +1,6 @@ config: DIRAC_CI_SETUP_SCRIPT: DIRAC/tests/Jenkins/dirac_ci.sh - PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 1 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug + PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug PILOT_JSON: "{ \"timestamp\": \"2023-02-13T14:34:26.725499\", \"CEs\": { diff --git a/tests/CI/install_client.sh b/tests/CI/install_client.sh index a02c6456e60..4a8cad685a9 100755 --- a/tests/CI/install_client.sh +++ b/tests/CI/install_client.sh @@ -57,13 +57,30 @@ echo -e "*** $(date -u) **** Client INSTALLATION START ****\n" installDIRAC -#-------------------------------------------------------------------------------# -echo -e "*** $(date -u) **** Submit a job ****\n" - echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Submit a job ****\n" + echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";' > test.jdl echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test.jdl echo "]" >> test.jdl -dirac-wms-job-submit test.jdl +dirac-wms-job-submit test.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt + +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** add a file ****\n" + +echo "I like pizza!" > test_lfn.txt +ls -l +echo $? +dirac-dms-add-file LFN:/vo/test_lfn.txt test_lfn.txt S3-DIRECT +echo $? + +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Submit a job with an input ****\n" + +echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";\n InputData = "/vo/test_lfn.txt";' > test_dl.jdl +echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test_dl.jdl +echo "]" >> test_dl.jdl +dirac-wms-job-submit test_dl.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py index 3dacf06c512..d525e933842 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py @@ -154,6 +154,7 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB): CPUTime = 86400; Executable = "dirac-jobexec"; InputData = ""; + InputDataModule = "DIRAC.WorkloadManagementSystem.Client.DownloadInputData"; InputSandbox = {{ "../../Integration/WorkloadManagementSystem/exe-script.py", diff --git a/tests/Jenkins/dirac-cfg-update-server.py b/tests/Jenkins/dirac-cfg-update-server.py index 97824e06ea1..50ac98a3a24 100644 --- a/tests/Jenkins/dirac-cfg-update-server.py +++ b/tests/Jenkins/dirac-cfg-update-server.py @@ -127,6 +127,7 @@ # DIRAC.Jenkins.ch # { # SE = SE-1 +# SE += S3-DIRECT # CEs # { # jenkins.cern.ch @@ -163,7 +164,7 @@ print(res["Message"]) sys.exit(1) -csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1") +csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1,S3-DIRECT") csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/CEType", "Test") csAPI.setOption( "Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/Queues/jenkins-queue_not_important/maxCPUTime", "200000" @@ -435,6 +436,31 @@ csAPI.setOption("Operations/Defaults/Services/Catalogs/CatalogList", "FileCatalog, TSCatalog, MultiVOFileCatalog") +# Adding InputDataPolicy section of Operations +# Operations +# { +# Defaults +# { +# InputDataPolicy +# { +# Download = DIRAC.WorkloadManagementSystem.Client.DownloadInputData +# InputDataModule = DIRAC.WorkloadManagementSystem.Client.DownloadInputData +# } +# } +# } + +res = csAPI.createSection("Operations/Defaults/InputDataPolicy") +if not res["OK"]: + print(res["Message"]) + sys.exit(1) +csAPI.setOption( + "Operations/Defaults/InputDataPolicy/Download", "DIRAC.WorkloadManagementSystem.Client.DownloadInputData" +) +csAPI.setOption( + "Operations/Defaults/InputDataPolicy/InputDataModule", "DIRAC.WorkloadManagementSystem.Client.DownloadInputData" +) + + # Adding DataManagement section of Operations # Operations # { diff --git a/tests/Workflow/Integration/Test_UserJobs.py b/tests/Workflow/Integration/Test_UserJobs.py index 29cbc6663a4..e7f64450eb3 100644 --- a/tests/Workflow/Integration/Test_UserJobs.py +++ b/tests/Workflow/Integration/Test_UserJobs.py @@ -3,22 +3,21 @@ Can be automatized. """ # pylint: disable=protected-access, wrong-import-position, invalid-name, missing-docstring +import multiprocessing import os import sys import unittest -import multiprocessing import DIRAC DIRAC.initialize(extra_config_files=["pilot.cfg"]) # Initialize configuration from DIRAC import gLogger, rootPath +from DIRAC.Interfaces.API.Dirac import Dirac +from DIRAC.Interfaces.API.Job import Job from DIRAC.tests.Utilities.IntegrationTest import IntegrationTest from DIRAC.tests.Utilities.utils import find_all -from DIRAC.Interfaces.API.Job import Job -from DIRAC.Interfaces.API.Dirac import Dirac - class UserJobTestCase(IntegrationTest): """Base class for the UserJob test cases""" @@ -154,6 +153,27 @@ def test_execute(self): self.assertTrue(res["OK"]) +class DownloadSuccess(UserJobTestCase): + def test_execute(self): + """Testing a job that downloads inputs""" + + job = Job() + + job.setName("dl-test") + job.setExecutable("/bin/ls", "-l") + job.setLogLevel("DEBUG") + try: + # This is the standard location in Jenkins + job.setInputSandbox(find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) + except (IndexError, KeyError): + job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) + job.setConfigArgs("pilot.cfg") + job.setInputData("/vo/test_lfn.txt") + job.setInputDataPolicy("Download") + res = job.runLocal(self.d) + self.assertTrue(res["OK"]) + + class MPSuccess(UserJobTestCase): def test_fixed(self): """this tests executes a job that requires exactly 4 processors""" @@ -245,6 +265,7 @@ def test_min1(self): suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldPlusSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(LSSuccess)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DownloadSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccessMinMax)) testResult = unittest.TextTestRunner(verbosity=2).run(suite) From a6e354ff1c52e864f79c64ecb37d5c19ecd8c508 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 24 May 2024 16:38:47 +0200 Subject: [PATCH 2/3] fix: adding log headers to InputDataResolution modules --- .../Client/DownloadInputData.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py index bdf522d25cd..d6c7e9581ce 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py @@ -31,6 +31,10 @@ class DownloadInputData: ############################################################################# def __init__(self, argumentsDict): """Standard constructor""" +<<<<<<< HEAD +======= + self.name = COMPONENT_NAME +>>>>>>> 7b851fb83 (fix: adding log headers to InputDataResolution modules) self.inputData = argumentsDict["InputData"] self.configuration = argumentsDict["Configuration"] self.jobID = self.configuration.get("JobID") @@ -38,7 +42,11 @@ def __init__(self, argumentsDict): self.fileCatalogResult = argumentsDict["FileCatalog"] # By default put each input data file into a separate directory self.inputDataDirectory = argumentsDict.get("InputDataDirectory", "PerFile") +<<<<<<< HEAD +======= + +>>>>>>> 7b851fb83 (fix: adding log headers to InputDataResolution modules) self.log = gLogger.getSubLogger(f"[{self.jobID}]{self.__class__.__name__}") self.log.showHeaders(True) @@ -52,11 +60,6 @@ def execute(self, dataToResolve=None): to leave room for any produced files. """ - print("in execute") - gLogger.always("in execute, gLogger") - print(self.log) - self.log.always("in execute, self.log") - # Define local configuration options present at every site localSESet = set(self.configuration["LocalSEList"]) From d16bda361070ceab0520bf217a2aab8c142638d4 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 5 Jun 2024 14:28:04 +0200 Subject: [PATCH 3/3] style: minor simplifications --- .../WorkloadManagementSystem/Client/DownloadInputData.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py index d6c7e9581ce..3ded1d0eb8e 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py @@ -31,10 +31,6 @@ class DownloadInputData: ############################################################################# def __init__(self, argumentsDict): """Standard constructor""" -<<<<<<< HEAD -======= - self.name = COMPONENT_NAME ->>>>>>> 7b851fb83 (fix: adding log headers to InputDataResolution modules) self.inputData = argumentsDict["InputData"] self.configuration = argumentsDict["Configuration"] self.jobID = self.configuration.get("JobID") @@ -42,11 +38,7 @@ def __init__(self, argumentsDict): self.fileCatalogResult = argumentsDict["FileCatalog"] # By default put each input data file into a separate directory self.inputDataDirectory = argumentsDict.get("InputDataDirectory", "PerFile") -<<<<<<< HEAD -======= - ->>>>>>> 7b851fb83 (fix: adding log headers to InputDataResolution modules) self.log = gLogger.getSubLogger(f"[{self.jobID}]{self.__class__.__name__}") self.log.showHeaders(True)