From 6b242d698e403c1b0952dbf303cf5aa9e465ff7c Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 31 Oct 2023 16:46:54 +0100 Subject: [PATCH 01/11] test: re-added RemoteLogging option --- Pilot/pilotCommands.py | 1 - tests/CI/pilot_newSchema.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 66f0c282..b47239cf 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -498,7 +498,6 @@ def execute(self): # environment variables to add? if self.pp.userEnvVariables: # User-requested environment variables (comma-separated, name and value separated by ":::") - # The case of dirac-install is handled in vanilla DIRAC Pilot newEnvVars = dict( name.split(":::", 1) for name in self.pp.userEnvVariables.replace(" ", "").split(",") ) diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 38b2b34a..ead2fc1d 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -123,7 +123,7 @@ "pilotRepo": "https://github.com/should_not_matter/Pilot.git", "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "False", + "RemoteLogging": "True", "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "PilotLogLevel": "DEBUG" } From fe19446d35ea5ce8d5d7199e48a315ac86784819 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 31 Oct 2023 17:19:06 +0100 Subject: [PATCH 02/11] test: added VO to matrix in main test --- .github/workflows/integration.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a72ddef4..c992dcc4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -18,6 +18,9 @@ jobs: pilot_schema: - pilot_oldSchema.json - pilot_newSchema.json + VO: + - dteam + - gridpp container: python:${{ matrix.python }}-slim @@ -45,7 +48,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} --debug integration-cvmfs: From 39d33434a325e09d049e2728fe97091abfcd9cf4 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 31 Oct 2023 17:26:06 +0100 Subject: [PATCH 03/11] fix: sets the security env variables as first --- Pilot/pilotCommands.py | 45 ++----------------- Pilot/pilotTools.py | 63 +++++++++++++++++++++------ Pilot/tests/Test_simplePilotLogger.py | 2 +- tests/CI/pilot_newSchema.json | 12 ++--- 4 files changed, 60 insertions(+), 62 deletions(-) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index b47239cf..c79b4dff 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -206,6 +206,7 @@ def execute(self): ) self.exitWithError(1) + class InstallDIRAC(CommandBase): """ Source from CVMFS, or install locally """ @@ -525,7 +526,7 @@ class ConfigureBasics(CommandBase): It calls dirac-configure to: - * download, by default, the CAs + * (maybe) download the CAs * creates a standard or custom (defined by self.pp.localConfigFile) cfg file (by default 'pilot.cfg') to be used where all the pilot configuration is to be set, e.g.: * adds to it basic info like the version @@ -604,37 +605,6 @@ def _getBasicsCFG(self): if self.pp.wnVO: self.cfg.append('-o "/Resources/Computing/CEDefaults/VirtualOrganization=%s"' % self.pp.wnVO) - def __checkSecurityDir(self, envName, dirName): - - if envName in os.environ and safe_listdir(os.environ[envName]): - self.log.debug( - "%s is set in the host environment as %s, aligning installEnv to it" - % (envName, os.environ[envName]) - ) - self.pp.installEnv[envName] = os.environ[envName] - else: - self.log.debug("%s is not set in the host environment" % envName) - # try and find it - for candidate in self.pp.CVMFS_locations: - candidateDir = os.path.join(candidate, - 'etc/grid-security', - dirName) - self.log.debug( - "Candidate directory for %s is %s" - % (envName, candidateDir) - ) - if safe_listdir(candidateDir): - - self.log.debug("Setting %s=%s" % (envName, candidateDir)) - self.pp.installEnv[envName] = candidateDir - os.environ[envName] = candidateDir - break - self.log.debug("%s not found or not a directory" % candidateDir) - - if envName not in self.pp.installEnv: - self.log.error("Could not find/set %s" % envName) - sys.exit(1) - def _getSecurityCFG(self): """ Sets security-related env variables, if needed """ @@ -644,17 +614,8 @@ def _getSecurityCFG(self): self.cfg.append("-o /DIRAC/Security/CertFile=%s/hostcert.pem" % self.pp.certsLocation) self.cfg.append("-o /DIRAC/Security/KeyFile=%s/hostkey.pem" % self.pp.certsLocation) - # If DIRAC (or its extension) is installed in CVMFS: + # If DIRAC (or its extension) is installed in CVMFS do not download VOMS and CAs if self.pp.preinstalledEnv: - - self.__checkSecurityDir("X509_CERT_DIR", "certificates") - self.__checkSecurityDir("X509_VOMS_DIR", "vomsdir") - self.__checkSecurityDir("X509_VOMSES", "vomses") - # This is needed for the integration tests - self.pp.installEnv["DIRAC_VOMSES"] = self.pp.installEnv["X509_VOMSES"] - os.environ["DIRAC_VOMSES"] = os.environ["X509_VOMSES"] - - # In any case do not download VOMS and CAs self.cfg.append("-DMH") diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8da986da..9f42feed 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -8,7 +8,6 @@ import imp import json import os -import pickle import re import select import signal @@ -17,7 +16,6 @@ import sys import threading from datetime import datetime -from distutils.version import LooseVersion from functools import partial, wraps from threading import RLock @@ -89,15 +87,6 @@ def parseVersion(releaseVersion, useLegacyStyle): return version -def printVersion(log): - log.info("Running %s" % " ".join(sys.argv)) - try: - with open("%s.run" % sys.argv[0], "w") as fd: - pickle.dump(sys.argv[1:], fd) - except OSError: - pass - - def pythonPathCheck(): try: os.umask(18) # 022 @@ -632,7 +621,13 @@ def sendMessage(url, pilotUUID, method, rawMessage): context = ssl.create_default_context() context.load_verify_locations(capath=caPath) - context.load_cert_chain(cert) + try: + context.load_cert_chain(cert) + except IsADirectoryError: + context.load_cert_chain( + os.path.join(cert, "hostcert.pem"), + os.path.join(cert, "hostkey.pem") + ) res = urlopen(url, data, context=context) res.close() @@ -940,6 +935,48 @@ def __init__(self): # Command line can override options from JSON self.__initCommandLine2() + self.__checkSecurityDir("X509_CERT_DIR", "certificates") + self.__checkSecurityDir("X509_VOMS_DIR", "vomsdir") + self.__checkSecurityDir("X509_VOMSES", "vomses") + # This is needed for the integration tests + self.installEnv["DIRAC_VOMSES"] = self.installEnv["X509_VOMSES"] + os.environ["DIRAC_VOMSES"] = os.environ["X509_VOMSES"] + + if self.useServerCertificate: + self.installEnv["X509_USER_PROXY"] = self.certsLocation + os.environ["X509_USER_PROXY"] = self.certsLocation + + def __checkSecurityDir(self, envName, dirName): + + if envName in os.environ and safe_listdir(os.environ[envName]): + self.log.debug( + "%s is set in the host environment as %s, aligning installEnv to it" + % (envName, os.environ[envName]) + ) + self.installEnv[envName] = os.environ[envName] + else: + self.log.debug("%s is not set in the host environment" % envName) + # try and find it + for candidate in self.CVMFS_locations: + candidateDir = os.path.join(candidate, + 'etc/grid-security', + dirName) + self.log.debug( + "Candidate directory for %s is %s" + % (envName, candidateDir) + ) + if safe_listdir(candidateDir): + self.log.debug("Setting %s=%s" % (envName, candidateDir)) + self.installEnv[envName] = candidateDir + os.environ[envName] = candidateDir + break + self.log.debug("%s not found or not a directory" % candidateDir) + + if envName not in self.installEnv: + self.log.error("Could not find/set %s" % envName) + sys.exit(1) + + def __initCommandLine1(self): """Parses and interpret options on the command line: first pass (essential things)""" @@ -1185,7 +1222,7 @@ def __getVO(self): except IOError as err: self.log.error("Could not read a proxy, setting vo to 'unknown': ", os.strerror(err.errno)) else: - self.log.error("Could not locate a proxy via X509_USER_PROXY, setting vo to 'unknown' ") + self.log.error("Could not locate a proxy via X509_USER_PROXY") # is there a token, and can we get a VO from the token? # TBD diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 701c1efc..44e2e2ed 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -95,7 +95,7 @@ def test_pilotOptions(self, argvmock, mockPaths): pp.gridCEType = "TEST" res = pp.getPilotOptionsDict() - logURL = "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging" + logURL = "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging" self.assertEqual(res.get("RemoteLoggerURL"), logURL) self.assertEqual(pp.loggerURL, logURL) self.assertEqual(res.get("RemoteLogging"), "False") diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index ead2fc1d..1a2c00d5 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -101,13 +101,13 @@ "Version": "VAR_DIRAC_VERSION", "Extensions": "None", "CheckVersion": "False", - "pilotFileServer": "lbvobox70.cern.ch:8443", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "pilotRepoBranch": "does_not_matter", "pilotRepo": "https://github.com/does_not_matter/Pilot.git", "GenericPilotGroup": "gridpp_pilot", "GenericPilotDN": "VAR_USERDN_GRIDPP", "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "UploadSE": "UKI-LT2-IC-HEP-disk", "UploadPath": "/gridpp/pilotlogs/", "LoggingShifterName": "GridPPLogManager", @@ -118,13 +118,13 @@ "Pilot": { "Version": "VAR_DIRAC_VERSION", "CheckVersion": "True", - "pilotFileServer": "lbvobox70.cern.ch:8443", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "pilotRepoBranch": "should_not_matter", "pilotRepo": "https://github.com/should_not_matter/Pilot.git", "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "True", - "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "PilotLogLevel": "DEBUG" } }, @@ -132,11 +132,11 @@ "Pilot": { "Version": "VAR_DIRAC_VERSION", "CheckVersion": "True", - "pilotFileServer": "lbvobox70.cern.ch:8443", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "PilotLogLevel": "DEBUG" } }, From a897891467c18762e5e071ae9c2ac4dc50fc97ed Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 2 Nov 2023 13:51:49 +0100 Subject: [PATCH 04/11] fix: using extraCredentials:hosts --- Pilot/pilotTools.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 9f42feed..92c5a15e 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -610,24 +610,29 @@ def sendMessage(url, pilotUUID, method, rawMessage): :param str rawMessage: a message to be sent, in JSON format :return: None. """ - message = json.dumps((json.dumps(rawMessage), pilotUUID)) - major, minor, micro, _, _ = sys.version_info - if major >= 3: - data = urlencode({"method": method, "args": message}).encode("utf-8") # encode to bytes ! for python3 - else: - data = urlencode({"method": method, "args": message}) caPath = os.getenv("X509_CERT_DIR") cert = os.getenv("X509_USER_PROXY") context = ssl.create_default_context() context.load_verify_locations(capath=caPath) + + message = json.dumps((json.dumps(rawMessage), pilotUUID)) + try: - context.load_cert_chain(cert) - except IsADirectoryError: + context.load_cert_chain(cert) # this is a proxy + raw_data = {"method": method, "args": message} + except IsADirectoryError: # assuming it'a dir containing cert and key context.load_cert_chain( os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem") ) + raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} + + if sys.version_info[0] == 3: + data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 + else: + data = urlencode(raw_data) + res = urlopen(url, data, context=context) res.close() From b596a518659451d116994883bc2a60110ccec889 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 2 Nov 2023 15:01:57 +0100 Subject: [PATCH 05/11] test: running on nodes with CVMFS --- .github/workflows/integration.yml | 28 ++++++++++++++++++++++----- Pilot/tests/Test_Pilot.py | 18 ++++++++++------- Pilot/tests/Test_proxyTools.py | 18 ++++++++--------- Pilot/tests/Test_simplePilotLogger.py | 16 ++++++++++++++- 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c992dcc4..9a37e42d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: - python: + python_version: - 2.7.18 - 3.6.15 - 3.9.17 @@ -22,10 +22,19 @@ jobs: - dteam - gridpp - container: python:${{ matrix.python }}-slim - steps: - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + + - name: Check Python version + run: python --version + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + - name: Retrieve the secret and decode it to a file env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} @@ -148,10 +157,19 @@ jobs: - pilot_oldSchema.json - pilot_newSchema.json - container: python:${{ matrix.python }}-slim - steps: - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + + - name: Check Python version + run: python --version + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + - name: Retrieve the secret and decode it to a file env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index adf69e82..1f5c5986 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -3,18 +3,18 @@ from __future__ import absolute_import, division, print_function -# pylint: disable=protected-access, missing-docstring, invalid-name, line-too-long - -# imports -import unittest import json -import stat -import sys import os import shutil +import stat +import sys + +# pylint: disable=protected-access, missing-docstring, invalid-name, line-too-long +# imports +import unittest -from Pilot.pilotTools import PilotParams from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from Pilot.pilotTools import PilotParams class PilotTestCase(unittest.TestCase): @@ -39,6 +39,10 @@ def setUp(self): }, fp, ) + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() def tearDown(self): for fileProd in [ diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 3cc53209..7a8688cb 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,12 +1,10 @@ -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function -import subprocess -import shlex -import sys import os +import shlex import shutil +import subprocess +import sys import unittest ############################ @@ -49,7 +47,7 @@ def test_getVOPopenFails(self, popenMock): data = fp.read() os.remove(cert) with self.assertRaises(Exception) as exc: - vo = getVO(data) + getVO(data) self.assertEqual(str(exc.exception), msg) @@ -58,7 +56,7 @@ def test_getVOPopenFails(self, popenMock): msg = "command not found: openssl" popenMock.side_effect = OSError(msg) with self.assertRaises(OSError) as exc: - res = getVO(data) + getVO(data) self.assertEqual(str(exc.exception), msg) @patch("Pilot.proxyTools.Popen") @@ -70,7 +68,7 @@ def test_parseASN1Fails(self, popenMock): msg = "Error when invoking openssl asn1parse" with self.assertRaises(Exception) as exc: - res = parseASN1("Anything") + parseASN1("Anything") self.assertEqual(str(exc.exception), msg) @@ -80,7 +78,7 @@ def test_parseASN1Fails(self, popenMock): msg = "command not found: openssl" popenMock.side_effect = OSError(msg) with self.assertRaises(OSError) as exc: - res = parseASN1("Anything") + parseASN1("Anything") self.assertEqual(str(exc.exception), msg) def test_createFakeProxy(self): diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 44e2e2ed..9c176c82 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -34,6 +34,10 @@ def test_pilotParamsInit(self, argvmock): "tests/pilot.json", ] + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() pp = PilotParams() argvmock.__getitem__.assert_called() @@ -89,6 +93,10 @@ def test_pilotOptions(self, argvmock, mockPaths): "/%s/Pilot" % vo ] mockPaths.return_value = paths + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() pp = PilotParams() lTESTcommands = "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements" @@ -112,6 +120,9 @@ def setUp(self): # These temporary files, opened in a binary mode, will act as standard stream pipes for `Popen` self.stdout_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False) self.stderr_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False) + os.environ["X509_CERT_DIR"] = '/some/thing/' + os.environ["X509_USER_PROXY"] = '/some/thing' + def tearDown(self): # At the end of the test, we'll close and remove the created files @@ -122,7 +133,10 @@ def tearDown(self): @patch(("sys.argv")) @patch("subprocess.Popen") def test_executeAndGetOutput(self, popenMock, argvmock): - + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() argvmock.__getitem__.return_value = [ "-d", "-g", From 4fce58270c63fbf80cc714638cc17697cf9308b9 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 3 Nov 2023 15:09:00 +0100 Subject: [PATCH 06/11] test: set RemoteLoggerTimerInterval to 0 --- Pilot/pilotTools.py | 2 +- tests/CI/pilot_newSchema.json | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 92c5a15e..8b72068f 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -850,7 +850,7 @@ def __init__(self): self.pilotCFGFile = "pilot.json" self.pilotLogging = False self.loggerURL = None - self.loggerTimerInterval = 600 + self.loggerTimerInterval = 0 self.pilotUUID = "unknown" self.modules = "" # see dirac-install "-m" option documentation self.userEnvVariables = "" # see dirac-install "--userEnvVariables" option documentation diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 1a2c00d5..9e3e49d2 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -110,6 +110,7 @@ "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "UploadSE": "UKI-LT2-IC-HEP-disk", "UploadPath": "/gridpp/pilotlogs/", + "RemoteLoggerTimerInterval": 0, "LoggingShifterName": "GridPPLogManager", "PilotLogLevel": "DEBUG" } @@ -125,6 +126,7 @@ "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "True", "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerTimerInterval": 0, "PilotLogLevel": "DEBUG" } }, From 3762c638ed5d84c162a3dd86ed5658cc44d8f986 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 3 Nov 2023 15:29:56 +0100 Subject: [PATCH 07/11] test: adding pilotUUID --- .github/workflows/integration.yml | 40 ++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9a37e42d..9052b6e2 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -57,7 +57,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug integration-cvmfs: @@ -105,7 +108,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --pilotUUID="${pilotUUID}" --debug integration-cvmfs_no_env: @@ -140,7 +146,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug @@ -192,7 +201,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --pilotUUID="${pilotUUID}" --debug integration-cvmfs_matching: @@ -236,7 +248,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug ################################## @@ -293,7 +308,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o lbRunOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o lbRunOnly --pilotUUID="${pilotUUID}" --debug ext-lhcb_integration_dirac_installer_no_env: runs-on: ubuntu-latest @@ -337,7 +355,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug ext-lhcb_integration_no_env: runs-on: ubuntu-latest @@ -381,4 +402,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug From 6873d0c65d8dcb81eface5baae9dbff237cd0e09 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 3 Nov 2023 15:47:24 +0100 Subject: [PATCH 08/11] fix: IsADirectoryError is only in python3 --- Pilot/pilotTools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8b72068f..79ec3c3e 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -51,6 +51,11 @@ except NameError: FileNotFoundError = OSError +try: + IsADirectoryError # pylint: disable=used-before-assignment +except NameError: + IsADirectoryError = OSError + # Timer 2.7 and < 3.3 versions issue where Timer is a function if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: from threading import _Timer as Timer # pylint: disable=no-name-in-module From 6d4cd5b896c3b04bd5f2a9b49c9936a81e45ee2a Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 3 Nov 2023 16:12:44 +0100 Subject: [PATCH 09/11] test: removed python versions from integration tests --- .github/workflows/integration.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9052b6e2..b57a2f84 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,10 +8,6 @@ jobs: strategy: matrix: - python_version: - - 2.7.18 - - 3.6.15 - - 3.9.17 dirac_version: - rel-v8r0 - integration @@ -25,12 +21,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: cvmfs-contrib/github-action-cvmfs@v3 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python_version }} - - - name: Check Python version - run: python --version - name: Test CernVM-FS run: ls /cvmfs/dirac.egi.eu @@ -158,10 +148,6 @@ jobs: strategy: matrix: - python: - - 2.7.18 - - 3.6.15 - - 3.9.17 pilot_schema: - pilot_oldSchema.json - pilot_newSchema.json @@ -169,12 +155,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: cvmfs-contrib/github-action-cvmfs@v3 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python_version }} - - - name: Check Python version - run: python --version - name: Test CernVM-FS run: ls /cvmfs/dirac.egi.eu From c7d9c5b69c80ef95b009699078c4e5b981ed55dd Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 3 Nov 2023 16:16:31 +0100 Subject: [PATCH 10/11] test: fix: do not checkVersion --- tests/CI/pilot_newSchema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 9e3e49d2..977001c5 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -118,7 +118,7 @@ "dteam": { "Pilot": { "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "True", + "CheckVersion": "False", "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "pilotRepoBranch": "should_not_matter", "pilotRepo": "https://github.com/should_not_matter/Pilot.git", @@ -133,12 +133,12 @@ "LHCb": { "Pilot": { "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "True", + "CheckVersion": "False", "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerTimerInterval": 0, "PilotLogLevel": "DEBUG" } }, From 0259c4a3d1a666c4db77fb861bc5ab9ce0175b40 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 3 Nov 2023 16:25:39 +0100 Subject: [PATCH 11/11] feat: getting CVMFS locations from pilot_newSchema --- Pilot/pilotTools.py | 4 ++++ tests/CI/pilot_newSchema.json | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 79ec3c3e..6f9a0f30 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1195,6 +1195,10 @@ def __initJSON2(self): self.releaseProject = pilotOptions.get("Project", self.releaseProject) # default from the code. self.log.debug("Release project: %s" % self.releaseProject) + self.CVMFS_locations = pilotOptions.get("CVMFS_locations", self.CVMFS_locations) # default from the code. + self.log.debug("CVMFS locations: %s" % self.CVMFS_locations) + + def getPilotOptionsDict(self): """ Get pilot option dictionary by searching paths in a certain order (commands, logging etc.). diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 977001c5..8caa6f5a 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -112,7 +112,8 @@ "UploadPath": "/gridpp/pilotlogs/", "RemoteLoggerTimerInterval": 0, "LoggingShifterName": "GridPPLogManager", - "PilotLogLevel": "DEBUG" + "PilotLogLevel": "DEBUG", + "CVMFS_locations": ["/cvmfs/grid.cern.ch", "/cvmfs/dirac.egi.eu"] } }, "dteam": { @@ -127,7 +128,8 @@ "RemoteLogging": "True", "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG" + "PilotLogLevel": "DEBUG", + "CVMFS_locations": ["/cvmfs/grid.cern.ch", "/cvmfs/dirac.egi.eu"] } }, "LHCb": { @@ -139,7 +141,8 @@ "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "False", "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG" + "PilotLogLevel": "DEBUG", + "CVMFS_locations": ["/cvmfs/lhcb.cern.ch/", "$VO_LHCB_SW_DIR"] } }, "ConfigurationServers": [