Skip to content

Commit

Permalink
Merge pull request #214 from fstagni/int_test_2
Browse files Browse the repository at this point in the history
[devel] Fixed format of json file in test, added to integration tests matrix
  • Loading branch information
fstagni authored Oct 12, 2023
2 parents f18ac29 + 8816ea1 commit ecaeec6
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 162 deletions.
28 changes: 23 additions & 5 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ jobs:
dirac_version:
- rel-v8r0
- integration
pilot_schema:
- pilot_oldSchema.json
- pilot_newSchema.json

container: python:${{ matrix.python }}-slim

Expand All @@ -34,7 +37,7 @@ jobs:
- name: tests
run: |
cd Pilot
cp ../tests/CI/pilot_oldSchema.json pilot.json
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
Expand All @@ -48,6 +51,12 @@ jobs:
integration-cvmfs:
runs-on: ubuntu-latest

strategy:
matrix:
pilot_schema:
- pilot_oldSchema.json
- pilot_newSchema.json

steps:
- uses: actions/checkout@v4
- uses: cvmfs-contrib/github-action-cvmfs@v3
Expand All @@ -73,7 +82,7 @@ jobs:
DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses
run: |
cd Pilot
cp ../tests/CI/pilot_oldSchema.json pilot.json
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
Expand All @@ -93,6 +102,9 @@ jobs:
- 2.7.18
- 3.6.15
- 3.9.17
pilot_schema:
- pilot_oldSchema.json
- pilot_newSchema.json

container: python:${{ matrix.python }}-slim

Expand All @@ -112,7 +124,7 @@ jobs:
- name: tests
run: |
cd Pilot
cp ../tests/CI/pilot_oldSchema.json pilot.json
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
Expand Down Expand Up @@ -151,7 +163,7 @@ jobs:
DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses
run: |
cd Pilot
cp ../tests/CI/pilot_oldSchema.json pilot.json
cp ../tests/CI/pilot_newSchema.json pilot.json
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
Expand All @@ -172,6 +184,12 @@ jobs:
ext-lhcb_integration:
runs-on: ubuntu-latest

strategy:
matrix:
pilot_schema:
- pilot_oldSchema.json
- pilot_newSchema.json

steps:
- uses: actions/checkout@v4
- uses: cvmfs-contrib/github-action-cvmfs@v3
Expand Down Expand Up @@ -201,7 +219,7 @@ jobs:
cd Pilot
export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot
curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py
cp ../tests/CI/pilot_oldSchema.json pilot.json
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
Expand Down
2 changes: 1 addition & 1 deletion Pilot/pilotCommands.py
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ def execute(self):
if self.pp.useServerCertificate:
pilotOwnerGroup = "certificate_group"
else:
pilotOwnerGroup = self.pp.userGroup
pilotOwnerGroup = self.pp.userGroup or self.pp.wnVO

if self.pp.localConfigFile:
self.cfg.extend(["--cfg", self.pp.localConfigFile]) # this file is as input
Expand Down
4 changes: 2 additions & 2 deletions Pilot/pilotTools.py
Original file line number Diff line number Diff line change
Expand Up @@ -1152,18 +1152,18 @@ def __getSearchPaths(self):
vo = self.__getVO()
paths = [
"/Defaults/Pilot",
"/%s/Pilot" % vo,
"/%s/Pilot" % self.setup,
"/%s/Defaults/Pilot" % vo,
"/%s/%s/Pilot" % (vo, self.setup),
"/%s/Pilot" % vo,
]

return paths

@staticmethod
def getOptionForPaths(paths, inDict):
"""
Get the preferred option from an input dict passed ond a path list. It modifies the inDict.
Get the preferred option from an input dict passed and a path list. It modifies the inDict.
:param list paths: list of paths to walk through to get a preferred option. An option found in
a path which comes later has a preference over options found in earlier paths.
Expand Down
28 changes: 13 additions & 15 deletions Pilot/tests/Test_simplePilotLogger.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@

import json
import os
import sys
import string
import random
import string
import sys
import tempfile

try:
Expand Down Expand Up @@ -49,13 +49,19 @@ def test_getOptionForPaths(self):
jsonFile = os.path.join(basedir, "../../tests/CI/pilot_newSchema.json")
vo = "gridpp"
setup = "DIRAC-Certification"
paths = ["/Defaults/Pilot", "/%s/Pilot" % setup, "/%s/Defaults/Pilot" % vo, "/%s/%s/Pilot" % (vo, setup)]
paths = [
"/Defaults/Pilot",
"/%s/Pilot" % setup,
"/%s/Defaults/Pilot" % vo,
"/%s/%s/Pilot" % (vo, setup),
"/%s/Pilot" % vo
]
with open(jsonFile, "r") as fp:
jsonDict = json.load(fp)
res = PilotParams.getOptionForPaths(paths, jsonDict)
self.assertEqual(res["RemoteLogging"], "True")
self.assertEqual(res["UploadSE"], "UKI-LT2-IC-HEP-disk")
del jsonDict[vo][setup]["Pilot"]["RemoteLogging"] # remove a vo-specific settings, a default value is False:
del jsonDict[vo]["Pilot"]["RemoteLogging"] # remove a vo-specific settings, a default value is False:
res = PilotParams.getOptionForPaths(paths, jsonDict)
self.assertEqual(res["RemoteLogging"], "False")

Expand All @@ -80,19 +86,11 @@ def test_pilotOptions(self, argvmock, mockPaths):
"/%s/Pilot" % setup,
"/%s/Defaults/Pilot" % vo,
"/%s/%s/Pilot" % (vo, setup),
"/%s/Pilot" % vo
]
mockPaths.return_value = paths
pp = PilotParams()
lTESTcommands = [
"CheckWorkerNode",
"InstallDIRAC",
"ConfigureBasics",
"CheckCECapabilities",
"CheckWNCapabilities",
"ConfigureSite",
"ConfigureArchitecture",
"ConfigureCPURequirements",
]
lTESTcommands = "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements"

pp.gridCEType = "TEST"

Expand All @@ -105,7 +103,7 @@ def test_pilotOptions(self, argvmock, mockPaths):
self.assertEqual(res.get("UploadPath"), "/gridpp/pilotlogs/")
self.assertTrue("Commands" in res)
self.assertEqual(res["Commands"].get(pp.gridCEType), lTESTcommands)
self.assertEqual(pp.commands, lTESTcommands)
self.assertEqual(', '.join(pp.commands), lTESTcommands)
self.assertEqual(pp.releaseVersion, "VAR_DIRAC_VERSION")


Expand Down
170 changes: 31 additions & 139 deletions tests/CI/pilot_newSchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -79,156 +79,48 @@
"GridCEType": "LHCbD"
}
},
"GenericPilotDNs": [],
"Defaults": {
"EMail": {
"Production": "[email protected]",
"Logging": "[email protected]"
},
"InputDataPolicy": {
"Default": "DIRAC.WorkloadManagementSystem.Client.DownloadInputData"
},
"DataManagement": {
"ForceSingleSitePerSE": "False",
"AccessProtocols": "srm, root, dips",
"RegistrationProtocols": "srm, dips, root",
"ThirdPartyProtocols": "srm, dips, root",
"WriteProtocols": "srm, root, dips",
"UseCatalogPFN": "False"
},
"Shifter": {
"GridPPLogManager": {
"User": "diracpilottest.grid.hep.ph.ic.ac.uk",
"Group": "gridpp_user"
}
},
"Pilot": {
"RemoteLogging": "False",
"Commands": {
"TEST": [
"CheckWorkerNode",
"InstallDIRAC",
"ConfigureBasics",
"CheckCECapabilities",
"CheckWNCapabilities",
"ConfigureSite",
"ConfigureArchitecture",
"ConfigureCPURequirements"
],
"TEST-MP": [
"CheckWorkerNode",
"InstallDIRAC",
"ConfigureBasics",
"CheckCECapabilities",
"CheckWNCapabilities",
"ConfigureSite",
"ConfigureArchitecture",
"ConfigureCPURequirements"
],
"TEST-FULL": [
"CheckWorkerNode",
"InstallDIRAC",
"ConfigureBasics",
"CheckCECapabilities",
"CheckWNCapabilities",
"ConfigureSite",
"ConfigureArchitecture",
"ConfigureCPURequirements",
"LaunchAgent"
],
"LHCb": [
"CheckWorkerNode",
"LHCbInstallDIRAC",
"LHCbConfigureBasics",
"RegisterPilot",
"CheckCECapabilities",
"LHCbAddCVMFSTags",
"CheckWNCapabilities",
"LHCbConfigureSite",
"LHCbConfigureArchitecture",
"LHCbConfigureCPURequirements"
],
"LHCbD": [
"CheckWorkerNode",
"InstallDIRAC",
"LHCbConfigureBasics",
"RegisterPilot",
"CheckCECapabilities",
"LHCbAddCVMFSTags",
"CheckWNCapabilities",
"LHCbConfigureSite",
"LHCbConfigureArchitecture",
"LHCbConfigureCPURequirements"
]
"TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
"TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
"TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent",
"LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, LHCbConfigureSite, LHCbConfigureArchitecture, LHCbConfigureCPURequirements",
"LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, LHCbConfigureSite, LHCbConfigureArchitecture, LHCbConfigureCPURequirements"
}
}
},
"gridpp": {
"DIRAC-Certification": {
"Pilot": {
"Version": "VAR_DIRAC_VERSION",
"Extensions": "None",
"CheckVersion": "False",
"pilotFileServer": "lbvobox70.cern.ch:8443",
"pilotRepoBranch": "does_not_matter",
"pilotRepo": "https://github.com/does_not_matter/Pilot.git",
"GenericPilotGroup": "gridpp_pilot",
"GenericPilotDN": "VAR_USERDN_GRIDPP",
"RemoteLogging": "True",
"RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
"UploadSE": "UKI-LT2-IC-HEP-disk",
"UploadPath": "/gridpp/pilotlogs/",
"LoggingShifterName": "GridPPLogManager",
"PilotLogLevel": "DEBUG"
},
"Shifter": {
"SAMManager": {
"User": "diracpilottest.grid.hep.ph.ic.ac.uk",
"Group": "gridpp_user"
},
"ProductionManager": {
"User": "diracpilottest.grid.hep.ph.ic.ac.uk",
"Group": "gridpp_user"
},
"DataManager": {
"User": "diracpilottest.grid.hep.ph.ic.ac.uk",
"Group": "gridpp_user"
}
},
"Services": {
"Catalogs": {
"CatalogList": "FileCatalog",
"FileCatalog": {
"Status": "Active",
"AccessType": "ReadWrite"
},
"ICRucioFileCatalog": {
"AccessType": "Read-Write",
"Status": "Active",
"Master": "True"
}
},
"JobMonitoring": {
"usePilotsLoggingFlag": "True",
"DownloadLocation": "https://lbvobox70.cern.ch:8444/WorkloadManagement/TornadoPilotLogging"
}
}
"Pilot": {
"Version": "VAR_DIRAC_VERSION",
"Extensions": "None",
"CheckVersion": "False",
"pilotFileServer": "lbvobox70.cern.ch:8443",
"pilotRepoBranch": "does_not_matter",
"pilotRepo": "https://github.com/does_not_matter/Pilot.git",
"GenericPilotGroup": "gridpp_pilot",
"GenericPilotDN": "VAR_USERDN_GRIDPP",
"RemoteLogging": "True",
"RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
"UploadSE": "UKI-LT2-IC-HEP-disk",
"UploadPath": "/gridpp/pilotlogs/",
"LoggingShifterName": "GridPPLogManager",
"PilotLogLevel": "DEBUG"
}
},
"dteam": {
"DIRAC-Certification": {
"Pilot": {
"Version": "VAR_DIRAC_VERSION",
"CheckVersion": "True",
"pilotFileServer": "lbvobox70.cern.ch:8443",
"pilotRepoBranch": "should_not_matter",
"pilotRepo": "https://github.com/should_not_matter/Pilot.git",
"GenericPilotGroup": "dteam_pilot",
"GenericPilotDN": "VAR_USERDN",
"RemoteLogging": "False",
"RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
"PilotLogLevel": "DEBUG"
}
"Pilot": {
"Version": "VAR_DIRAC_VERSION",
"CheckVersion": "True",
"pilotFileServer": "lbvobox70.cern.ch:8443",
"pilotRepoBranch": "should_not_matter",
"pilotRepo": "https://github.com/should_not_matter/Pilot.git",
"GenericPilotGroup": "dteam_pilot",
"GenericPilotDN": "VAR_USERDN",
"RemoteLogging": "False",
"RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
"PilotLogLevel": "DEBUG"
}
},
"ConfigurationServers": [
Expand Down

0 comments on commit ecaeec6

Please sign in to comment.