Skip to content

Commit

Permalink
fix: more setup removals
Browse files Browse the repository at this point in the history
  • Loading branch information
fstagni committed Nov 29, 2024
1 parent f48deae commit 0835dd8
Show file tree
Hide file tree
Showing 12 changed files with 41 additions and 76 deletions.
22 changes: 10 additions & 12 deletions src/DIRAC/Core/Base/AgentModule.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,25 @@
"""
Base class for all agent modules
"""
import datetime
import importlib.metadata
import inspect
import os
import signal
import threading
import time
import signal
import importlib.metadata
import inspect
import datetime

import psutil

import DIRAC
from DIRAC import S_OK, S_ERROR, gConfig, gLogger, rootPath
from DIRAC.Core.Utilities.File import mkDir
from DIRAC import S_ERROR, S_OK, gConfig, gLogger, rootPath
from DIRAC.ConfigurationSystem.Client import PathFinder
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Utilities import Network, TimeUtilities
from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv
from DIRAC.Core.Utilities.File import mkDir
from DIRAC.Core.Utilities.ReturnValues import isReturnStructure
from DIRAC.ConfigurationSystem.Client import PathFinder
from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv
from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations


class AgentModule:
Expand Down Expand Up @@ -68,7 +69,6 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}):
They are used to populate __codeProperties
The following Options are used from the Configuration:
- /DIRAC/Setup
- Status
- Enabled
- PollingTime default = 120
Expand Down Expand Up @@ -113,7 +113,6 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}):
"loadSection": PathFinder.getAgentSection(loadName),
"cyclesDone": 0,
"totalElapsedTime": 0,
"setup": gConfig.getValue("/DIRAC/Setup", "Unknown"),
"alive": True,
}
self.__moduleProperties["system"], self.__moduleProperties["agentName"] = agentName.split("/")
Expand Down Expand Up @@ -201,7 +200,6 @@ def am_initialize(self, *initArgs):
self.log.notice("=" * 40)
self.log.notice(f"Loaded agent module {self.__moduleProperties['fullName']}")
self.log.notice(f" Site: {DIRAC.siteName()}")
self.log.notice(f" Setup: {gConfig.getValue('/DIRAC/Setup')}")
self.log.notice(f" Agent version: {self.__codeProperties['version']}")
self.log.notice(f" DIRAC version: {DIRAC.version}")
self.log.notice(f" DIRAC platform: {DIRAC.getPlatform()}")
Expand Down
6 changes: 3 additions & 3 deletions src/DIRAC/Core/Base/ExecutorModule.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
Just provides a number of functions used by all executors
"""
import os
from DIRAC import S_OK, S_ERROR, gConfig, gLogger, rootPath

from DIRAC import S_ERROR, S_OK, gConfig, gLogger, rootPath
from DIRAC.ConfigurationSystem.Client import PathFinder
from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv
from DIRAC.Core.Utilities.ReturnValues import isReturnStructure
from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv


class ExecutorModule:
Expand All @@ -21,7 +22,6 @@ def _ex_initialize(cls, exeName, loadName):
"loadSection": PathFinder.getExecutorSection(loadName),
"messagesProcessed": 0,
"reconnects": 0,
"setup": gConfig.getValue("/DIRAC/Setup", "Unknown"),
}
cls.__defaults = {}
cls.__defaults["MonitoringEnabled"] = True
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def main():
site = switch[1]

# imports
from DIRAC import gConfig, gLogger
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
Expand All @@ -68,11 +68,6 @@ def main():

ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
DIRAC.exit(2)

res = getProxyInfo()
if not res["OK"]:
Expand Down
8 changes: 2 additions & 6 deletions src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,15 @@ def main():
sites = switch[1].split(",")

# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers, resolveSEGroup
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers

ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
DIRAC.exit(2)

res = getProxyInfo()
if not res["OK"]:
Expand Down
15 changes: 5 additions & 10 deletions src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ def main():
Script.registerArgument("Comment: Reason of the action")
Script.parseCommandLine(ignoreErrors=True)

from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC import exit as DIRACExit
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC import exit as DIRACExit, gConfig, gLogger
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin

def getBoolean(value):
if value.lower() == "true":
Expand All @@ -42,11 +43,6 @@ def getBoolean(value):
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
exitCode = 2
DIRACExit(exitCode)

# result = promptUser(
# 'All the elements that are associated with this site will be active, '
Expand All @@ -70,10 +66,9 @@ def getBoolean(value):
exitCode = 2
DIRACExit(exitCode)
userName = userName["Value"]
subject = f"{site} is added in site mask for {setup} setup"
body = "Site {} is added to the site mask for {} setup by {} on {}.\n\n".format(
subject = f"{site} is added in site mask"
body = "Site {} is added to the site mask by {} on {}.\n\n".format(
site,
setup,
userName,
time.asctime(),
)
Expand Down
16 changes: 5 additions & 11 deletions src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ def main():
Script.registerArgument("Comment: Reason of the action")
Script.parseCommandLine(ignoreErrors=True)

from DIRAC import exit as DIRACExit, gConfig, gLogger
from DIRAC.Core.Utilities.PromptUser import promptUser
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC import exit as DIRACExit
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin

def getBoolean(value):
if value.lower() == "true":
Expand All @@ -43,11 +43,6 @@ def getBoolean(value):
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
exitCode = 2
DIRACExit(exitCode)

# result = promptUser(
# 'All the elements that are associated with this site will be banned,'
Expand All @@ -71,10 +66,9 @@ def getBoolean(value):
exitCode = 2
DIRACExit(exitCode)
userName = userName["Value"]
subject = f"{site} is banned for {setup} setup"
body = "Site {} is removed from site mask for {} setup by {} on {}.\n\n".format(
subject = f"{site} is banned"
body = "Site {} is removed from site mask by {} on {}.\n\n".format(
site,
setup,
userName,
time.asctime(),
)
Expand Down
4 changes: 0 additions & 4 deletions src/DIRAC/Resources/Computing/SingularityComputingElement.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,6 @@ def __getInstallFlags(self, infoDict=None):
setup = infoDict.get("DefaultSetup")
if not setup:
setup = list(infoDict.get("Setups"))[0]
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
setup = str(setup)

diracProject = "DIRAC"
Expand Down Expand Up @@ -205,8 +203,6 @@ def __getConfigFlags(infoDict=None):
cfgOpts = []

setup = infoDict.get("DefaultSetup")
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
cfgOpts.append(f"-S '{setup}'")

csServers = infoDict.get("ConfigurationServers")
Expand Down
7 changes: 1 addition & 6 deletions src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,12 +610,7 @@ def _getPilotOptions(self, queue: str) -> list[str]:
queueDict = self.queueDict[queue]["ParametersDict"]
pilotOptions = []

setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == "unknown":
self.log.error("Setup is not defined in the configuration")
return []
pilotOptions.append(f"-S {setup}")
opsHelper = Operations(vo=self.vo, setup=setup)
opsHelper = Operations(vo=self.vo)

# Installation defined?
installationName = opsHelper.getValue("Pilot/Installation", "")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
""" Test class for JobWrapper
"""
import os
from pathlib import Path
import shutil
import tempfile
import time
from pathlib import Path
from unittest.mock import MagicMock

import pytest

import DIRAC
from DIRAC import gLogger, S_OK
from DIRAC import S_OK, gLogger
from DIRAC.Core.Utilities import DErrno
from DIRAC.Core.Utilities.ReturnValues import S_ERROR
from DIRAC.DataManagementSystem.Client.test.mock_DM import dm_mock
Expand All @@ -19,7 +19,6 @@
from DIRAC.WorkloadManagementSystem.JobWrapper.JobExecutionCoordinator import JobExecutionCoordinator
from DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper import JobWrapper


gLogger.setLevel("DEBUG")

# -------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -229,7 +228,7 @@ def test_processSuccessfulDiracJobExec(mocker):
jw.outputFile = std_out.name
jw.errorFile = std_err.name
result = jw.process(
command=f"{executable} {os.path.dirname(os.path.abspath(__file__))}/jobDescription.xml --o /DIRAC/Setup=Test",
command=f"{executable} {os.path.dirname(os.path.abspath(__file__))}/jobDescription.xml",
env={},
)

Expand Down Expand Up @@ -573,7 +572,7 @@ def test_postProcess_subprocess_not_complete(setup_job_wrapper, mocker, mock_rep
("script-RESC.sh", None, "src/DIRAC/WorkloadManagementSystem/JobWrapper/test/", "Going to reschedule job"),
(
"src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py",
"src/DIRAC/WorkloadManagementSystem/JobWrapper/test/jobDescription.xml -o /DIRAC/Setup=Test",
"src/DIRAC/WorkloadManagementSystem/JobWrapper/test/jobDescription.xml",
None,
"Application Finished Successfully",
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
these are incorporated to the file.
The module uploads to a web server the latest version of the pilot scripts.
"""
import os
import datetime
import glob
import os
import shutil
import tarfile
import datetime
from typing import Any

from git import Repo

from DIRAC import gLogger, gConfig, S_OK
from DIRAC import S_OK, gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
from DIRAC.Core.Utilities.ReturnValues import DReturnType, DOKReturnType
from DIRAC.Core.Utilities.ReturnValues import DOKReturnType, DReturnType


class PilotCStoJSONSynchronizer:
Expand All @@ -40,7 +40,6 @@ def __init__(self):
# pilot sync default parameters
self.pilotRepo = "https://github.com/DIRACGrid/Pilot.git" # repository of the pilot
self.pilotVORepo = "" # repository of the VO that can contain a pilot extension
self.pilotSetup = gConfig.getValue("/DIRAC/Setup", "")
self.projectDir = ""
# where the find the pilot scripts in the VO pilot repository
self.pilotScriptPath = "Pilot" # where the find the pilot scripts in the pilot repository
Expand Down Expand Up @@ -87,7 +86,7 @@ def getCSDict(self, includeMasterCS: bool = True) -> DReturnType[Any]:
pilotDict.update(opRes["Value"])

# we still need a pilotVOVersion
self.opsHelper = Operations(setup=self.pilotSetup)
self.opsHelper = Operations()
self.pilotVOVersion = self.opsHelper.getValue("/Pilot/Version")
# if self.pilotVORepo is defined and self.pilotVOVersion is not, syncScripts is likely to fail.
if self.pilotVOVersion is None and self.pilotVORepo:
Expand Down
1 change: 0 additions & 1 deletion tests/CI/install_client.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ echo -e "*** $(date -u) **** Got the DIRAC tests ****\n"

source "${DIRAC_CI_SETUP_SCRIPT}"
# shellcheck disable=SC2034
DIRACSETUP=$(< "${INSTALL_CFG_FILE}" grep "Setup = " | cut -f5 -d " ")

echo -e "*** $(date -u) **** Client INSTALLATION START ****\n"

Expand Down
11 changes: 5 additions & 6 deletions tests/Jenkins/utilities.sh
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ getCFGFile() {
# - can get a $DIRAC_RELEASE env var defined
# - or list of $ALTERNATIVE_MODULES
#
# it also wants the env variables $DIRACSETUP and $CSURL
# it also wants the env variable $CSURL
#
# dirac-configure also accepts a env variable $CONFIGUREOPTIONS
# (e.g. useful for extensions or for using the certificates:
Expand Down Expand Up @@ -272,7 +272,7 @@ installDIRAC() {
echo "$PATH"

# now configuring
cmd="dirac-configure -S ${DIRACSETUP} -C ${CSURL} --SkipCAChecks ${CONFIGUREOPTIONS} ${DEBUG}"
cmd="dirac-configure -C ${CSURL} --SkipCAChecks ${CONFIGUREOPTIONS} ${DEBUG}"
if ! bash -c "${cmd}"; then
echo 'ERROR: dirac-configure failed' >&2
exit 1
Expand Down Expand Up @@ -305,7 +305,6 @@ function installDIRACX() {
# it needs the following environment variables:
# $DIRACUSERDN for the DN of the user used to submit the job
# $DIRACUSERROLE for the role of the proxy of the user used to submit the job
# $DIRACSETUP for the setup

submitJob() {
#This has to be executed from the ${CLIENTINSTALLDIR}
Expand All @@ -330,13 +329,13 @@ submitJob() {
set -e
fi

dirac-admin-get-proxy "${DIRACUSERDN}" "${DIRACUSERROLE}" -o /DIRAC/Security/UseServerCertificate=True -o /DIRAC/Security/CertFile=/home/dirac/certs/hostcert.pem -o /DIRAC/Security/KeyFile=/home/dirac/certs/hostkey.pem -o /DIRAC/Setup="${DIRACSETUP}" --out="/tmp/x509up_u${UID}" -ddd
dirac-admin-get-proxy "${DIRACUSERDN}" "${DIRACUSERROLE}" -o /DIRAC/Security/UseServerCertificate=True -o /DIRAC/Security/CertFile=/home/dirac/certs/hostcert.pem -o /DIRAC/Security/KeyFile=/home/dirac/certs/hostkey.pem --out="/tmp/x509up_u${UID}" -ddd
if [[ -f "${TESTCODE}/${VO}DIRAC/tests/Jenkins/dirac-test-job.py" ]]; then
cp "${TESTCODE}/${VO}DIRAC/tests/Jenkins/dirac-test-job.py" "."
else
cp "${TESTCODE}/DIRAC/tests/Jenkins/dirac-test-job.py" "."
fi
python dirac-test-job.py -o "/DIRAC/Setup=${DIRACSETUP}" "${DEBUG}"
python dirac-test-job.py "${DEBUG}"

echo '==> Done submitJob'
}
Expand All @@ -353,7 +352,7 @@ getUserProxy() {
cfgFile="${CLIENTINSTALLDIR}/diracos/etc/dirac.cfg"
fi

if ! python dirac-cfg-update.py -S "${DIRACSETUP}" --cfg "${cfgFile}" -F "${cfgFile}" -o /DIRAC/Security/UseServerCertificate=True -o /DIRAC/Security/CertFile=/home/dirac/certs/hostcert.pem -o /DIRAC/Security/KeyFile=/home/dirac/certs/hostkey.pem "${DEBUG}"; then
if ! python dirac-cfg-update.py --cfg "${cfgFile}" -F "${cfgFile}" -o /DIRAC/Security/UseServerCertificate=True -o /DIRAC/Security/CertFile=/home/dirac/certs/hostcert.pem -o /DIRAC/Security/KeyFile=/home/dirac/certs/hostkey.pem "${DEBUG}"; then
echo 'ERROR: dirac-cfg-update failed' >&2
exit 1
fi
Expand Down

0 comments on commit 0835dd8

Please sign in to comment.