diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed45bc09b..cea9b76f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,3 +111,24 @@ jobs: - name: Run mypy run: | mypy . + + convert-script: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: environment.yml + init-shell: bash + post-cleanup: 'all' + - name: Set up environment + run: | + pip install git+https://github.com/DIRACGrid/DIRAC.git@integration + pip install . + - name: Run convert script + run: | + cd csSync + git init /tmp/integration/ + DIRAC_COMPAT_ENABLE_CS_CONVERSION=yes python convert-from-legacy.py tests/integration_test.cfg tests/convert_integration_test.yaml /tmp/integration/ + python -c "import yaml; assert(yaml.safe_load('/tmp/integration/default.yml')==yaml.safe_load('tests/integration_test.yaml'))" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 211d75473..408671b1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,4 +33,4 @@ repos: - types-PyYAML - types-cachetools - types-requests - exclude: ^(src/diracx/client/|tests/|build) + exclude: ^(src/diracx/client/|tests/|build|csSync) diff --git a/csSync/convert-from-legacy.py b/csSync/convert-from-legacy.py new file mode 100755 index 000000000..1b231ceca --- /dev/null +++ b/csSync/convert-from-legacy.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +import diraccfg +import yaml +from pydantic import BaseModel + +from diracx.core.config import DEFAULT_CONFIG_FILE, Config + + +def _git_path(value: str) -> Path: + repo = Path(value) + if not (repo / ".git").is_dir(): + raise ValueError(f"{repo} does not appear to be a git repository") + return repo + + +def _list_to_str(original: str) -> list[str]: + return [x.strip() for x in original.split(",") if x.strip()] + + +class IdPConfig(BaseModel): + URL: str + ClientID: str + + +class VOConfig(BaseModel): + DefaultGroup: str + IdP: IdPConfig + UserSubjects: dict[str, str] + + +class ConversionConfig(BaseModel): + VOs: dict[str, VOConfig] + + +def parse_args(): + parser = argparse.ArgumentParser("Convert the legacy DIRAC CS to the new format") + parser.add_argument("old_file", type=Path) + parser.add_argument("conversion_config", type=Path) + parser.add_argument("repo", type=_git_path) + args = parser.parse_args() + + if not os.environ.get("DIRAC_COMPAT_ENABLE_CS_CONVERSION"): + raise RuntimeError( + "DIRAC_COMPAT_ENABLE_CS_CONVERSION must be set for the conversion to be possible" + ) + + main(args.old_file, args.conversion_config, args.repo / DEFAULT_CONFIG_FILE) + + +def main(old_file: Path, conversion_config: Path, new_file: Path): + """Load the old CS and convert it to the new YAML format""" + old_data = old_file.read_text() + cfg = diraccfg.CFG().loadFromBuffer(old_data) + raw = cfg.getAsDict() + + apply_fixes(raw, conversion_config) + + config = Config.parse_obj(raw) + new_data = json.loads(config.json(exclude_unset=True)) + new_file.write_text(yaml.safe_dump(new_data, line_break=False)) + + +def apply_fixes(raw, conversion_config: Path): + """Modify raw in place to make any layout changes between the old and new structure""" + + conv_config = ConversionConfig.parse_obj( + yaml.safe_load(conversion_config.read_text()) + ) + + raw.pop("DiracX", None) + # Remove dips specific parts from the CS + raw["DIRAC"].pop("Extensions", None) + raw["DIRAC"].pop("Framework", None) + raw["DIRAC"].pop("Security", None) + + # This is VOMS specific and no longer reqired + raw["DIRAC"].pop("ConnConf", None) + + # Setups are no longer supported + raw["DIRAC"].pop("DefaultSetup", None) + raw["DIRAC"].pop("Setups", None) + raw["DIRAC"].pop("Configuration", None) + + # All installations are no multi-VO + raw["DIRAC"].pop("VirtualOrganization", None) + + # The default group now lives in /Registry + raw["DIRAC"].pop("DefaultGroup", None) + + # Check that we have the config for all the VOs + vos = set(raw["Registry"]["VO"]) + if non_configured_vos := vos - set(conv_config.VOs): + print(f"{non_configured_vos} don't have a migration config, ignoring") + + # Modify the registry to be fully multi-VO + original_registry = raw.pop("Registry") + raw["Registry"] = {} + + for vo, vo_meta in conv_config.VOs.items(): + raw["Registry"][vo] = { + "IdP": vo_meta.IdP, + "DefaultGroup": vo_meta.DefaultGroup, + "Users": {}, + "Groups": {}, + } + if "DefaultStorageQuota" in original_registry: + raw["Registry"][vo]["DefaultStorageQuota"] = original_registry[ + "DefaultStorageQuota" + ] + if "DefaultProxyLifeTime" in original_registry: + raw["Registry"][vo]["DefaultProxyLifeTime"] = original_registry[ + "DefaultProxyLifeTime" + ] + # Find the groups that belong to this VO + vo_users = set() + for name, info in original_registry["Groups"].items(): + if "VO" not in info: + print( + f"Can't convert group {name} because it is not associated to any VO" + ) + continue + if info.get("VO", None) == vo: + raw["Registry"][vo]["Groups"][name] = { + k: v for k, v in info.items() if k not in {"IdPRole", "VO"} + } + nicknames = {u.strip() for u in info["Users"].split(",") if u.strip()} + vo_users |= nicknames + raw["Registry"][vo]["Groups"][name]["Users"] = [ + vo_meta.UserSubjects[n] + for n in nicknames + if n in vo_meta.UserSubjects + ] + # Find the users that belong to this VO + for name, info in original_registry["Users"].items(): + if name in vo_users: + if subject := vo_meta.UserSubjects.get(name): + raw["Registry"][vo]["Users"][subject] = info | { + "PreferedUsername": name + } + # We ignore the DN and CA + raw["Registry"][vo]["Users"][subject].pop("DN", None) + raw["Registry"][vo]["Users"][subject].pop("CA", None) + + +if __name__ == "__main__": + parse_args() diff --git a/csSync/tests/convert_integration_test.yaml b/csSync/tests/convert_integration_test.yaml new file mode 100644 index 000000000..3c31d0a2b --- /dev/null +++ b/csSync/tests/convert_integration_test.yaml @@ -0,0 +1,19 @@ +VOs: + Jenkins: + DefaultGroup: jenkins_user + IdP: + ClientID: 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd + URL: https://jenkins.invalid/ + UserSubjects: + adminusername: e2cb28ec-1a1e-40ee-a56d-d899b79879ce + ciuser: 26dbe36e-cf5c-4c52-a834-29a1c904ef74 + trialUser: a95ab678-3fa4-41b9-b863-fe62ce8064ce + vo: + DefaultGroup: dirac_user + IdP: + ClientID: 072afab5-ed92-46e0-a61d-4ecbc96e0770 + URL: https://vo.invalid/ + UserSubjects: + adminusername: 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e + ciuser: d3adc733-6588-4d6f-8581-5986b02d0c87 + trialUser: ff2152ff-34f4-4739-b106-3def37e291e3 diff --git a/csSync/tests/integration_test.cfg b/csSync/tests/integration_test.cfg new file mode 100644 index 000000000..1f34dfbea --- /dev/null +++ b/csSync/tests/integration_test.cfg @@ -0,0 +1,1849 @@ +DIRAC +{ + VirtualOrganization = vo + Configuration + { + Name = Production + Version = 2023-10-02 12:38:38.353704 + MasterServer = https://server:9135/Configuration/Server + } + Setups + { + dirac-JenkinsSetup + { + Accounting = Production + Configuration = Production + DataManagement = Production + Framework = Production + Monitoring = Production + RequestManagement = Production + ResourceStatus = Production + StorageManagement = Production + Production = Production + Transformation = Production + WorkloadManagement = Production + Tornado = Production + } + } +} +Systems +{ + Configuration + { + Production + { + Services + { + Server + { + HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py + Port = 9135 + } + TornadoConfiguration + { + Protocol = https + #Subsection to configure authorization over the service + Authorization + { + #Default authorization + Default = authenticated + #Define who can commit new configuration + commitNewData = CSAdministrator + #Define who can roll back the configuration to a previous version + rollbackToVersion = CSAdministrator + #Define who can get version contents + getVersionContents = ServiceAdministrator + getVersionContents += CSAdministrator + forceGlobalConfigurationUpdate = CSAdministrator + } + } + } + URLs + { + Configuration = https://server:8443/Configuration/TornadoConfiguration + } + FailoverURLs + { + } + Agents + { + RucioSynchronizerAgent + { + #Time between cycles in seconds + PollingTime = 120 + } + VOMS2CSAgent + { + #Time between cycles in seconds + PollingTime = 14400 + MailFrom = noreply@dirac.system + #If users will be added automatically + AutoAddUsers = True + #If users will be modified automatically + AutoModifyUsers = True + #Users no longer registered in VOMS are automatically deleted from DIRAC + AutoDeleteUsers = True + #If suspended status is lifted, if lifted in VOMS + AutoLiftSuspendedStatus = True + #Detailed report on users per group send to the VO administrator + DetailedReport = True + #Automatically create user home directory in the File Catalog + MakeHomeDirectory = False + #List of VO names + VO = Any + #Flag to turn to False if you want this agent to write in the CS (more granularity within other options) + DryRun = True + #Name of the plugin to validate or expand user's info. See :py:mod:`DIRAC.ConfigurationSystem.Client.SyncPlugins.DummySyncPlugin` + SyncPluginName = + } + } + } + } + Framework + { + Production + { + Services + { + TornadoComponentMonitoring + { + Protocol = https + Authorization + { + Default = ServiceAdministrator + componentExists = authenticated + getComponents = authenticated + hostExists = authenticated + getHosts = authenticated + installationExists = authenticated + getInstallations = authenticated + updateLog = Operator + } + } + SystemAdministrator + { + Port = 9162 + Authorization + { + Default = ServiceAdministrator + storeHostInfo = Operator + } + } + TornadoNotification + { + Protocol = https + SMSSwitch = sms.switch.ch + Authorization + { + Default = AlarmsManagement + sendMail = authenticated + sendSMS = authenticated + removeNotificationsForUser = authenticated + markNotificationsAsRead = authenticated + getNotifications = authenticated + ping = authenticated + } + } + TornadoProxyManager + { + Protocol = https + #Email to use as a sender for the expiration reminder + MailFrom = "proxymanager@diracgrid.org" + #Description of rules for access to methods + Authorization + { + Default = authenticated + getProxy = FullDelegation + getProxy += LimitedDelegation + getProxy += PrivateLimitedDelegation + getVOMSProxy = FullDelegation + getVOMSProxy += LimitedDelegation + getVOMSProxy += PrivateLimitedDelegation + getProxyWithToken = FullDelegation + getProxyWithToken += LimitedDelegation + getProxyWithToken += PrivateLimitedDelegation + getVOMSProxyWithToken = FullDelegation + getVOMSProxyWithToken += LimitedDelegation + getVOMSProxyWithToken += PrivateLimitedDelegation + getLogContents = ProxyManagement + setPersistency = ProxyManagement + } + } + TornadoTokenManager + { + Protocol = https + #Description of rules for access to methods + Authorization + { + #Settings by default: + Default = authenticated + getUsersTokensInfo = ProxyManagement + } + } + TornadoUserProfileManager + { + Protocol = https + Authorization + { + Default = authenticated + } + } + BundleDelivery + { + Port = 9158 + Authorization + { + Default = authenticated + FileTransfer + { + Default = authenticated + } + } + } + SecurityLogging + { + Port = 9153 + #Directory where log info is kept + DataLocation = data/securityLog + Authorization + { + Default = authenticated + } + } + } + URLs + { + ComponentMonitoring = https://server:8443/Framework/TornadoComponentMonitoring + SystemAdministrator = dips://server:9162/Framework/SystemAdministrator + Notification = https://server:8443/Framework/TornadoNotification + ProxyManager = https://server:8443/Framework/TornadoProxyManager + TokenManager = https://server:8443/Framework/TornadoTokenManager + UserProfileManager = https://server:8443/Framework/TornadoUserProfileManager + BundleDelivery = dips://server:9158/Framework/BundleDelivery + SecurityLogging = dips://server:9153/Framework/SecurityLogging + } + FailoverURLs + { + } + Databases + { + InstalledComponentsDB + { + DBName = InstalledComponentsDB + Host = mysql + Port = 3306 + } + AuthDB + { + DBName = AuthDB + Host = mysql + Port = 3306 + } + NotificationDB + { + DBName = NotificationDB + Host = mysql + Port = 3306 + } + ProxyDB + { + DBName = ProxyDB + Host = mysql + Port = 3306 + } + TokenDB + { + DBName = TokenDB + Host = mysql + Port = 3306 + } + UserProfileDB + { + DBName = UserProfileDB + Host = mysql + Port = 3306 + } + } + } + } + Tornado + { + Production + { + Port = 8443 + } + } + ResourceStatus + { + Production + { + Services + { + TornadoResourceStatus + { + Protocol = https + Authorization + { + Default = SiteManager + select = all + } + } + TornadoPublisher + { + Protocol = https + Authorization + { + Default = Authenticated + } + } + TornadoResourceManagement + { + Protocol = https + Authorization + { + Default = SiteManager + select = all + } + } + } + URLs + { + ResourceStatus = https://server:8443/ResourceStatus/TornadoResourceStatus + Publisher = https://server:8443/ResourceStatus/TornadoPublisher + ResourceManagement = https://server:8443/ResourceStatus/TornadoResourceManagement + } + FailoverURLs + { + } + Databases + { + ResourceStatusDB + { + DBName = ResourceStatusDB + Host = mysql + Port = 3306 + } + ResourceManagementDB + { + DBName = ResourceManagementDB + Host = mysql + Port = 3306 + } + } + Agents + { + ElementInspectorAgent + { + #Time between cycles in seconds + PollingTime = 300 + #Maximum number of threads used by the agent + maxNumberOfThreads = 15 + #Type of element that this agent will run on (Resource or Site) + elementType = Resource + } + EmailAgent + { + #Time between cycles in seconds + PollingTime = 1800 + } + RucioRSSAgent + { + #Time between cycles in seconds + PollingTime = 120 + } + SiteInspectorAgent + { + #Time between cycles in seconds + PollingTime = 300 + #Maximum number of threads used by the agent + maxNumberOfThreads = 15 + } + SummarizeLogsAgent + { + #Time between cycles in seconds + PollingTime = 300 + #Months of history to keep + Months = 36 + } + TokenAgent + { + #Time between cycles in seconds + PollingTime = 3600 + #hours to notify the owner of the token in advance to the token expiration + notifyHours = 12 + #admin e-mail to where to notify about expiring tokens (on top of existing notifications to tokwn owners) + adminMail = + } + } + } + } + DataManagement + { + Production + { + Services + { + StorageElement + { + #Local path where the data is stored + BasePath = storageElement + #Port exposed + Port = 9148 + #Maximum size in MB you allow to store (0 meaning no limits) + MaxStorageSize = 0 + Authorization + { + Default = authenticated + FileTransfer + { + Default = authenticated + } + } + } + SE-1 + { + Module = StorageElement + BasePath = /home/dirac/ServerInstallDIR/Storage/SE-1 + Port = 9148 + } + SE-2 + { + Module = StorageElement + BasePath = /home/dirac/ServerInstallDIR/Storage/SE-2 + Port = 9147 + } + TornadoDataIntegrity + { + Protocol = https + Authorization + { + Default = authenticated + } + } + TornadoFTS3Manager + { + Protocol = https + Authorization + { + Default = authenticated + } + } + TornadoFileCatalog + { + Protocol = https + UserGroupManager = UserAndGroupManagerDB + SEManager = SEManagerDB + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + SecurityManager = VOMSSecurityManager + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + DirectoryManager = DirectoryClosure + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + FileManager = FileManagerPs + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + UniqueGUID = True + GlobalReadAccess = True + LFNPFNConvention = Strong + ResolvePFN = True + DefaultUmask = 509 + VisibleStatus = AprioriGood + Authorization + { + Default = authenticated + } + } + TornadoS3Gateway + { + Protocol = https + Authorization + { + Default = authenticated + } + } + TornadoMultiVOFileCatalog + { + Module = TornadoFileCatalog + Port = 9198 + Protocol = https + Database = MultiVOFileCatalogDB + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + DirectoryManager = DirectoryClosure + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + FileManager = FileManagerPs + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + SecurityManager = NoSecurityManager + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + UniqueGUID = True + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + FileMetadata = MultiVOFileMetadata + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:07 + DirectoryMetadata = MultiVODirectoryMetadata + } + } + URLs + { + StorageElement = dips://server:9148/DataManagement/StorageElement + SE-1 = dips://server:9148/DataManagement/SE-1 + SE-2 = dips://server:9147/DataManagement/SE-2 + DataIntegrity = https://server:8443/DataManagement/TornadoDataIntegrity + FTS3Manager = https://server:8443/DataManagement/TornadoFTS3Manager + FileCatalog = https://server:8443/DataManagement/TornadoFileCatalog + S3Gateway = https://server:8443/DataManagement/TornadoS3Gateway + MultiVOFileCatalog = https://server:9198/DataManagement/TornadoMultiVOFileCatalog + } + FailoverURLs + { + } + Databases + { + DataIntegrityDB + { + DBName = DataIntegrityDB + Host = mysql + Port = 3306 + } + FTS3DB + { + DBName = FTS3DB + Host = mysql + Port = 3306 + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + FileCatalogDB + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + DBName = FileCatalogDB + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Host = mysql + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Port = 3306 + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + MultiVOFileCatalogDB + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + DBName = MultiVOFileCatalogDB + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Host = mysql + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Port = 3306 + } + } + Agents + { + FTS3Agent + { + PollingTime = 120 + MaxThreads = 10 + #How many Operation we will treat in one loop + OperationBulkSize = 20 + #How many Job we will monitor in one loop + JobBulkSize = 20 + #Max number of files to go in a single job + MaxFilesPerJob = 100 + #Max number of attempt per file + MaxAttemptsPerFile = 256 + #days before removing jobs + DeleteGraceDays = 180 + #Max number of deletes per cycle + DeleteLimitPerCycle = 100 + #hours before kicking jobs with old assignment tag + KickAssignedHours = 1 + #Max number of kicks per cycle + KickLimitPerCycle = 100 + #Lifetime in sec of the Proxy we download to delegate to FTS3 (default 12h) + ProxyLifetime = 43200 + } + } + } + } + Accounting + { + Production + { + Databases + { + AccountingDB + { + DBName = AccountingDB + Host = mysql + Port = 3306 + } + } + Services + { + DataStore + { + Port = 9133 + #Run compaction, has to be True for Master, False for others + RunBucketing = True + Authorization + { + Default = authenticated + compactDB = ServiceAdministrator + deleteType = ServiceAdministrator + registerType = ServiceAdministrator + setBucketsLength = ServiceAdministrator + regenerateBuckets = ServiceAdministrator + } + } + ReportGenerator + { + Port = 9134 + #folder relative to instance path, where data is stored + DataLocation = data/accountingGraphs + Authorization + { + Default = authenticated + FileTransfer + { + Default = authenticated + } + } + } + } + URLs + { + DataStore = dips://server:9133/Accounting/DataStore + ReportGenerator = dips://server:9134/Accounting/ReportGenerator + } + FailoverURLs + { + } + } + } + Production + { + Production + { + Databases + { + ProductionDB + { + DBName = ProductionDB + Host = mysql + Port = 3306 + } + } + Services + { + TornadoProductionManager + { + Protocol = https + Authorization + { + Default = authenticated + } + } + } + URLs + { + ProductionManager = https://server:8443/Production/TornadoProductionManager + } + FailoverURLs + { + } + } + } + RequestManagement + { + Production + { + Databases + { + ReqDB + { + DBName = ReqDB + Host = mysql + Port = 3306 + } + } + Services + { + TornadoReqManager + { + Protocol = https + #If > 0, delay retry for this many minutes + ConstantRequestDelay = 0 + Authorization + { + Default = authenticated + } + } + ReqProxy + { + Port = 9161 + #Number of request to sweep at once + SweepSize = 10 + Authorization + { + Default = authenticated + } + } + } + URLs + { + ReqManager = https://server:8443/RequestManagement/TornadoReqManager + ReqProxy = dips://server:9161/RequestManagement/ReqProxy + } + FailoverURLs + { + } + Agents + { + CleanReqDBAgent + { + PollingTime = 60 + ControlDirectory = control/RequestManagement/CleanReqDBAgent + #How many days, until finished requests are deleted + DeleteGraceDays = 60 + #How many requests are deleted per cycle + DeleteLimit = 100 + #If failed requests are deleted + DeleteFailed = False + #How many hours a request can stay assigned + KickGraceHours = 1 + #How many requests are kicked per cycle + KickLimit = 10000 + #Number of Days before a Request is cancelled, + #regardless of State + #if set to 0 (default) Requests are never cancelled + CancelGraceDays = 0 + } + RequestExecutingAgent + { + PollingTime = 60 + #number of Requests to execute per cycle + RequestsPerCycle = 100 + #minimum number of workers process in the ProcessPool + MinProcess = 20 + #maximum number of workers process in the ProcessPool; recommended to set it to the same value as MinProcess + MaxProcess = 20 + #queue depth of the ProcessPool + ProcessPoolQueueSize = 20 + #timeout for the ProcessPool finalization + ProcessPoolTimeout = 900 + #sleep time before retrying to get a free slot in the ProcessPool + ProcessPoolSleep = 5 + #If a positive integer n is given, we fetch n requests at once from the DB. Otherwise, one by one + BulkRequest = 0 + OperationHandlers + { + ForwardDISET + { + Location = DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET + LogLevel = INFO + MaxAttempts = 256 + TimeOut = 120 + } + ReplicateAndRegister + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister + FTSMode = False + FTSBannedGroups = dirac_user + FTSBannedGroups += lhcb_user + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 600 + } + PutAndRegister + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 600 + } + RegisterReplica + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 120 + } + RemoveReplica + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 120 + } + RemoveFile + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 120 + } + RegisterFile + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 120 + } + SetFileStatus + { + Location = DIRAC/TransformationSystem/Agent/RequestOperations/SetFileStatus + LogLevel = INFO + MaxAttempts = 256 + TimeOutPerFile = 120 + } + } + } + } + } + } + StorageManagement + { + Production + { + Databases + { + StorageManagementDB + { + DBName = StorageManagementDB + Host = mysql + Port = 3306 + } + } + Services + { + TornadoStorageManager + { + Protocol = https + Authorization + { + Default = authenticated + } + } + } + URLs + { + StorageManager = https://server:8443/StorageManagement/TornadoStorageManager + } + FailoverURLs + { + } + Agents + { + RequestFinalizationAgent + { + PollingTime = 120 + } + RequestPreparationAgent + { + PollingTime = 120 + } + StageMonitorAgent + { + PollingTime = 120 + #only use these Plugins to query StorageElements. All if empty + StoragePlugins = + } + StageRequestAgent + { + PollingTime = 120 + } + } + } + } + Transformation + { + Production + { + Databases + { + TransformationDB + { + DBName = TransformationDB + Host = mysql + Port = 3306 + } + } + Services + { + TornadoTransformationManager + { + Protocol = https + Authorization + { + Default = authenticated + } + } + } + URLs + { + TransformationManager = https://server:8443/Transformation/TornadoTransformationManager + } + FailoverURLs + { + } + Agents + { + DataRecoveryAgent + { + PollingTime = 3600 + EnableFlag = False + MailTo = + MailFrom = + #List of TransformationIDs that will not be treated + TransformationsToIgnore = + #List of Transformation Statuses to treat + TransformationStatus = Active + TransformationStatus += Completing + #List of transformations that do not have input data, by default Operations/Transformation/ExtendableTransfTypes + TransformationsNoInput = + #List of transformations that do have input data, by default Operations/Transformation/DataProcessing (- ExtendableTransfTypes) + TransformationsWithInput = + #Print every N treated jobs to monitor progress + PrintEvery = 200 + #Instead of obtaining the job information from the JobMonitor service, pick them from the JDL. This is slightly faster but requires the ProductionOutputData information to be in the JDL + JobInfoFromJDLOnly = False + } + InputDataAgent + { + PollingTime = 120 + FullUpdatePeriod = 86400 + RefreshOnly = False + } + MCExtensionAgent + { + PollingTime = 120 + } + RequestTaskAgent + { + #Use a dedicated proxy to submit requests to the RMS + shifterProxy = + #Use delegated credentials. Use this instead of the shifterProxy option (New in v6r20p5) + ShifterCredentials = + #Transformation types to be taken into account by the agent. If the option is empty, + #the value is taken from *Operations/Transformations/DataManipulation* + #with a default of "Replication, Removal" + TransType = + #Location of the transformation plugins + PluginLocation = DIRAC.TransformationSystem.Client.TaskManagerPlugin + #maximum number of threads to use in this agent + maxNumberOfThreads = 15 + #Give this option a value if the agent should submit Requests + SubmitTasks = yes + #Status of transformations for which to submit Requests + SubmitStatus = Active + SubmitStatus += Completing + #Number of tasks to submit in one execution cycle per transformation + TasksPerLoop = 50 + #Give this option a value if the agent should update the status of tasks + MonitorTasks = + #Status of transformations for which to monitor tasks + UpdateTasksStatus = Active + UpdateTasksStatus += Completing + UpdateTasksStatus += Stopped + #Task statuses considered transient that should be monitored for updates + TaskUpdateStatus = Checking + TaskUpdateStatus += Deleted + TaskUpdateStatus += Killed + TaskUpdateStatus += Staging + TaskUpdateStatus += Stalled + TaskUpdateStatus += Matched + TaskUpdateStatus += Scheduled + TaskUpdateStatus += Rescheduled + TaskUpdateStatus += Completed + TaskUpdateStatus += Submitted + TaskUpdateStatus += Assigned + TaskUpdateStatus += Received + TaskUpdateStatus += Waiting + TaskUpdateStatus += Running + #Number of tasks to be updated in one call + TaskUpdateChunkSize = 0 + #Give this option a value if the agent should update the status for files + MonitorFiles = + #Status of transformations for which to monitor Files + UpdateFilesStatus = Active + UpdateFilesStatus += Completing + UpdateFilesStatus += Stopped + #Give this option a value if the agent should check Reserved tasks + CheckReserved = + #Status of transformations for which to check reserved tasks + CheckReservedStatus = Active + CheckReservedStatus += Completing + CheckReservedStatus += Stopped + #Time between cycles in seconds + PollingTime = 120 + } + TransformationAgent + { + #Time between cycles in seconds + PollingTime = 120 + } + TransformationCleaningAgent + { + #MetaData key to use to identify output data + TransfIDMeta = TransformationID + #Location of the OutputData, if the OutputDirectories parameter is not set for + #transformations only 'MetadataCatalog has to be used + DirectoryLocations = TransformationDB + DirectoryLocations += MetadataCatalog + #Enable or disable, default enabled + EnableFlag = True + #How many days to wait before archiving transformations + ArchiveAfter = 7 + #Shifter to use for removal operations, default is empty and + #using the transformation owner for cleanup + shifterProxy = + #Which transformation types to clean + #If not filled, transformation types are taken from + #Operations/Transformations/DataManipulation + #and Operations/Transformations/DataProcessing + TransformationTypes = + #Time between cycles in seconds + PollingTime = 3600 + } + ValidateOutputDataAgent + { + #Time between cycles in seconds + PollingTime = 120 + } + WorkflowTaskAgent + { + #Transformation types to be taken into account by the agent + TransType = MCSimulation + TransType += DataReconstruction + TransType += DataStripping + TransType += MCStripping + TransType += Merge + #Task statuses considered transient that should be monitored for updates + TaskUpdateStatus = Submitted + TaskUpdateStatus += Received + TaskUpdateStatus += Waiting + TaskUpdateStatus += Running + TaskUpdateStatus += Matched + TaskUpdateStatus += Completed + TaskUpdateStatus += Failed + #Status of transformations for which to monitor tasks + UpdateTasksStatus = Active + UpdateTasksStatus += Completing + UpdateTasksStatus += Stopped + #Number of tasks to be updated in one call + TaskUpdateChunkSize = 0 + #Give this option a value if the agent should submit workflow tasks (Jobs) + SubmitTasks = yes + #Status of transformations for which to submit jobs to WMS + SubmitStatus = Active + SubmitStatus += Completing + #Number of tasks to submit in one execution cycle per transformation + TasksPerLoop = 50 + #Use a dedicated proxy to submit jobs to the WMS + shifterProxy = + #Use delegated credentials. Use this instead of the shifterProxy option (New in v6r20p5) + ShifterCredentials = + #Give this option a value if the agent should check Reserved tasks + CheckReserved = + #Give this option a value if the agent should monitor tasks + MonitorTasks = + #Give this option a value if the agent should monitor files + MonitorFiles = + #Status of transformations for which to monitor Files + UpdateFilesStatus = Active + UpdateFilesStatus += Completing + UpdateFilesStatus += Stopped + #Status of transformations for which to check reserved tasks + CheckReservedStatus = Active + CheckReservedStatus += Completing + CheckReservedStatus += Stopped + #Location of the transformation plugins + PluginLocation = DIRAC.TransformationSystem.Client.TaskManagerPlugin + #maximum number of threads to use in this agent + maxNumberOfThreads = 15 + #Time between cycles in seconds + PollingTime = 120 + #Fill in this option if you want to activate bulk submission (for speed up) + BulkSubmission = false + } + } + } + } + WorkloadManagement + { + Production + { + Databases + { + JobDB + { + DBName = JobDB + Host = mysql + Port = 3306 + } + JobLoggingDB + { + DBName = JobLoggingDB + Host = mysql + Port = 3306 + } + PilotAgentsDB + { + DBName = PilotAgentsDB + Host = mysql + Port = 3306 + } + SandboxMetadataDB + { + DBName = SandboxMetadataDB + Host = mysql + Port = 3306 + } + TaskQueueDB + { + DBName = TaskQueueDB + Host = mysql + Port = 3306 + } + } + Services + { + TornadoJobManager + { + Protocol = https + Authorization + { + Default = authenticated + } + } + TornadoJobMonitoring + { + Protocol = https + Authorization + { + Default = authenticated + } + } + TornadoJobStateUpdate + { + Protocol = https + Authorization + { + Default = authenticated + } + } + TornadoPilotLogging + { + Protocol = https + Authorization + { + Default = authenticated + sendMessage = Operator + sendMessage += GenericPilot + getMetadata = Operator + getMetadata += TrustedHost + finaliseLogs = Operator + finaliseLogs += Pilot + finaliseLogs += GenericPilot + } + } + TornadoSandboxStore + { + Protocol = https + LocalSE = ProductionSandboxSE + MaxThreads = 200 + toClientMaxThreads = 100 + Backend = local + MaxSandboxSizeMiB = 10 + SandboxPrefix = Sandbox + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + BasePath = dirac-JenkinsSetup/sandboxes + DelayedExternalDeletion = True + Authorization + { + Default = authenticated + FileTransfer + { + Default = authenticated + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + LogLevel = DEBUG + } + TornadoWMSAdministrator + { + Protocol = https + Authorization + { + Default = Operator + getJobPilotOutput = authenticated + getSiteMask = authenticated + getSiteMaskStatus = authenticated + ping = authenticated + allowSite = SiteManager + allowSite += Operator + banSite = SiteManager + banSite += Operator + } + } + Matcher + { + Port = 9170 + MaxThreads = 20 + Authorization + { + Default = authenticated + getActiveTaskQueues = JobAdministrator + } + } + OptimizationMind + { + Port = 9175 + } + PilotManager + { + Port = 9171 + Authorization + { + Default = authenticated + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SandboxStore + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + BasePath = dirac-JenkinsSetup/sandboxes + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + LogLevel = DEBUG + } + } + URLs + { + JobManager = https://server:8443/WorkloadManagement/TornadoJobManager + JobMonitoring = https://server:8443/WorkloadManagement/TornadoJobMonitoring + JobStateUpdate = https://server:8443/WorkloadManagement/TornadoJobStateUpdate + PilotLogging = https://server:8443/WorkloadManagement/TornadoPilotLogging + SandboxStore = https://server:8443/WorkloadManagement/TornadoSandboxStore + WMSAdministrator = https://server:8443/WorkloadManagement/TornadoWMSAdministrator + Matcher = dips://server:9170/WorkloadManagement/Matcher + OptimizationMind = dips://server:9175/WorkloadManagement/OptimizationMind + PilotManager = dips://server:9171/WorkloadManagement/PilotManager + } + FailoverURLs + { + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Executors + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Optimizers + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + JobScheduling + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + RescheduleDelays = 0 + } + } + } + Agents + { + JobCleaningAgent + { + PollingTime = 3600 + #Maximum number of jobs to be processed in one cycle + MaxJobsAtOnce = 500 + #Maximum number of jobs to be processed in one cycle for HeartBeatLoggingInfo removal + MaxHBJobsAtOnce = 0 + RemoveStatusDelay + { + #Number of days after which Done jobs are removed + Done = 7 + #Number of days after which Killed jobs are removed + Killed = 7 + #Number of days after which Failed jobs are removed + Failed = 7 + #Number of days after which any jobs, irrespective of status is removed (-1 for disabling this feature) + Any = -1 + } + RemoveStatusDelayHB + { + #Number of days after which HeartBeatLoggingInfo for Done jobs are removed, positive to enable + Done = -1 + #Number of days after which HeartBeatLoggingInfo for Killed jobs are removed + Killed = -1 + #Number of days after which HeartBeatLoggingInfo for Failed jobs are removed + Failed = -1 + } + #Which production type jobs _not_ to remove, takes default from Operations/Transformations/DataProcessing + ProductionTypes = + } + PilotLoggingAgent + { + PollingTime = 600 + } + } + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Bookkeeping + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Production + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + Databases + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + BookkeepingDB + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + LHCbDIRACBookkeepingTNS = FILL_ME + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + LHCbDIRACBookkeepingUser = FILL_ME + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + LHCbDIRACBookkeepingPassword = FILL_ME + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:33:22 + LHCbDIRACBookkeepingServer = FILL_ME + } + } + } + } + Monitoring + { + Production + { + Services + { + TornadoMonitoring + { + Protocol = https + Authorization + { + Default = authenticated + FileTransfer + { + Default = authenticated + } + } + } + } + URLs + { + Monitoring = https://server:8443/Monitoring/TornadoMonitoring + } + FailoverURLs + { + } + } + } +} +Registry +{ + Users + { + adminusername + { + DN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser + Email = lhcb-dirac-ci@cern.ch + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:54 + ciuser + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:54 + DN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:54 + Email = lhcb-dirac-ci@cern.ch + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:55 + trialUser + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:55 + DN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=trialUser + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:55 + Email = lhcb-dirac-ci@cern.ch + } + } + Groups + { + dirac_user + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + Users = adminusername + Users += ciuser + Users += trialUser + Properties = NormalUser + VO = vo + } + dirac_admin + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + Users = adminusername + Properties = AlarmsManagement + Properties += ServiceAdministrator + Properties += CSAdministrator + Properties += JobAdministrator + Properties += FullDelegation + Properties += ProxyManagement + Properties += Operator + VO = vo + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:56 + prod + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + Users = adminusername + Users += ciuser + Users += trialUser + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:56 + Properties = Operator + Properties += FullDelegation + Properties += ProxyManagement + Properties += ServiceAdministrator + Properties += JobAdministrator + Properties += CSAdministrator + Properties += AlarmsManagement + Properties += FileCatalogManagement + Properties += SiteManager + Properties += NormalUser + Properties += ProductionManagement + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:57 + jenkins_fcadmin + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + Users = adminusername + Users += ciuser + Users += trialUser + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:57 + Properties = FileCatalogManagement + Properties += NormalUser + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + VO = Jenkins + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + jenkins_user + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + Users = adminusername + Users += ciuser + Users += trialUser + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:58 + Properties = NormalUser + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + VO = Jenkins + } + } + Hosts + { + server + { + DN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=server + Properties = TrustedHost + Properties += CSAdministrator + Properties += JobAdministrator + Properties += FullDelegation + Properties += ProxyManagement + Properties += Operator + Properties += ProductionManagement + } + } + DefaultGroup = dirac_user + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + VO + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Jenkins + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + VOMSName = myVOMS + } + } +} +Operations +{ + Defaults + { + EMail + { + Production = lhcb-dirac-ci@cern.ch + Logging = lhcb-dirac-ci@cern.ch + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + ResourceStatus + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Config + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Cache = 600 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + State = Active + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + FromAddress = fstagni@cern.ch + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + notificationGroups = ShiftersGroup + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + StatusTypes + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + default = all + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + StorageElement = ReadAccess + StorageElement += WriteAccess + StorageElement += CheckAccess + StorageElement += RemoveAccess + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Policies + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AlwaysActiveForResource + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + policyType = AlwaysActive + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + matchParams + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + element = Resource + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AlwaysBannedForSE1SE2 + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + policyType = AlwaysBanned + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + matchParams + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + name = SE1 + name += SE2 + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AlwaysBannedForSite + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + matchParams + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + element = Site + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + policyType = AlwaysBanned + } + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Services + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Catalogs + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + CatalogList = FileCatalog + CatalogList += TSCatalog + CatalogList += MultiVOFileCatalog + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DataManagement + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + RegistrationProtocols = srm + RegistrationProtocols += dips + RegistrationProtocols += s3 + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Transformations + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DataProcessing = MCSimulation + DataProcessing += Merge + DataProcessing += DataReprocessing + } + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:59 + vo + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:59 + Shifter + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:59 + DataManager + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:59 + User = adminusername + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:31:59 + Group = prod + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:00 + TestManager + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:00 + User = adminusername + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:00 + Group = prod + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:00 + ProductionManager + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:00 + User = adminusername + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:00 + Group = prod + } + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:01 + LHCbPR + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:01 + User = adminusername + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:32:01 + Group = prod + } + } + } +} +WebApp +{ + Access + { + upload = TrustedHost + } +} +#@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 +Resources +{ + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 + Sites + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 + DIRAC + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 + DIRAC.Jenkins.ch + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 + Name = aNameWhatSoEver + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 + CEs + { + #@@-dirac_admin - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:30:48 + jenkins.cern.ch + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + CEType = Test + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Queues + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + jenkins-queue_not_important + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + maxCPUTime = 200000 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SI00 = 2400 + } + } + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + jenkins.cern.ch + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + QueuesResources + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Sites + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DIRAC + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DIRAC.Jenkins.ch + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + jenkins.cern.ch + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Queues + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + jenkins-queue_not_important + { + } + } + } + } + } + } + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SE = SE-1 + } + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + StorageElements + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + ProductionSandboxSE + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + BackendType = DISET + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessProtocol = dips + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DIP + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Host = localhost + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Port = 9196 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + ProtocolName = DIP + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Protocol = dips + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Access = remote + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Path = dirac-JenkinsSetup/sandboxes + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SE-1 + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DIP + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Host = server + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Port = 9148 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Protocol = dips + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Path = /DataManagement/SE-1 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Access = remote + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessProtocol = dips + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SE-2 + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + DIP + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Host = server + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Port = 9147 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Protocol = dips + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Path = /DataManagement/SE-2 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Access = remote + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessProtocol = dips + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + S3-DIRECT + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + S3 + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Host = s3-direct + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Port = 9090 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Protocol = s3 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Path = my-first-bucket + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Access = remote + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SecureConnection = False + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Aws_access_key_id = FakeId + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Aws_secret_access_key = True + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessProtocols = s3 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + WriteProtocols = s3 + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + S3-INDIRECT + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + S3 + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Host = s3-direct + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Port = 9090 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Protocol = s3 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Path = my-first-bucket + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Access = remote + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + SecureConnection = False + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessProtocols = s3 + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + WriteProtocols = s3 + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + FileCatalogs + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + FileCatalog + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessType = Read-Write + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Status = Active + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Master = True + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + TSCatalog + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + CatalogType = TSCatalog + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessType = Write + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Status = Active + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + CatalogURL = Transformation/TransformationManager + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + MultiVOFileCatalog + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + CatalogType = FileCatalog + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + AccessType = Read-Write + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + Status = Active + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + CatalogURL = DataManagement/MultiVOFileCatalog + } + } + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + FTSEndpoints + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + FTS3 + { + #@@-prod - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - 2023-10-02 12:36:08 + JENKINS-FTS3 = https://jenkins-fts3.cern.ch:8446 + } + } +} diff --git a/csSync/tests/integration_test.yaml b/csSync/tests/integration_test.yaml new file mode 100644 index 000000000..bbb2c5b83 --- /dev/null +++ b/csSync/tests/integration_test.yaml @@ -0,0 +1,859 @@ +DIRAC: {} +Operations: + Defaults: + DataManagement: + RegistrationProtocols: srm, dips, s3 + EMail: + Logging: lhcb-dirac-ci@cern.ch + Production: lhcb-dirac-ci@cern.ch + ResourceStatus: + Config: + Cache: '600' + FromAddress: fstagni@cern.ch + State: Active + StatusTypes: + StorageElement: ReadAccess, WriteAccess, CheckAccess, RemoveAccess + default: all + notificationGroups: ShiftersGroup + Policies: + AlwaysActiveForResource: + matchParams: + element: Resource + policyType: AlwaysActive + AlwaysBannedForSE1SE2: + matchParams: + name: SE1, SE2 + policyType: AlwaysBanned + AlwaysBannedForSite: + matchParams: + element: Site + policyType: AlwaysBanned + Services: + Catalogs: + CatalogList: FileCatalog, TSCatalog, MultiVOFileCatalog + Transformations: + DataProcessing: MCSimulation, Merge, DataReprocessing + vo: + Shifter: + DataManager: + Group: prod + User: adminusername + LHCbPR: + Group: prod + User: adminusername + ProductionManager: + Group: prod + User: adminusername + TestManager: + Group: prod + User: adminusername +Registry: + Jenkins: + DefaultGroup: jenkins_user + Groups: + jenkins_fcadmin: + Properties: + - FileCatalogManagement + - NormalUser + Users: + - e2cb28ec-1a1e-40ee-a56d-d899b79879ce + - 26dbe36e-cf5c-4c52-a834-29a1c904ef74 + - a95ab678-3fa4-41b9-b863-fe62ce8064ce + jenkins_user: + Properties: + - NormalUser + Users: + - e2cb28ec-1a1e-40ee-a56d-d899b79879ce + - 26dbe36e-cf5c-4c52-a834-29a1c904ef74 + - a95ab678-3fa4-41b9-b863-fe62ce8064ce + IdP: + ClientID: 995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd + URL: https://jenkins.invalid/ + Users: + 26dbe36e-cf5c-4c52-a834-29a1c904ef74: + Email: lhcb-dirac-ci@cern.ch + PreferedUsername: ciuser + a95ab678-3fa4-41b9-b863-fe62ce8064ce: + Email: lhcb-dirac-ci@cern.ch + PreferedUsername: trialUser + e2cb28ec-1a1e-40ee-a56d-d899b79879ce: + Email: lhcb-dirac-ci@cern.ch + PreferedUsername: adminusername + vo: + DefaultGroup: dirac_user + Groups: + dirac_admin: + Properties: + - AlarmsManagement + - ServiceAdministrator + - CSAdministrator + - JobAdministrator + - FullDelegation + - ProxyManagement + - Operator + Users: + - 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e + dirac_user: + Properties: + - NormalUser + Users: + - 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e + - d3adc733-6588-4d6f-8581-5986b02d0c87 + - ff2152ff-34f4-4739-b106-3def37e291e3 + IdP: + ClientID: 072afab5-ed92-46e0-a61d-4ecbc96e0770 + URL: https://vo.invalid/ + Users: + 26b14fc9-6d40-4ca5-b014-6234eaf0fb6e: + Email: lhcb-dirac-ci@cern.ch + PreferedUsername: adminusername + d3adc733-6588-4d6f-8581-5986b02d0c87: + Email: lhcb-dirac-ci@cern.ch + PreferedUsername: ciuser + ff2152ff-34f4-4739-b106-3def37e291e3: + Email: lhcb-dirac-ci@cern.ch + PreferedUsername: trialUser +Resources: + FTSEndpoints: + FTS3: + JENKINS-FTS3: https://jenkins-fts3.cern.ch:8446 + FileCatalogs: + FileCatalog: + AccessType: Read-Write + Master: 'True' + Status: Active + MultiVOFileCatalog: + AccessType: Read-Write + CatalogType: FileCatalog + CatalogURL: DataManagement/MultiVOFileCatalog + Status: Active + TSCatalog: + AccessType: Write + CatalogType: TSCatalog + CatalogURL: Transformation/TransformationManager + Status: Active + Sites: + DIRAC: + DIRAC.Jenkins.ch: + CEs: + jenkins.cern.ch: + CEType: Test + Queues: + jenkins-queue_not_important: + SI00: '2400' + maxCPUTime: '200000' + Name: aNameWhatSoEver + SE: SE-1 + jenkins.cern.ch: + QueuesResources: + Sites: + DIRAC: + DIRAC.Jenkins.ch: + jenkins.cern.ch: + Queues: + jenkins-queue_not_important: {} + StorageElements: + ProductionSandboxSE: + AccessProtocol: dips + BackendType: DISET + DIP: + Access: remote + Host: localhost + Path: dirac-JenkinsSetup/sandboxes + Port: '9196' + Protocol: dips + ProtocolName: DIP + S3-DIRECT: + AccessProtocols: s3 + S3: + Access: remote + Aws_access_key_id: FakeId + Aws_secret_access_key: 'True' + Host: s3-direct + Path: my-first-bucket + Port: '9090' + Protocol: s3 + SecureConnection: 'False' + WriteProtocols: s3 + S3-INDIRECT: + AccessProtocols: s3 + S3: + Access: remote + Host: s3-direct + Path: my-first-bucket + Port: '9090' + Protocol: s3 + SecureConnection: 'False' + WriteProtocols: s3 + SE-1: + AccessProtocol: dips + DIP: + Access: remote + Host: server + Path: /DataManagement/SE-1 + Port: '9148' + Protocol: dips + SE-2: + AccessProtocol: dips + DIP: + Access: remote + Host: server + Path: /DataManagement/SE-2 + Port: '9147' + Protocol: dips +Systems: + Accounting: + Production: + Databases: + AccountingDB: + DBName: AccountingDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + DataStore: + Authorization: + Default: authenticated + compactDB: ServiceAdministrator + deleteType: ServiceAdministrator + regenerateBuckets: ServiceAdministrator + registerType: ServiceAdministrator + setBucketsLength: ServiceAdministrator + Port: '9133' + RunBucketing: 'True' + ReportGenerator: + Authorization: + Default: authenticated + FileTransfer: + Default: authenticated + DataLocation: data/accountingGraphs + Port: '9134' + URLs: + DataStore: dips://server:9133/Accounting/DataStore + ReportGenerator: dips://server:9134/Accounting/ReportGenerator + Bookkeeping: + Production: + Databases: + BookkeepingDB: + LHCbDIRACBookkeepingPassword: FILL_ME + LHCbDIRACBookkeepingServer: FILL_ME + LHCbDIRACBookkeepingTNS: FILL_ME + LHCbDIRACBookkeepingUser: FILL_ME + Configuration: + Production: + Agents: + RucioSynchronizerAgent: + PollingTime: '120' + VOMS2CSAgent: + AutoAddUsers: 'True' + AutoDeleteUsers: 'True' + AutoLiftSuspendedStatus: 'True' + AutoModifyUsers: 'True' + DetailedReport: 'True' + DryRun: 'True' + MailFrom: noreply@dirac.system + MakeHomeDirectory: 'False' + PollingTime: '14400' + SyncPluginName: '' + VO: Any + FailoverURLs: {} + Services: + Server: + HandlerPath: DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py + Port: '9135' + TornadoConfiguration: + Authorization: + Default: authenticated + commitNewData: CSAdministrator + forceGlobalConfigurationUpdate: CSAdministrator + getVersionContents: ServiceAdministrator, CSAdministrator + rollbackToVersion: CSAdministrator + Protocol: https + URLs: + Configuration: https://server:8443/Configuration/TornadoConfiguration + DataManagement: + Production: + Agents: + FTS3Agent: + DeleteGraceDays: '180' + DeleteLimitPerCycle: '100' + JobBulkSize: '20' + KickAssignedHours: '1' + KickLimitPerCycle: '100' + MaxAttemptsPerFile: '256' + MaxFilesPerJob: '100' + MaxThreads: '10' + OperationBulkSize: '20' + PollingTime: '120' + ProxyLifetime: '43200' + Databases: + DataIntegrityDB: + DBName: DataIntegrityDB + Host: mysql + Port: '3306' + FTS3DB: + DBName: FTS3DB + Host: mysql + Port: '3306' + FileCatalogDB: + DBName: FileCatalogDB + Host: mysql + Port: '3306' + MultiVOFileCatalogDB: + DBName: MultiVOFileCatalogDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + SE-1: + BasePath: /home/dirac/ServerInstallDIR/Storage/SE-1 + Module: StorageElement + Port: '9148' + SE-2: + BasePath: /home/dirac/ServerInstallDIR/Storage/SE-2 + Module: StorageElement + Port: '9147' + StorageElement: + Authorization: + Default: authenticated + FileTransfer: + Default: authenticated + BasePath: storageElement + MaxStorageSize: '0' + Port: '9148' + TornadoDataIntegrity: + Authorization: + Default: authenticated + Protocol: https + TornadoFTS3Manager: + Authorization: + Default: authenticated + Protocol: https + TornadoFileCatalog: + Authorization: + Default: authenticated + DefaultUmask: '509' + DirectoryManager: DirectoryClosure + FileManager: FileManagerPs + GlobalReadAccess: 'True' + LFNPFNConvention: Strong + Protocol: https + ResolvePFN: 'True' + SEManager: SEManagerDB + SecurityManager: VOMSSecurityManager + UniqueGUID: 'True' + UserGroupManager: UserAndGroupManagerDB + VisibleStatus: AprioriGood + TornadoMultiVOFileCatalog: + Database: MultiVOFileCatalogDB + DirectoryManager: DirectoryClosure + DirectoryMetadata: MultiVODirectoryMetadata + FileManager: FileManagerPs + FileMetadata: MultiVOFileMetadata + Module: TornadoFileCatalog + Port: '9198' + Protocol: https + SecurityManager: NoSecurityManager + UniqueGUID: 'True' + TornadoS3Gateway: + Authorization: + Default: authenticated + Protocol: https + URLs: + DataIntegrity: https://server:8443/DataManagement/TornadoDataIntegrity + FTS3Manager: https://server:8443/DataManagement/TornadoFTS3Manager + FileCatalog: https://server:8443/DataManagement/TornadoFileCatalog + MultiVOFileCatalog: https://server:9198/DataManagement/TornadoMultiVOFileCatalog + S3Gateway: https://server:8443/DataManagement/TornadoS3Gateway + SE-1: dips://server:9148/DataManagement/SE-1 + SE-2: dips://server:9147/DataManagement/SE-2 + StorageElement: dips://server:9148/DataManagement/StorageElement + Framework: + Production: + Databases: + AuthDB: + DBName: AuthDB + Host: mysql + Port: '3306' + InstalledComponentsDB: + DBName: InstalledComponentsDB + Host: mysql + Port: '3306' + NotificationDB: + DBName: NotificationDB + Host: mysql + Port: '3306' + ProxyDB: + DBName: ProxyDB + Host: mysql + Port: '3306' + TokenDB: + DBName: TokenDB + Host: mysql + Port: '3306' + UserProfileDB: + DBName: UserProfileDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + BundleDelivery: + Authorization: + Default: authenticated + FileTransfer: + Default: authenticated + Port: '9158' + SecurityLogging: + Authorization: + Default: authenticated + DataLocation: data/securityLog + Port: '9153' + SystemAdministrator: + Authorization: + Default: ServiceAdministrator + storeHostInfo: Operator + Port: '9162' + TornadoComponentMonitoring: + Authorization: + Default: ServiceAdministrator + componentExists: authenticated + getComponents: authenticated + getHosts: authenticated + getInstallations: authenticated + hostExists: authenticated + installationExists: authenticated + updateLog: Operator + Protocol: https + TornadoNotification: + Authorization: + Default: AlarmsManagement + getNotifications: authenticated + markNotificationsAsRead: authenticated + ping: authenticated + removeNotificationsForUser: authenticated + sendMail: authenticated + sendSMS: authenticated + Protocol: https + SMSSwitch: sms.switch.ch + TornadoProxyManager: + Authorization: + Default: authenticated + getLogContents: ProxyManagement + getProxy: FullDelegation, LimitedDelegation, PrivateLimitedDelegation + getProxyWithToken: FullDelegation, LimitedDelegation, PrivateLimitedDelegation + getVOMSProxy: FullDelegation, LimitedDelegation, PrivateLimitedDelegation + getVOMSProxyWithToken: FullDelegation, LimitedDelegation, PrivateLimitedDelegation + setPersistency: ProxyManagement + MailFrom: '"proxymanager@diracgrid.org"' + Protocol: https + TornadoTokenManager: + Authorization: + Default: authenticated + getUsersTokensInfo: ProxyManagement + Protocol: https + TornadoUserProfileManager: + Authorization: + Default: authenticated + Protocol: https + URLs: + BundleDelivery: dips://server:9158/Framework/BundleDelivery + ComponentMonitoring: https://server:8443/Framework/TornadoComponentMonitoring + Notification: https://server:8443/Framework/TornadoNotification + ProxyManager: https://server:8443/Framework/TornadoProxyManager + SecurityLogging: dips://server:9153/Framework/SecurityLogging + SystemAdministrator: dips://server:9162/Framework/SystemAdministrator + TokenManager: https://server:8443/Framework/TornadoTokenManager + UserProfileManager: https://server:8443/Framework/TornadoUserProfileManager + Monitoring: + Production: + FailoverURLs: {} + Services: + TornadoMonitoring: + Authorization: + Default: authenticated + FileTransfer: + Default: authenticated + Protocol: https + URLs: + Monitoring: https://server:8443/Monitoring/TornadoMonitoring + Production: + Production: + Databases: + ProductionDB: + DBName: ProductionDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + TornadoProductionManager: + Authorization: + Default: authenticated + Protocol: https + URLs: + ProductionManager: https://server:8443/Production/TornadoProductionManager + RequestManagement: + Production: + Agents: + CleanReqDBAgent: + CancelGraceDays: '0' + ControlDirectory: control/RequestManagement/CleanReqDBAgent + DeleteFailed: 'False' + DeleteGraceDays: '60' + DeleteLimit: '100' + KickGraceHours: '1' + KickLimit: '10000' + PollingTime: '60' + RequestExecutingAgent: + BulkRequest: '0' + MaxProcess: '20' + MinProcess: '20' + OperationHandlers: + ForwardDISET: + Location: DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET + LogLevel: INFO + MaxAttempts: '256' + TimeOut: '120' + PutAndRegister: + Location: DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '600' + RegisterFile: + Location: DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '120' + RegisterReplica: + Location: DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '120' + RemoveFile: + Location: DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '120' + RemoveReplica: + Location: DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '120' + ReplicateAndRegister: + FTSBannedGroups: dirac_user, lhcb_user + FTSMode: 'False' + Location: DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '600' + SetFileStatus: + Location: DIRAC/TransformationSystem/Agent/RequestOperations/SetFileStatus + LogLevel: INFO + MaxAttempts: '256' + TimeOutPerFile: '120' + PollingTime: '60' + ProcessPoolQueueSize: '20' + ProcessPoolSleep: '5' + ProcessPoolTimeout: '900' + RequestsPerCycle: '100' + Databases: + ReqDB: + DBName: ReqDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + ReqProxy: + Authorization: + Default: authenticated + Port: '9161' + SweepSize: '10' + TornadoReqManager: + Authorization: + Default: authenticated + ConstantRequestDelay: '0' + Protocol: https + URLs: + ReqManager: https://server:8443/RequestManagement/TornadoReqManager + ReqProxy: dips://server:9161/RequestManagement/ReqProxy + ResourceStatus: + Production: + Agents: + ElementInspectorAgent: + PollingTime: '300' + elementType: Resource + maxNumberOfThreads: '15' + EmailAgent: + PollingTime: '1800' + RucioRSSAgent: + PollingTime: '120' + SiteInspectorAgent: + PollingTime: '300' + maxNumberOfThreads: '15' + SummarizeLogsAgent: + Months: '36' + PollingTime: '300' + TokenAgent: + PollingTime: '3600' + adminMail: '' + notifyHours: '12' + Databases: + ResourceManagementDB: + DBName: ResourceManagementDB + Host: mysql + Port: '3306' + ResourceStatusDB: + DBName: ResourceStatusDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + TornadoPublisher: + Authorization: + Default: Authenticated + Protocol: https + TornadoResourceManagement: + Authorization: + Default: SiteManager + select: all + Protocol: https + TornadoResourceStatus: + Authorization: + Default: SiteManager + select: all + Protocol: https + URLs: + Publisher: https://server:8443/ResourceStatus/TornadoPublisher + ResourceManagement: https://server:8443/ResourceStatus/TornadoResourceManagement + ResourceStatus: https://server:8443/ResourceStatus/TornadoResourceStatus + StorageManagement: + Production: + Agents: + RequestFinalizationAgent: + PollingTime: '120' + RequestPreparationAgent: + PollingTime: '120' + StageMonitorAgent: + PollingTime: '120' + StoragePlugins: '' + StageRequestAgent: + PollingTime: '120' + Databases: + StorageManagementDB: + DBName: StorageManagementDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + TornadoStorageManager: + Authorization: + Default: authenticated + Protocol: https + URLs: + StorageManager: https://server:8443/StorageManagement/TornadoStorageManager + Tornado: + Production: + Port: '8443' + Transformation: + Production: + Agents: + DataRecoveryAgent: + EnableFlag: 'False' + JobInfoFromJDLOnly: 'False' + MailFrom: '' + MailTo: '' + PollingTime: '3600' + PrintEvery: '200' + TransformationStatus: Active, Completing + TransformationsNoInput: '' + TransformationsToIgnore: '' + TransformationsWithInput: '' + InputDataAgent: + FullUpdatePeriod: '86400' + PollingTime: '120' + RefreshOnly: 'False' + MCExtensionAgent: + PollingTime: '120' + RequestTaskAgent: + CheckReserved: '' + CheckReservedStatus: Active, Completing, Stopped + MonitorFiles: '' + MonitorTasks: '' + PluginLocation: DIRAC.TransformationSystem.Client.TaskManagerPlugin + PollingTime: '120' + ShifterCredentials: '' + SubmitStatus: Active, Completing + SubmitTasks: 'yes' + TaskUpdateChunkSize: '0' + TaskUpdateStatus: Checking, Deleted, Killed, Staging, Stalled, Matched, + Scheduled, Rescheduled, Completed, Submitted, Assigned, Received, Waiting, + Running + TasksPerLoop: '50' + TransType: '' + UpdateFilesStatus: Active, Completing, Stopped + UpdateTasksStatus: Active, Completing, Stopped + maxNumberOfThreads: '15' + shifterProxy: '' + TransformationAgent: + PollingTime: '120' + TransformationCleaningAgent: + ArchiveAfter: '7' + DirectoryLocations: TransformationDB, MetadataCatalog + EnableFlag: 'True' + PollingTime: '3600' + TransfIDMeta: TransformationID + TransformationTypes: '' + shifterProxy: '' + ValidateOutputDataAgent: + PollingTime: '120' + WorkflowTaskAgent: + BulkSubmission: 'false' + CheckReserved: '' + CheckReservedStatus: Active, Completing, Stopped + MonitorFiles: '' + MonitorTasks: '' + PluginLocation: DIRAC.TransformationSystem.Client.TaskManagerPlugin + PollingTime: '120' + ShifterCredentials: '' + SubmitStatus: Active, Completing + SubmitTasks: 'yes' + TaskUpdateChunkSize: '0' + TaskUpdateStatus: Submitted, Received, Waiting, Running, Matched, Completed, + Failed + TasksPerLoop: '50' + TransType: MCSimulation, DataReconstruction, DataStripping, MCStripping, + Merge + UpdateFilesStatus: Active, Completing, Stopped + UpdateTasksStatus: Active, Completing, Stopped + maxNumberOfThreads: '15' + shifterProxy: '' + Databases: + TransformationDB: + DBName: TransformationDB + Host: mysql + Port: '3306' + FailoverURLs: {} + Services: + TornadoTransformationManager: + Authorization: + Default: authenticated + Protocol: https + URLs: + TransformationManager: https://server:8443/Transformation/TornadoTransformationManager + WorkloadManagement: + Production: + Agents: + JobCleaningAgent: + MaxHBJobsAtOnce: '0' + MaxJobsAtOnce: '500' + PollingTime: '3600' + ProductionTypes: '' + RemoveStatusDelay: + Any: '-1' + Done: '7' + Failed: '7' + Killed: '7' + RemoveStatusDelayHB: + Done: '-1' + Failed: '-1' + Killed: '-1' + PilotLoggingAgent: + PollingTime: '600' + Databases: + JobDB: + DBName: JobDB + Host: mysql + Port: '3306' + JobLoggingDB: + DBName: JobLoggingDB + Host: mysql + Port: '3306' + PilotAgentsDB: + DBName: PilotAgentsDB + Host: mysql + Port: '3306' + SandboxMetadataDB: + DBName: SandboxMetadataDB + Host: mysql + Port: '3306' + TaskQueueDB: + DBName: TaskQueueDB + Host: mysql + Port: '3306' + Executors: + Optimizers: + JobScheduling: + RescheduleDelays: '0' + FailoverURLs: {} + Services: + Matcher: + Authorization: + Default: authenticated + getActiveTaskQueues: JobAdministrator + MaxThreads: '20' + Port: '9170' + OptimizationMind: + Port: '9175' + PilotManager: + Authorization: + Default: authenticated + Port: '9171' + SandboxStore: + BasePath: dirac-JenkinsSetup/sandboxes + LogLevel: DEBUG + TornadoJobManager: + Authorization: + Default: authenticated + Protocol: https + TornadoJobMonitoring: + Authorization: + Default: authenticated + Protocol: https + TornadoJobStateUpdate: + Authorization: + Default: authenticated + Protocol: https + TornadoPilotLogging: + Authorization: + Default: authenticated + finaliseLogs: Operator, Pilot, GenericPilot + getMetadata: Operator, TrustedHost + sendMessage: Operator, GenericPilot + Protocol: https + TornadoSandboxStore: + Authorization: + Default: authenticated + FileTransfer: + Default: authenticated + Backend: local + BasePath: dirac-JenkinsSetup/sandboxes + DelayedExternalDeletion: 'True' + LocalSE: ProductionSandboxSE + LogLevel: DEBUG + MaxSandboxSizeMiB: '10' + MaxThreads: '200' + Protocol: https + SandboxPrefix: Sandbox + toClientMaxThreads: '100' + TornadoWMSAdministrator: + Authorization: + Default: Operator + allowSite: SiteManager, Operator + banSite: SiteManager, Operator + getJobPilotOutput: authenticated + getSiteMask: authenticated + getSiteMaskStatus: authenticated + ping: authenticated + Protocol: https + URLs: + JobManager: https://server:8443/WorkloadManagement/TornadoJobManager + JobMonitoring: https://server:8443/WorkloadManagement/TornadoJobMonitoring + JobStateUpdate: https://server:8443/WorkloadManagement/TornadoJobStateUpdate + Matcher: dips://server:9170/WorkloadManagement/Matcher + OptimizationMind: dips://server:9175/WorkloadManagement/OptimizationMind + PilotLogging: https://server:8443/WorkloadManagement/TornadoPilotLogging + PilotManager: dips://server:9171/WorkloadManagement/PilotManager + SandboxStore: https://server:8443/WorkloadManagement/TornadoSandboxStore + WMSAdministrator: https://server:8443/WorkloadManagement/TornadoWMSAdministrator +WebApp: + Access: + upload: TrustedHost