forked from DIRACGrid/DIRAC
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dirac.cfg
368 lines (358 loc) · 13.1 KB
/
dirac.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
Systems
{
# the systems section is automatically obtained from the ConfigTemplate.cfg files and can be found at
# https://dirac.readthedocs.org/en/latest/AdministratorGuide/Configuration/ExampleConfig.html
DataManagementSystem
{
Agents
{
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts3.html#fts3agent
FTS3Agent
{
OperationBulkSize = 20 # How many Operation we will treat in one loop
JobBulkSize = 20 # How many Job we will monitor in one loop
MaxFilesPerJob = 100 # Max number of files to go in a single job
MaxAttemptsPerFile = 256 # Max number of attempt per file
DeleteGraceDays = 180 # days before removing jobs
DeleteLimitPerCycle = 100 # Max number of deletes per cycle
KickAssignedHours = 1 # hours before kicking jobs with old assignment tag
KickLimitPerCycle = 100 # Max number of kicks per cycle
}
}
Services
{
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/dfc.html#filecataloghandler
FileCatalogHandler
{
Port = 9197
DatasetManager = DatasetManager
DefaultUmask = 0775
DirectoryManager = DirectoryLevelTree
DirectoryMetadata = DirectoryMetadata
FileManager = FileManager
FileMetadata = FileMetadata
GlobalReadAccess = True
LFNPFNConvention = Strong
ResolvePFN = True
SecurityManager = NoSecurityManager
SEManager = SEManagerDB
UniqueGUID = False
UserGroupManager = UserAndGroupManagerDB
ValidFileStatus = [AprioriGoodTrashRemovingProbing]
ValidReplicaStatus = [AprioriGoodTrashRemovingProbing]
VisibleFileStatus = [AprioriGood]
VisibleReplicaStatus = [AprioriGood]
}
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html#ftsmanager
FTS3ManagerHandler
{
# No specific configuration
Port = 9193
}
}
Databases
{
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/dfc.html#filecatalogdb
FileCatalogDB
{
# No specific configuration
DBName = FileCatalogDB
}
FTS3DB
{
# No specific configuration
DBName = FTS3DB
}
}
}
RequestManagementSystem
{
Agents
{
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#cleanreqdbagent
CleanReqDBAgent
{
DeleteGraceDays = 60 # Delay after which Requests are removed
DeleteLimit = 100 # Maximum number of Requests to remove per cycle
DeleteFailed = False # Whether to delete also Failed request
KickGraceHours = 1 # After how long we should kick the Requests in `Assigned`
KickLimit = 10000 # Maximum number of requests kicked by cycle
}
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#requestexecutingagent
RequestExecutingAgent
{
BulkRequest = 0
MinProcess = 1
MaxProcess = 8
ProcessPoolQueueSize = 25
ProcessPoolTimeout = 900
ProcessTaskTimeout = 900
ProcessPoolSleep = 4
RequestsPerCycle = 50
# Define the different Operation types
# see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsObjects.html#operation-types
OperationHandlers
{
DummyOperation
{
# These parameters can be defined for all handlers
# The location of the python file, without .py, is mandatory
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/DummyHandler # Mandatory
LogLevel = DEBUG # self explanatory
MaxAttemts = 256 # Maximum attempts per file
TimeOut = 300 # Timeout in seconds of the operation
TimeOutPerFile = 40 # Additional delay per file
}
ForwardDISET{
Location = DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET
}
MoveReplica
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica
}
PutAndRegister
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister
}
RegisterFile
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile
}
RegisterReplica
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica
}
RemoveFile
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile
}
RemoveReplica
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica
}
ReplicateAndRegister
{
Location = DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister
FTSMode = True # If True, will use FTS to transfer files
FTSBannedGroups = lhcb_user # list of groups for which not to use FTS
}
SetFileStatus
{
Location = DIRAC/TransformationSystem/Agent/RequestOperations/SetFileStatus
}
}
}
}
Databases
{
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#requestdb
RequestDB
{
# No specific configuration
DBName = RequestDB
}
}
Services
{
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#reqmanager
ReqManager
{
Port = 9140
constantRequestDelay = 0 # Constant delay when retrying a request
}
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#reqproxy
ReqProxy
{
Port = 9161
}
}
URLs
{
# Yes.... it is ReqProxyURLs, and not ReqProxy...
# http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#reqproxy
ReqProxyURLs = dips://server1:9161/RequestManagement/ReqProxy, dips://server2:9161/RequestManagement/ReqProxy
}
}
TransformationSystem
{
Agents
{
##BEGIN TransformationCleaningAgent
TransformationCleaningAgent
{
# MetaData key to use to identify output data
TransfIDMeta=TransformationID
# Location of the OutputData, if the OutputDirectories parameter is not set for
# transformations only 'MetadataCatalog has to be used
DirectoryLocations=TransformationDB,MetadataCatalog
# Enable or disable, default enabled
EnableFlag=True
# How many days to wait before archiving transformations
ArchiveAfter=7
# Shifter to use for removal operations, default is empty and
# using the transformation owner for cleanup
shifterProxy=
# Which transformation types to clean
# If not filled, transformation types are taken from
# Operations/Transformations/DataManipulation
# and Operations/Transformations/DataProcessing
TransformationTypes=
#Time between cycles in seconds
PollingTime = 3600
}
##END
}
}
}
Resources
{
#Where all your Catalogs are defined
FileCatalogs
{
#There is one section per catalog
#See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html
<MyCatalog>
{
CatalogType = <myCatalogType> # used for plugin selection
CatalogURL = <myCatalogURL> # used for DISET URL
}
}
#FTS endpoint definition http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.htmlfts-servers-definition
<anyOptions> # Passed to the constructor of the pluginFTSEndpoints
{
FTS3
{
CERN-FTS3 = https://fts3.cern.ch:8446
}
}
#Abstract definition of storage elements, used to be inherited.
#see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelementbases
StorageElementBases
{
#The base SE definition can contain all the options of a normal SE
#http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelements
CERN-EOS
{
BackendType = eos # backend type of storage element
SEType = T0D1 # Tape or Disk SE
UseCatalogURL = True # used the stored url or generate it (default False)
ReadAccess = True # Allowed for Read if no RSS enabled
WriteAccess = True # Allowed for Write if no RSS enabled
CheckAccess = True # Allowed for Check if no RSS enabled
RemoveAccess = True # Allowed for Remove if no RSS enabled
#Protocol section, see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlavailable-protocol-plugins
GFAL2_SRM2
{
Host = srm-eoslhcb.cern.ch
Port = 8443
PluginName = GFAL2_SRM2 # If different from the section name
Protocol = srm # primary protocol
Path = /eos/lhcb/grid/prod # base path
Access = remote
SpaceToken = LHCb-EOS
WSUrl = /srm/v2/server?SFN=
}
}
}
#http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelements
StorageElements
{
#Just inherit everything from CERN-EOS, without change
CERN-DST-EOS
{
BaseSE = CERN-EOS
}
CERN-USER # inherit from CERN-EOS
{
BaseSE = CERN-EOS
#Modify the options for Gfal2
GFAL2_SRM2
{
Path = /eos/lhcb/grid/user
SpaceToken = LHCb_USER
}
#Add an extra protocol
GFAL2_XROOT
{
Host = eoslhcb.cern.ch
Port = 8443
Protocol = root
Path = /eos/lhcb/grid/user
Access = remote
SpaceToken = LHCb-EOS
WSUrl = /srm/v2/server?SFN=
}
}
CERN-ALIAS
{
Alias = CERN-USER # Use CERN-USER when instanciating CERN-ALIAS
}
}
#See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelementgroups
StorageElementGroups
{
CERN-Storages = CERN-DST-EOS
CERN-Storages += CERN-USER
}
}
Operations
{
#This is the default section of operations.
#Any value here can be overwriten in the setup specific section
Defaults
{
DataManagement
{
#see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.htmlmulti-protocol
#for the next 4 options
AccessProtocols = srm
AccessProtocols += dips
RegistrationProtocols = srm
RegistrationProtocols += dips
ThirdPartyProtocols = srm
WriteProtocols = srm
WriteProtocols += dips
#FTS related options. See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html
FTSVersion = FTS3 # should only be that...
FTSPlacement
{
FTS3
{
ServerPolicy = Random # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html#ftsserver-policy
}
}
}
Services
{
#See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html
Catalogs
{
CatalogList = Catalog1
CatalogList += Catalog2
CatalogList += etc # List of catalogs defined in Resources to use
#Each catalog defined in Resources should also contain some runtime options here
<MyCatalog>
{
Status = Active # enable the catalog or not (default Active)
AccessType = Read-Write # No default
AccessType += must be set
Master = True # See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html#master-catalog
#Dynamic conditions to enable or not the catalog
#See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.htmlconditional-filecatalogs
Conditions
{
WRITE = <myWriteCondition>
READ = <myReadCondition>
ALL = <valid for all conditions>
<myMethod> = <myCondition valid only for myMethod>
}
}
}
}
}
#Options in this section will only be used when running with the
#<MySetup> setup
<MySetup>
{
}
}