From 620592630e9cd7289f5689d88b1d05a2b46520c9 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 16 Feb 2024 20:51:07 +0100 Subject: [PATCH 1/6] Add cmsDriver option --maxmem_profile to prefix LD_PRELOAD="libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so" to cmsRun --- Configuration/Applications/python/Options.py | 6 +++++ .../Applications/python/cmsDriverOptions.py | 26 ++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 5abc482d9d7bb..7eb853c75dd1b 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -428,6 +428,12 @@ type=str, dest="heap_profile") +expertSettings.add_argument("--maxmem_profile", + help="add the PerfTools/MaxMemoryPreload Service", + default=None, + type=str, + dest="maxmem_profile") + expertSettings.add_argument("--io", help="Create a json file with io informations", default=None, diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index eaf702762de64..45c9cce19e6c7 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -17,7 +17,7 @@ def checkModifier(era): def checkOptions(): return - + def adaptOptions(): return @@ -34,7 +34,7 @@ def OptionsFromCommand(command): def OptionsFromCommandLine(): import sys options=OptionsFromItems(sys.argv[1:]) - # memorize the command line arguments + # memorize the command line arguments options.arguments = reduce(lambda x, y: x+' '+y, sys.argv[1:]) return options @@ -60,8 +60,8 @@ def OptionsFromItems(items): if options.triggerResultsProcess == None and "ALCAOUTPUT" in options.step: print("ERROR: If ALCA splitting is requested, the name of the process in which the alca producers ran needs to be specified. E.g. via --triggerResultsProcess RECO") sys.exit(1) - - if not options.evt_type: + + if not options.evt_type: options.evt_type=sys.argv[1] #now adjust the given parameters before passing it to the ConfigBuilder @@ -123,7 +123,7 @@ def OptionsFromItems(items): # add on the end of job sequence... addEndJob = True - if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "": + if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "": addEndJob = False if ("SKIM" in options.step and not "RECO" in options.step): addEndJob = False @@ -131,7 +131,7 @@ def OptionsFromItems(items): addEndJob = False if ('DQMIO' in options.datatier): addEndJob = False - if addEndJob: + if addEndJob: options.step=options.step+',ENDJOB' @@ -180,7 +180,7 @@ def OptionsFromItems(items): options.name = 'RESIM' elif 'reDIGI' in options.trimmedStep: options.name = 'REDIGI' - elif 'HLT' in options.trimmedStep: + elif 'HLT' in options.trimmedStep: options.name = 'HLT' elif 'RECO' in options.trimmedStep: options.name = 'RECO' @@ -244,13 +244,19 @@ def OptionsFromItems(items): raise Exception("Not a valid profiler type %s. Alternatives are pp, mp, fp=."%(profilerType)) options.prefix = "igprof -t cmsRun -%s" % profilerType - + if options.heap_profile: - if options.profile and options.prefix: + if options.heap_profile and options.prefix: raise Exception("--heap_profile and --prefix are incompatible") profilerType = 'pp' options.prefix = "MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out cmsRunJE " + if options.maxmem_profile: + if options.maxmem_profile and and options.prefix: + raise Exception("--maxmem_profile and --prefix are incompatible") + options.prefix = 'LD_PRELOAD="libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so" cmsRun ' + + # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era : from Configuration.StandardSequences.Eras import eras @@ -262,7 +268,7 @@ def OptionsFromItems(items): validOptions="" # Create a stringified list of valid options to print to the user for key in eras.__dict__ : if checkModifier(eras.__dict__[key]): - if validOptions!="" : validOptions+=", " + if validOptions!="" : validOptions+=", " validOptions+="'"+key+"'" raise Exception( "'%s' is not a valid option for '--era'. Valid options are %s." % (eraName, validOptions) ) # If the "--fast" option was supplied automatically enable the fastSim era From f7b13e86a34f7d4af8019bc7c4f3d55b1689aa80 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 16 Feb 2024 21:46:18 +0100 Subject: [PATCH 2/6] Remove redundant checkout on options.maxmem_profile --- Configuration/Applications/python/cmsDriverOptions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index 45c9cce19e6c7..29caaa0f63d0e 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -246,16 +246,15 @@ def OptionsFromItems(items): options.prefix = "igprof -t cmsRun -%s" % profilerType if options.heap_profile: - if options.heap_profile and options.prefix: + if options.prefix: raise Exception("--heap_profile and --prefix are incompatible") profilerType = 'pp' options.prefix = "MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out cmsRunJE " if options.maxmem_profile: - if options.maxmem_profile and and options.prefix: + if options.prefix: raise Exception("--maxmem_profile and --prefix are incompatible") - options.prefix = 'LD_PRELOAD="libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so" cmsRun ' - + options.prefix = "LD_PRELOAD=\"libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so\" cmsRun " # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era : From a6f32eabebd46f914efac8edfc599c005c18cc22 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 16 Feb 2024 21:47:19 +0100 Subject: [PATCH 3/6] Change Service to monitor --- Configuration/Applications/python/Options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 7eb853c75dd1b..6551894864983 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -429,7 +429,7 @@ dest="heap_profile") expertSettings.add_argument("--maxmem_profile", - help="add the PerfTools/MaxMemoryPreload Service", + help="add the PerfTools/MaxMemoryPreload monitor", default=None, type=str, dest="maxmem_profile") From 2fc876ba51cb1c099e3d57ae0d94520598f40460 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 16 Feb 2024 23:49:11 +0100 Subject: [PATCH 4/6] Add env command --- .../Applications/python/ConfigBuilder.py | 31 +++---------------- Configuration/Applications/python/Options.py | 8 ++--- .../Applications/python/cmsDriverOptions.py | 9 +++--- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index bf1d7f5089ed4..baa0c4b5578f2 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -72,7 +72,8 @@ class Options: defaultOptions.dropDescendant = False defaultOptions.relval = None defaultOptions.profile = None -defaultOptions.heap_profile = None +defaultOptions.heap_profile = False +defaultOptions.maxmem_profile = False defaultOptions.isRepacked = False defaultOptions.restoreRNDSeeds = False defaultOptions.donotDropOnInput = '' @@ -324,37 +325,13 @@ def heapProfileOptions(self): Function to add the jemalloc heap profile service so that you can dump in the middle of the run. """ - profileOpts = self._options.profile.split(':') + profileOpts = [] profilerStart = 1 profilerInterval = 100 - profilerFormat = None + profilerFormat = "jeprof_%s.heap" profilerJobFormat = None - if len(profileOpts): - #type, given as first argument is unused here - profileOpts.pop(0) - if len(profileOpts): - startEvent = profileOpts.pop(0) - if not startEvent.isdigit(): - raise Exception("%s is not a number" % startEvent) - profilerStart = int(startEvent) - if len(profileOpts): - eventInterval = profileOpts.pop(0) - if not eventInterval.isdigit(): - raise Exception("%s is not a number" % eventInterval) - profilerInterval = int(eventInterval) - if len(profileOpts): - profilerFormat = profileOpts.pop(0) - - if not profilerFormat: - profilerFormat = "%s___%s___%%I.heap" % ( - self._options.evt_type.replace("_cfi", ""), - hashlib.md5( - (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) + - str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8') - ).hexdigest() - ) if not profilerJobFormat and profilerFormat.endswith(".heap"): profilerJobFormat = profilerFormat.replace(".heap", "_EndOfJob.heap") elif not profilerJobFormat: diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 6551894864983..9f7e48151eef4 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -424,14 +424,14 @@ expertSettings.add_argument("--heap_profile", help="add the JeProfService with the parameter provided PROFILER:START:STEP:PEREVENOUTPUTFORMAT:ENDOFJOBOUTPUTFORMAT", - default=None, - type=str, + default="False", + action="store_true", dest="heap_profile") expertSettings.add_argument("--maxmem_profile", help="add the PerfTools/MaxMemoryPreload monitor", - default=None, - type=str, + default=False, + action="store_true", dest="maxmem_profile") expertSettings.add_argument("--io", diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index 29caaa0f63d0e..9541016931741 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -246,15 +246,14 @@ def OptionsFromItems(items): options.prefix = "igprof -t cmsRun -%s" % profilerType if options.heap_profile: - if options.prefix: + if options.heap_profile and options.prefix: raise Exception("--heap_profile and --prefix are incompatible") - profilerType = 'pp' - options.prefix = "MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out cmsRunJE " + options.prefix = "env MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out LD_PRELOAD=libjemalloc-prof.so " if options.maxmem_profile: - if options.prefix: + if options.maxmem_profile and options.prefix: raise Exception("--maxmem_profile and --prefix are incompatible") - options.prefix = "LD_PRELOAD=\"libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so\" cmsRun " + options.prefix = 'env LD_PRELOAD="libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so" ' # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era : From daeaf5aa3145fcab0e2ba5a9d90a9584ee750b43 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Sat, 17 Feb 2024 00:19:11 +0100 Subject: [PATCH 5/6] Set correct default for heap_profile --- Configuration/Applications/python/ConfigBuilder.py | 5 +++-- Configuration/Applications/python/Options.py | 2 +- Configuration/Applications/python/cmsDriverOptions.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index baa0c4b5578f2..60e1de3876af3 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -71,9 +71,10 @@ class Options: defaultOptions.inputEventContent = '' defaultOptions.dropDescendant = False defaultOptions.relval = None +defaultOptions.prefix = None defaultOptions.profile = None -defaultOptions.heap_profile = False -defaultOptions.maxmem_profile = False +defaultOptions.heap_profile = None +defaultOptions.maxmem_profile = None defaultOptions.isRepacked = False defaultOptions.restoreRNDSeeds = False defaultOptions.donotDropOnInput = '' diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 9f7e48151eef4..2ba1f0240ced9 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -424,7 +424,7 @@ expertSettings.add_argument("--heap_profile", help="add the JeProfService with the parameter provided PROFILER:START:STEP:PEREVENOUTPUTFORMAT:ENDOFJOBOUTPUTFORMAT", - default="False", + default=False, action="store_true", dest="heap_profile") diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index 9541016931741..f1cf41057d890 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -246,14 +246,14 @@ def OptionsFromItems(items): options.prefix = "igprof -t cmsRun -%s" % profilerType if options.heap_profile: - if options.heap_profile and options.prefix: + if options.prefix: raise Exception("--heap_profile and --prefix are incompatible") options.prefix = "env MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out LD_PRELOAD=libjemalloc-prof.so " if options.maxmem_profile: - if options.maxmem_profile and options.prefix: + if options.prefix: raise Exception("--maxmem_profile and --prefix are incompatible") - options.prefix = 'env LD_PRELOAD="libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so" ' + options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so env LD_PRELOAD=libPerfToolsMaxMemoryPreload.so " # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era : From 8a8e495eecf6c5cad32b36d2011b7e182f626617 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Sat, 17 Feb 2024 00:26:31 +0100 Subject: [PATCH 6/6] Combine LD_PRELOAD libraries with : --- Configuration/Applications/python/cmsDriverOptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index f1cf41057d890..412ef9f036707 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -253,7 +253,7 @@ def OptionsFromItems(items): if options.maxmem_profile: if options.prefix: raise Exception("--maxmem_profile and --prefix are incompatible") - options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so env LD_PRELOAD=libPerfToolsMaxMemoryPreload.so " + options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so " # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era :