From 282a6d89d8e3fd845739c40a66ee36f5d8d9605e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 14 Nov 2024 22:21:25 +0100 Subject: [PATCH 1/8] Add additional runtime information through the eessi_mixin class --- eessi/testsuite/eessi_mixin.py | 53 ++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 53508117..1bbe027a 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -1,12 +1,13 @@ from reframe.core.builtins import parameter, run_after, variable -from reframe.core.exceptions import ReframeFatalError +from reframe.core.exceptions import ReframeFatalError, SanityError from reframe.core.pipeline import RegressionMixin from reframe.utility.sanity import make_performance_function +import reframe.utility.sanity as sn from eessi.testsuite import hooks from eessi.testsuite.constants import DEVICE_TYPES, SCALES, COMPUTE_UNIT, TAGS from eessi.testsuite.utils import log - +from eessi.testsuite import __version__ as testsuite_version # Hooks from the Mixin class seem to be executed _before_ those of the child class # Thus, if the Mixin class needs self.X to be defined in after setup, the child class would have to define it before @@ -42,6 +43,14 @@ class EESSI_Mixin(RegressionMixin): bench_name = None bench_name_ci = None + # Create ReFrame variables for logging runtime environment information + cvmfs_repo_name = variable(str, value='None') + cvmfs_software_subdir = variable(str, value='None') + full_modulepath = variable(str, value='None') + + # Make sure the version of the EESSI test suite gets logged in the ReFrame report + eessi_testsuite_version = variable(str, value=testsuite_version) + # Note that the error for an empty parameter is a bit unclear for ReFrame 4.6.2, but that will hopefully improve # see https://github.com/reframe-hpc/reframe/issues/3254 # If that improves: uncomment the following to force the user to set module_name @@ -165,3 +174,43 @@ def assign_tasks_per_compute_unit(self): def request_mem(self): """Call hook to request the required amount of memory per node""" hooks.req_memory_per_node(self, app_mem_req=self.required_mem_per_node()) + + @run_after('setup') + def log_runtime_info(self): + """Log additional runtime information: which CVMFS repo was used (or if it was testing local software), + path to the modulefile, EESSI software subdir, EESSI testsuite version""" + self.postrun_cmds.append('echo "EESSI_CVMFS_REPO: $EESSI_CVMFS_REPO"') + self.postrun_cmds.append('echo "EESSI_SOFTWARE_SUBDIR: $EESSI_SOFTWARE_SUBDIR"') + if self.module_name: + # Get full modulepath, stripping leading spaces with sed and trailing colon by string substitution + get_full_modpath = f'modpath=$(module show {self.module_name} 2>&1 | ' + get_full_modpath += 'grep ".lua" | ' + get_full_modpath += 'sed "s/^[[:space:]]*//") && echo "FULL_MODULEPATH: ${modpath%:*}"' + self.postrun_cmds.append(get_full_modpath) + + @run_after('run') + def extract_runtime_info_from_log(self): + """Extracts the printed runtime info from the job log and logs it as reframe variables""" + # If EESSI_CVMFS_REPO environment variable was set, extract it and store it in self.cvmfs_repo_name + # Try block is needed to surpress sanity error if there is no match + try: + repo_name = sn.extractsingle(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', 'repo', str) + if repo_name: + self.cvmfs_repo_name = f'{repo_name}' + except SanityError: + pass + + try: + software_subdir = sn.extractsingle(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', f'{self.stagedir}/{self.stdout}', 'subdir', str) + if software_subdir: + self.cvmfs_software_subdir = f'{software_subdir}' + except SanityError: + pass + + try: + module_path = sn.extractsingle(r'FULL_MODULEPATH: (?P.*)$', f'{self.stagedir}/{self.stdout}', 'modpath', str) + print(f"Full modulepath: {module_path}") + if module_path: + self.full_modulepath = f'{module_path}' + except SanityError: + pass From d9af48056a993a3e2988efcb02fd387a0f3729af Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 14 Nov 2024 22:26:23 +0100 Subject: [PATCH 2/8] fix pep8 stuff --- eessi/testsuite/eessi_mixin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 1bbe027a..d375f2c1 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -9,6 +9,7 @@ from eessi.testsuite.utils import log from eessi.testsuite import __version__ as testsuite_version + # Hooks from the Mixin class seem to be executed _before_ those of the child class # Thus, if the Mixin class needs self.X to be defined in after setup, the child class would have to define it before # setup. That's a disadvantage and might not always be possible - let's see how far we get. It also seems that, @@ -194,7 +195,8 @@ def extract_runtime_info_from_log(self): # If EESSI_CVMFS_REPO environment variable was set, extract it and store it in self.cvmfs_repo_name # Try block is needed to surpress sanity error if there is no match try: - repo_name = sn.extractsingle(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', 'repo', str) + repo_name = sn.extractsingle(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', + 'repo', str) if repo_name: self.cvmfs_repo_name = f'{repo_name}' except SanityError: From 4dc634d79d22ccb56e8f64132b77578fd577a842 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 14 Nov 2024 22:28:27 +0100 Subject: [PATCH 3/8] fix pep8 stuff --- eessi/testsuite/eessi_mixin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index d375f2c1..6324f1ff 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -200,17 +200,19 @@ def extract_runtime_info_from_log(self): if repo_name: self.cvmfs_repo_name = f'{repo_name}' except SanityError: - pass + pass try: - software_subdir = sn.extractsingle(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', f'{self.stagedir}/{self.stdout}', 'subdir', str) + software_subdir = sn.extractsingle(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', f'{self.stagedir}/{self.stdout}', + 'subdir', str) if software_subdir: self.cvmfs_software_subdir = f'{software_subdir}' except SanityError: pass try: - module_path = sn.extractsingle(r'FULL_MODULEPATH: (?P.*)$', f'{self.stagedir}/{self.stdout}', 'modpath', str) + module_path = sn.extractsingle(r'FULL_MODULEPATH: (?P.*)$', f'{self.stagedir}/{self.stdout}', + 'modpath', str) print(f"Full modulepath: {module_path}") if module_path: self.full_modulepath = f'{module_path}' From 6659e912caac9991426b3e4b83f01454481a077b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 14 Nov 2024 22:33:57 +0100 Subject: [PATCH 4/8] fix pep8 stuff --- eessi/testsuite/eessi_mixin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 6324f1ff..0ea55200 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -203,8 +203,8 @@ def extract_runtime_info_from_log(self): pass try: - software_subdir = sn.extractsingle(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', f'{self.stagedir}/{self.stdout}', - 'subdir', str) + software_subdir = sn.extractsingle(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', + f'{self.stagedir}/{self.stdout}', 'subdir', str) if software_subdir: self.cvmfs_software_subdir = f'{software_subdir}' except SanityError: From 808b5aa97394839d18d84d62c67f73057c70f0f7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 28 Nov 2024 13:05:49 +0100 Subject: [PATCH 5/8] Apply suggestions from code review Co-authored-by: Sam Moors --- eessi/testsuite/eessi_mixin.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 0ea55200..4d97388d 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -183,17 +183,15 @@ def log_runtime_info(self): self.postrun_cmds.append('echo "EESSI_CVMFS_REPO: $EESSI_CVMFS_REPO"') self.postrun_cmds.append('echo "EESSI_SOFTWARE_SUBDIR: $EESSI_SOFTWARE_SUBDIR"') if self.module_name: - # Get full modulepath, stripping leading spaces with sed and trailing colon by string substitution - get_full_modpath = f'modpath=$(module show {self.module_name} 2>&1 | ' - get_full_modpath += 'grep ".lua" | ' - get_full_modpath += 'sed "s/^[[:space:]]*//") && echo "FULL_MODULEPATH: ${modpath%:*}"' + # Get full modulepath + get_full_modpath = f'echo "FULL_MODULEPATH: $(module --location show {self.module_name})"' self.postrun_cmds.append(get_full_modpath) @run_after('run') def extract_runtime_info_from_log(self): """Extracts the printed runtime info from the job log and logs it as reframe variables""" # If EESSI_CVMFS_REPO environment variable was set, extract it and store it in self.cvmfs_repo_name - # Try block is needed to surpress sanity error if there is no match + # Try block is needed to suppress sanity error if there is no match try: repo_name = sn.extractsingle(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', 'repo', str) From a401c72fd39c1aaac7c4a29c7f04dd6ffd922aad Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 28 Nov 2024 13:09:01 +0100 Subject: [PATCH 6/8] Use extractall so that no error is raised if the pattern is not found. This way, we don't need the try block. --- eessi/testsuite/eessi_mixin.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 4d97388d..3226ac3f 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -191,28 +191,17 @@ def log_runtime_info(self): def extract_runtime_info_from_log(self): """Extracts the printed runtime info from the job log and logs it as reframe variables""" # If EESSI_CVMFS_REPO environment variable was set, extract it and store it in self.cvmfs_repo_name - # Try block is needed to suppress sanity error if there is no match - try: - repo_name = sn.extractsingle(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', - 'repo', str) - if repo_name: - self.cvmfs_repo_name = f'{repo_name}' - except SanityError: - pass - - try: - software_subdir = sn.extractsingle(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', + repo_name = sn.extractall(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', + 'repo', str) + if repo_name: + self.cvmfs_repo_name = f'{repo_name}' + + software_subdir = sn.extractall(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', f'{self.stagedir}/{self.stdout}', 'subdir', str) - if software_subdir: - self.cvmfs_software_subdir = f'{software_subdir}' - except SanityError: - pass + if software_subdir: + self.cvmfs_software_subdir = f'{software_subdir}' - try: - module_path = sn.extractsingle(r'FULL_MODULEPATH: (?P.*)$', f'{self.stagedir}/{self.stdout}', + module_path = sn.extractall(r'FULL_MODULEPATH: (?P.*)$', f'{self.stagedir}/{self.stdout}', 'modpath', str) - print(f"Full modulepath: {module_path}") - if module_path: - self.full_modulepath = f'{module_path}' - except SanityError: - pass + if module_path: + self.full_modulepath = f'{module_path}' From 5bf9dab8e8272b11a69761785e2c08909a8f519e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 28 Nov 2024 13:12:32 +0100 Subject: [PATCH 7/8] Fix flake8 issues --- eessi/testsuite/eessi_mixin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 3226ac3f..5d36ace6 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -1,5 +1,5 @@ from reframe.core.builtins import parameter, run_after, variable -from reframe.core.exceptions import ReframeFatalError, SanityError +from reframe.core.exceptions import ReframeFatalError from reframe.core.pipeline import RegressionMixin from reframe.utility.sanity import make_performance_function import reframe.utility.sanity as sn @@ -197,11 +197,11 @@ def extract_runtime_info_from_log(self): self.cvmfs_repo_name = f'{repo_name}' software_subdir = sn.extractall(r'EESSI_SOFTWARE_SUBDIR: (?P.*)$', - f'{self.stagedir}/{self.stdout}', 'subdir', str) + f'{self.stagedir}/{self.stdout}', 'subdir', str) if software_subdir: self.cvmfs_software_subdir = f'{software_subdir}' module_path = sn.extractall(r'FULL_MODULEPATH: (?P.*)$', f'{self.stagedir}/{self.stdout}', - 'modpath', str) + 'modpath', str) if module_path: self.full_modulepath = f'{module_path}' From ebce23abbba8f3d74ef8c57e9e1a082e58b6ddf3 Mon Sep 17 00:00:00 2001 From: Sam Moors Date: Sat, 30 Nov 2024 14:10:11 +0100 Subject: [PATCH 8/8] don't extract runtime info when doing a dry-run --- eessi/testsuite/eessi_mixin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 5d36ace6..d70d3015 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -190,6 +190,9 @@ def log_runtime_info(self): @run_after('run') def extract_runtime_info_from_log(self): """Extracts the printed runtime info from the job log and logs it as reframe variables""" + if self.is_dry_run(): + return + # If EESSI_CVMFS_REPO environment variable was set, extract it and store it in self.cvmfs_repo_name repo_name = sn.extractall(r'EESSI_CVMFS_REPO: /cvmfs/(?P.*)$', f'{self.stagedir}/{self.stdout}', 'repo', str)