[feat] Introduce permanent result storage and support performance results comparisons between runs

reframe/core/exceptions.py

@@ -388,3 +388,17 @@ def what(exc_type, exc_value, tb):
             reason += f': {exc_value}'
 
     return reason
+
+
+class reraise_as:
+    def __init__(self, new_exc, exceptions=(Exception,), message=''):
+        self.__new_exc = new_exc
+        self.__exceptions = exceptions
+        self.__message = message
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if isinstance(exc_val, self.__exceptions):
+            raise self.__new_exc(self.__message) from exc_val

reframe/core/logging.py

@@ -433,7 +433,7 @@ def _create_file_handler(site_config, config_prefix):
 
 def _create_filelog_handler(site_config, config_prefix):
     basedir = os.path.abspath(os.path.join(
-        site_config.get(f'systems/0/prefix'),
+        site_config.get('systems/0/prefix'),
         osext.expandvars(site_config.get(f'{config_prefix}/basedir'))
     ))
     prefix  = osext.expandvars(site_config.get(f'{config_prefix}/prefix'))
@@ -581,7 +581,7 @@ def _create_httpjson_handler(site_config, config_prefix):
 
 def _record_to_json(record, extras, ignore_keys):
     def _can_send(key):
-        return not key.startswith('_') and not key in ignore_keys
+        return not key.startswith('_') and key not in ignore_keys
 
     def _sanitize(s):
         return re.sub(r'\W', '_', s)
@@ -860,9 +860,10 @@ def log_performance(self, level, task, msg=None, multiline=False):
         if self.check is None or not self.check.is_performance_check():
             return
 
-        self.extra['check_partition'] = task.testcase.partition.name
-        self.extra['check_environ'] = task.testcase.environ.name
-        self.extra['check_result'] = 'pass' if task.succeeded else 'fail'
+        _, part, env = task.testcase
+        self.extra['check_partition'] = part.name
+        self.extra['check_environ'] = env.name
+        self.extra['check_result'] = task.result
         fail_reason = what(*task.exc_info) if not task.succeeded else None
         self.extra['check_fail_reason'] = fail_reason
         if msg is None:

reframe/core/pipeline.py

@@ -46,7 +46,8 @@
 
 
 class _NoRuntime(ContainerPlatform):
-    '''Proxy container runtime for storing container platform info early enough.
+    '''Proxy container runtime for storing container platform info early
+    enough.
 
     This will be replaced by the framework with a concrete implementation
     based on the current partition info.
@@ -847,8 +848,8 @@ def pipeline_hooks(cls):
     #: .. deprecated:: 4.0.0
     #:    Please use :attr:`env_vars` instead.
     variables = deprecate(variable(alias=env_vars),
-                          f"the use of 'variables' is deprecated; "
-                          f"please use 'env_vars' instead")
+                          "the use of 'variables' is deprecated; "
+                          "please use 'env_vars' instead")
 
     #: Time limit for this test.
     #:
@@ -1517,8 +1518,7 @@ def _job_exitcode(self):
     @loggable_as('job_nodelist')
     @property
     def _job_nodelist(self):
-        if self.job:
-            return self.job.nodelist
+        return self.job.nodelist if self.job else []
 
     def info(self):
         '''Provide live information for this test.
@@ -1705,7 +1705,7 @@ def _setup_build_job(self, **job_opts):
         )
 
     def _setup_run_job(self, **job_opts):
-        self._job = self._create_job(f'run', self.local, **job_opts)
+        self._job = self._create_job('run', self.local, **job_opts)
 
     def _setup_container_platform(self):
         try:
@@ -2217,7 +2217,7 @@ def check_performance(self):
 
         if perf_patterns is not None and self.perf_variables:
             raise ReframeSyntaxError(
-                f"you cannot mix 'perf_patterns' and 'perf_variables' syntax"
+                "you cannot mix 'perf_patterns' and 'perf_variables' syntax"
             )
 
         # Convert `perf_patterns` to `perf_variables`
@@ -2365,7 +2365,7 @@ def cleanup(self, remove_files=False):
         aliased = os.path.samefile(self._stagedir, self._outputdir)
         if aliased:
             self.logger.debug(
-                f'outputdir and stagedir are the same; copying skipped'
+                'outputdir and stagedir are the same; copying skipped'
             )
         else:
             self._copy_to_outputdir()

reframe/core/runtime.py

@@ -27,11 +27,12 @@ class RuntimeContext:
     .. versionadded:: 2.13
     '''
 
-    def __init__(self, site_config):
+    def __init__(self, site_config, *, use_timestamps=False):
         self._site_config = site_config
         self._system = System.create(site_config)
         self._current_run = 0
         self._timestamp = time.localtime()
+        self._use_timestamps = use_timestamps
 
     def _makedir(self, *dirs, wipeout=False):
         ret = os.path.join(*dirs)
@@ -110,7 +111,11 @@ def perflogdir(self):
 
     @property
     def timestamp(self):
-        timefmt = self.site_config.get('general/0/timestamp_dirs')
+        if self._use_timestamps:
+            timefmt = self.site_config.get('general/0/timestamp_dirs')
+        else:
+            timefmt = ''
+
         return time.strftime(timefmt, self._timestamp)
 
     @property
@@ -192,11 +197,11 @@ def get_default(self, option):
 _runtime_context = None
 
 
-def init_runtime(site_config):
+def init_runtime(site_config, **kwargs):
     global _runtime_context
 
     if _runtime_context is None:
-        _runtime_context = RuntimeContext(site_config)
+        _runtime_context = RuntimeContext(site_config, **kwargs)
 
 
 def runtime():

reframe/core/schedulers/__init__.py

@@ -180,8 +180,6 @@ def filter_nodes_by_state(nodelist, state):
             }
 
     return nodelist
-    nodes[part.fullname] = [n.name for n in nodelist]
-
 
 
 class Job(jsonext.JSONSerializable, metaclass=JobMeta):
@@ -377,7 +375,7 @@ def __init__(self,
         self._jobid = None
         self._exitcode = None
         self._state = None
-        self._nodelist = None
+        self._nodelist = []
         self._submit_time = None
         self._completion_time = None
 
@@ -515,7 +513,7 @@ def nodelist(self):
         This attribute is supported by the ``local``, ``pbs``, ``slurm``,
         ``squeue``, ``ssh``, and ``torque`` scheduler backends.
 
-        This attribute is :class:`None` if no nodes are assigned to the job
+        This attribute is an empty list if no nodes are assigned to the job
         yet.
 
         The ``squeue`` scheduler backend, i.e., Slurm *without* accounting,
@@ -531,7 +529,10 @@ def nodelist(self):
 
         .. versionadded:: 2.17
 
-        :type: :class:`List[str]` or :class:`None`
+        .. versionchanged:: 4.7
+           Default value is the empty list.
+
+        :type: :class:`List[str]`
         '''
         return self._nodelist
 
@@ -554,7 +555,7 @@ def prepare(self, commands, environs=None, prepare_cmds=None,
                 strict_flex=False, **gen_opts):
         environs = environs or []
         if self.num_tasks is not None and self.num_tasks <= 0:
-            getlogger().debug(f'[F] Flexible node allocation requested')
+            getlogger().debug('[F] Flexible node allocation requested')
             num_tasks_per_node = self.num_tasks_per_node or 1
             min_num_tasks = (-self.num_tasks if self.num_tasks else
                              num_tasks_per_node)
@@ -635,7 +636,7 @@ def finished(self):
         return done
 
     def __eq__(self, other):
-        return type(self) == type(other) and self.jobid == other.jobid
+        return type(self) is type(other) and self.jobid == other.jobid
 
     def __hash__(self):
         return hash(self.jobid)

reframe/frontend/argparse.py

@@ -40,6 +40,24 @@
 # that essentially associate environment variables with configuration
 # arguments, without having to define a corresponding command line option.
 
+class _Undefined:
+    pass
+
+
+# We use a special value for denoting const values that are to be set from the
+# configuration default. This placeholder must be used as the `const` argument
+#  for options with `nargs='?'`. The underlying `ArugmentParser` will use the
+# `const` value as if it were supplied from the command-line thus fooling our
+# machinery of environment variables and configuration options overriding any
+# defaults. For this reason, we use a unique placeholder so that we can
+# distinguish whether this value is a default or actually supplied from the
+# command-line.
+CONST_DEFAULT = _Undefined()
+
+
+def _undefined(val):
+    return val is None or val is CONST_DEFAULT
+
 
 class _Namespace:
     def __init__(self, namespace, option_map):
@@ -76,7 +94,10 @@ def __getattr__(self, name):
             return ret
 
         envvar, _, action, arg_type, default = self.__option_map[name]
-        if ret is None and envvar is not None:
+        if ret is CONST_DEFAULT:
+            default = CONST_DEFAULT
+
+        if _undefined(ret) and envvar is not None:
             # Try the environment variable
             envvar, *delim = envvar.split(maxsplit=2)
             delim = delim[0] if delim else ','
@@ -120,14 +141,14 @@ def update_config(self, site_config):
                 errors.append(e)
                 continue
 
-            if value is not None:
+            if not _undefined(value):
                 site_config.add_sticky_option(confvar, value)
 
         return errors
 
     def __repr__(self):
         return (f'{type(self).__name__}({self.__namespace!r}, '
-                '{self.__option_map})')
+                f'{self.__option_map})')
 
 
 class _ArgumentHolder:
@@ -149,6 +170,12 @@ def __getattr__(self, name):
 
         return getattr(self._holder, name)
 
+    def __setattr__(self, name, value):
+        if name.startswith('_'):
+            super().__setattr__(name, value)
+        else:
+            setattr(self._holder, name, value)
+
     def add_argument(self, *flags, **kwargs):
         try:
             opt_name = kwargs['dest']
@@ -233,6 +260,14 @@ def add_argument_group(self, *args, **kwargs):
         self._groups.append(group)
         return group
 
+    def add_mutually_exclusive_group(self, *args, **kwargs):
+        group = _ArgumentGroup(
+            self._holder.add_mutually_exclusive_group(*args, **kwargs),
+            self._option_map
+        )
+        self._groups.append(group)
+        return group
+
     def _resolve_attr(self, attr, namespaces):
         for ns in namespaces:
             if ns is None:
@@ -248,7 +283,7 @@ def _update_defaults(self):
         for g in self._groups:
             self._defaults.__dict__.update(g._defaults.__dict__)
 
-    def parse_args(self, args=None, namespace=None):
+    def parse_args(self, args=None, namespace=None, suppress_required=False):
         '''Convert argument strings to objects and return them as attributes of
         a namespace.
 
@@ -260,7 +295,30 @@ def parse_args(self, args=None, namespace=None):
         for it will be looked up first in `namespace` and if not found there,
         it will be assigned the default value as specified in its corresponding
         `add_argument()` call. If no default value was specified either, the
-        attribute will be set to `None`.'''
+        attribute will be set to `None`.
+
+        If `suppress_required` is true, required mutually-exclusive groups will
+        be treated as optional for this parsing operation.
+        '''
+
+        class suppress_required_groups:
+            '''Temporarily suppress required groups if `suppress_required`
+            is true.'''
+            def __init__(this):
+                this._changed_grp = []
+
+            def __enter__(this):
+                if suppress_required:
+                    for grp in self._groups:
+                        if hasattr(grp, 'required') and grp.required:
+                            this._changed_grp.append(grp)
+                            grp.required = False
+
+                return this
+
+            def __exit__(this, *args, **kwargs):
+                for grp in this._changed_grp:
+                    grp.required = True
 
         # Enable auto-completion
         argcomplete.autocomplete(self._holder)
@@ -270,7 +328,8 @@ def parse_args(self, args=None, namespace=None):
         # newly parsed options to completely override any options defined in
         # namespace. The implementation of `argparse.ArgumentParser` does not
         # do this in options with an 'append' action.
-        options = self._holder.parse_args(args, None)
+        with suppress_required_groups():
+            options = self._holder.parse_args(args, None)
 
         # Check if namespace refers to our namespace and take the cmd options
         # namespace suitable for ArgumentParser