From 5e90f6918917452d252d28c3382dd16a8a3f8e91 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 14 May 2024 11:07:19 +0200 Subject: [PATCH 01/69] Index job reports --- reframe/frontend/cli.py | 17 +++- reframe/frontend/runreport.py | 176 ++++++++++++++++++++++++++++++---- reframe/schemas/config.json | 2 + 3 files changed, 176 insertions(+), 19 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 2fe96a3eb6..56ee2f1e00 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -564,6 +564,10 @@ def main(): metavar='PARAM', help='Print the value of configuration parameter PARAM and exit' ) + misc_options.add_argument( + '--index-db', action='store_true', + help='Index old job reports in the database', + ) misc_options.add_argument( '--system', action='store', help='Load configuration for SYSTEM', envvar='RFM_SYSTEM' @@ -906,14 +910,23 @@ def restrict_logging(): with open(topofile, 'w') as fp: json.dump(s_cpuinfo, fp, indent=2) fp.write('\n') - except OSError as e: - getlogger().error( + except OSError: + logging.getlogger().error( f'could not write topology file: {topofile!r}' ) sys.exit(1) sys.exit(0) + if options.index_db: + try: + runreport.reindex_db() + except Exception as e: + printer.error(f'could not index old job reports: {e}') + sys.exit(1) + + sys.exit(0) + autodetect.detect_topology() printer.debug(format_env(options.env_vars)) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 47b59df0c7..d0bdbe0d48 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -5,14 +5,17 @@ import decimal import functools +import glob import json import jsonschema import lxml.etree as etree import os import re +import sqlite3 import reframe as rfm import reframe.core.exceptions as errors +import reframe.core.runtime as runtime import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext from reframe.core.logging import getlogger @@ -190,26 +193,41 @@ def write_report(report, filename, compress=False, link_to_last=False): jsonext.dump(report, fp, indent=2) fp.write('\n') - if not link_to_last: - return + # Try to save the report in the database + try: + site_config = runtime.runtime().site_config + database_file = osext.expandvars( + site_config.get('general/0/report_database_file') + ) + con, cur = initialize_db(database_file) + add_report_in_db(report, cur, filename) + con.commit() + con.close() + except Exception as e: + getlogger().error( + f"Could not save report {filename} in the database: {e}" + ) - # Add a symlink to the latest report - basedir = os.path.dirname(filename) - with osext.change_dir(basedir): - link_name = 'latest.json' - create_symlink = functools.partial( - os.symlink, os.path.basename(filename), link_name - ) - if not os.path.exists(link_name): + if not link_to_last: + return + + # Add a symlink to the latest report + basedir = os.path.dirname(filename) + with osext.change_dir(basedir): + link_name = 'latest.json' + create_symlink = functools.partial( + os.symlink, os.path.basename(filename), link_name + ) + if not os.path.exists(link_name): + create_symlink() + else: + if os.path.islink(link_name): + os.remove(link_name) create_symlink() else: - if os.path.islink(link_name): - os.remove(link_name) - create_symlink() - else: - getlogger().warning('could not create a symlink ' - 'to the latest report file: ' - 'path exists and is not a symlink') + getlogger().warning('could not create a symlink ' + 'to the latest report file: ' + 'path exists and is not a symlink') def junit_xml_report(json_report): @@ -270,3 +288,127 @@ def junit_dump(xml, fp): etree.tostring(xml, encoding='utf8', pretty_print=True, method='xml', xml_declaration=True).decode() ) + + +def get_reports_files(directory): + return [f for f in glob.glob(f"{directory}/*") + if os.path.isfile(f) and not f.endswith('/latest.json')] + + +def initialize_db(database_file): + con = sqlite3.connect(database_file) + cur = con.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS testcases( + name TEXT, + hash TEXT, + system TEXT, + partition TEXT, + environment TEXT, + time_start TEXT, + time_end TEXT, + run_index INTEGER, + test_index INTEGER, + json_file TEXT + ) + """) + con.commit() + return (con, cur) + + +def reindex_db(): + site_config = runtime.runtime().site_config + database_file = osext.expandvars( + site_config.get('general/0/report_database_file') + ) + # Check if the database file exists and remove it + if os.path.exists(database_file): + os.remove(database_file) + + con, cur = initialize_db(database_file) + + # Populate the database with old reports + reports_directory = os.path.dirname( + osext.expandvars(site_config.get('general/0/report_file')) + ) + for f_path in get_reports_files(reports_directory): + with open(f_path, 'r') as file: + report = json.load(file) + #TODO: Validate report with schema + + add_report_in_db(report, cur, f_path) + con.commit() + + con.close() + + +def add_report_in_db(report, cursor, report_file_path): + time_start = report['session_info']['time_start'] + time_end = report['session_info']['time_end'] + for run_index, run in enumerate(report['runs']): + for test_index, test in enumerate(run['testcases']): + name = test['name'] + hash = test['hash'] + system = test['system'].split(':')[0] + partition = test['system'].split(':')[1] + environment = test['environment'] + run_index = run_index + test_index = test_index + json_file = report_file_path + + cursor.execute( + ( + "INSERT INTO testcases VALUES(?, ?, ?, ?, ?, ?, ?, ?, " + "?, ?)" + ), + ( + name, + hash, + system, + partition, + environment, + time_start, + time_end, + run_index, + test_index, + json_file + ) + ) + + +def search_db(name=None, hash=None, system=None, partition=None, + environment=None, time_start=None, time_end=None, + run_index=None, test_index=None): + site_config = runtime.runtime().site_config + database_file = osext.expandvars( + site_config.get('general/0/report_database_file') + ) + con = sqlite3.connect(database_file) + cur = con.cursor() + + query = "SELECT test_index, run_index, json_file FROM testcases" + first_condition = False + for condition in ['name', 'hash', 'system', 'partition', 'environment', + 'time_start', 'time_end', 'run_index', 'test_index']: + value = eval(condition) + if value: + if not first_condition: + query += " WHERE " + first_condition = True + else: + query += " AND " + query += f"{condition}='{eval(condition)}'" + + res = cur.execute(query) + results = res.fetchall() + con.close() + + # Retrieve files + testcases = [] + for test_index, run_index, json_file in results: + with open(json_file, 'r') as file: + report = json.load(file) + + testcases.append(report['runs'][run_index]['testcases'][test_index]) + + return testcases diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 61d9c58e18..2dcbec053c 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -518,6 +518,7 @@ "purge_environment": {"type": "boolean"}, "remote_detect": {"type": "boolean"}, "remote_workdir": {"type": "string"}, + "report_database_file": {"type": "string"}, "report_file": {"type": "string"}, "report_junit": {"type": ["string", "null"]}, "resolve_module_conflicts": {"type": "boolean"}, @@ -569,6 +570,7 @@ "general/purge_environment": false, "general/remote_detect": false, "general/remote_workdir": ".", + "general/report_database_file": "${HOME}/.reframe/reports_index.db", "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/report_junit": null, "general/resolve_module_conflicts": true, From 36a3efdd1d68d8e9418cb917d21c316c37413d3d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 22 May 2024 10:56:27 +0200 Subject: [PATCH 02/69] Refine report indexing --- reframe/core/logging.py | 2 +- reframe/frontend/cli.py | 50 +++--- reframe/frontend/executors/__init__.py | 13 ++ reframe/frontend/runreport.py | 202 +++++++++---------------- reframe/frontend/statistics.py | 79 +++------- reframe/schemas/runreport.json | 61 +------- 6 files changed, 141 insertions(+), 266 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index d34006fed1..e52de9e221 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -862,7 +862,7 @@ def log_performance(self, level, task, msg=None, multiline=False): self.extra['check_partition'] = task.testcase.partition.name self.extra['check_environ'] = task.testcase.environ.name - self.extra['check_result'] = 'pass' if task.succeeded else 'fail' + self.extra['check_result'] = task.result fail_reason = what(*task.exc_info) if not task.succeeded else None self.extra['check_fail_reason'] = fail_reason if msg is None: diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 56ee2f1e00..f9ed7ccb6c 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -584,6 +584,10 @@ def main(): '-q', '--quiet', action='count', default=0, help='Decrease verbosity level of output', ) + misc_options.add_argument( + '--fetch-cases', action='store', + help='Experimental: query report' + ) # Options not associated with command-line arguments argparser.add_argument( @@ -727,7 +731,7 @@ def restrict_logging(): ''' - if (options.show_config or + if (options.show_config or options.fetch_cases or options.detect_host_topology or options.describe): logging.getlogger().setLevel(logging.ERROR) return True @@ -918,18 +922,14 @@ def restrict_logging(): sys.exit(0) - if options.index_db: - try: - runreport.reindex_db() - except Exception as e: - printer.error(f'could not index old job reports: {e}') - sys.exit(1) - - sys.exit(0) - autodetect.detect_topology() printer.debug(format_env(options.env_vars)) + if options.fetch_cases: + testcases = runreport.fetch_cases_raw(options.fetch_cases) + print(jsonext.dumps(testcases, indent=2)) + sys.exit(0) + # Setup the check loader if options.restore_session is not None: # We need to load the failed checks only from a list of reports @@ -1478,19 +1478,35 @@ def module_unuse(*paths): for c in restored_cases: json_report['restored_cases'].append(report.case(*c)) + # Save the report file report_file = runreport.next_report_filename(report_file) - default_loc = os.path.dirname( - osext.expandvars(rt.get_default('general/report_file')) - ) try: - runreport.write_report(json_report, report_file, - rt.get_option( - 'general/0/compress_report'), - os.path.dirname(report_file) == default_loc) + runreport.save_report( + json_report, report_file, + rt.get_option('general/0/compress_report') + ) except OSError as e: printer.warning( f'failed to generate report in {report_file!r}: {e}' ) + else: + default_loc = os.path.dirname( + osext.expandvars(rt.get_default('general/report_file')) + ) + if default_loc == os.path.dirname(report_file): + try: + runreport.link_latest_report(report_file, 'latest.json') + except Exception as e: + printer.warning( + f'failed to create symlink to latest report: {e}' + ) + + # Index the generated report + try: + runreport.index_report(json_report, report_file) + except Exception as e: + printer.warning(f'failed to index the report: {e}') + raise # Generate the junit xml report for this session junit_report_file = rt.get_option('general/0/report_junit') diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index dd95746987..6c741d6e61 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -282,6 +282,19 @@ def aborted(self): def skipped(self): return self._skipped + @property + def result(self): + if self.succeeded: + return 'pass' + elif self.failed: + return 'fail' + elif self.aborted: + return 'abort' + elif self.skipped: + return 'skip' + else: + return '' + def _notify_listeners(self, callback_name): for l in self._listeners: callback = getattr(l, callback_name) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index d0bdbe0d48..1368ea33e9 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -24,7 +24,7 @@ # The schema data version # Major version bumps are expected to break the validation of previous schemas -DATA_VERSION = '3.1' +DATA_VERSION = '4.0' _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') @@ -169,7 +169,7 @@ def _load_report(filename): raise errors.ReframeError( f'invalid report {filename!r} ' f'(required data version: {DATA_VERSION}), found: {found_ver})' - ) from e + ) return _RunReport(report) @@ -185,7 +185,7 @@ def load_report(*filenames): return rpt -def write_report(report, filename, compress=False, link_to_last=False): +def save_report(report, filename, compress=False): with open(filename, 'w') as fp: if compress: jsonext.dump(report, fp) @@ -193,31 +193,10 @@ def write_report(report, filename, compress=False, link_to_last=False): jsonext.dump(report, fp, indent=2) fp.write('\n') - # Try to save the report in the database - try: - site_config = runtime.runtime().site_config - database_file = osext.expandvars( - site_config.get('general/0/report_database_file') - ) - con, cur = initialize_db(database_file) - add_report_in_db(report, cur, filename) - con.commit() - con.close() - except Exception as e: - getlogger().error( - f"Could not save report {filename} in the database: {e}" - ) - - if not link_to_last: - return - - # Add a symlink to the latest report - basedir = os.path.dirname(filename) - with osext.change_dir(basedir): - link_name = 'latest.json' - create_symlink = functools.partial( - os.symlink, os.path.basename(filename), link_name - ) +def link_latest_report(filename, link_name): + prefix, target_name = os.path.split(filename) + with osext.change_dir(prefix): + create_symlink = functools.partial(os.symlink, target_name, link_name) if not os.path.exists(link_name): create_symlink() else: @@ -225,9 +204,7 @@ def write_report(report, filename, compress=False, link_to_last=False): os.remove(link_name) create_symlink() else: - getlogger().warning('could not create a symlink ' - 'to the latest report file: ' - 'path exists and is not a symlink') + raise errors.ReframeError('path exists and is not a symlink') def junit_xml_report(json_report): @@ -251,11 +228,8 @@ def junit_xml_report(json_report): 'timestamp': json_report['session_info']['time_start'][:-5], } ) - testsuite_properties = etree.SubElement(xml_testsuite, 'properties') for tc in rfm_run['testcases']: - casename = ( - f"{tc['unique_name']}[{tc['system']}, {tc['environment']}]" - ) + casename = f"{tc['name']} @{tc['system']}+{tc['environ']}" testcase = etree.SubElement( xml_testsuite, 'testcase', attrib={ @@ -268,10 +242,10 @@ def junit_xml_report(json_report): 'time': str(decimal.Decimal(tc['time_total'] or 0)), } ) - if tc['result'] == 'failure': + if tc['result'] == 'fail': testcase_msg = etree.SubElement( - testcase, 'failure', attrib={'type': 'failure', - 'message': tc['fail_phase']} + testcase, 'fail', attrib={'type': 'fail', + 'message': tc['fail_phase']} ) testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}" @@ -295,113 +269,77 @@ def get_reports_files(directory): if os.path.isfile(f) and not f.endswith('/latest.json')] -def initialize_db(database_file): - con = sqlite3.connect(database_file) - cur = con.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS testcases( +def index_report(report, filename): + prefix = os.path.dirname(filename) + index_file = os.path.join(prefix, 'index.db') + if not os.path.exists(index_file): + getlogger().info('Re-building the report index...') + _build_index(index_file) + + with sqlite3.connect(index_file) as conn: + _index_report(conn, report, filename) + + +def _build_index(filename): + with sqlite3.connect(filename) as conn: + conn.execute( +'''CREATE TABLE IF NOT EXISTS testcases( name TEXT, - hash TEXT, system TEXT, partition TEXT, - environment TEXT, + environ TEXT, time_start TEXT, time_end TEXT, run_index INTEGER, test_index INTEGER, - json_file TEXT - ) - """) - con.commit() - return (con, cur) - - -def reindex_db(): - site_config = runtime.runtime().site_config - database_file = osext.expandvars( - site_config.get('general/0/report_database_file') - ) - # Check if the database file exists and remove it - if os.path.exists(database_file): - os.remove(database_file) - - con, cur = initialize_db(database_file) - - # Populate the database with old reports - reports_directory = os.path.dirname( - osext.expandvars(site_config.get('general/0/report_file')) - ) - for f_path in get_reports_files(reports_directory): - with open(f_path, 'r') as file: - report = json.load(file) - #TODO: Validate report with schema - - add_report_in_db(report, cur, f_path) - con.commit() - - con.close() - - -def add_report_in_db(report, cursor, report_file_path): + report_file TEXT +)''' + ) + prefix = os.path.dirname(filename) + for report_file in glob.iglob(f'{prefix}/*'): + if not os.path.islink(report_file): + try: + report = load_report(report_file) + # except errors.ReframeError as e: + except Exception as e: + getlogger().info(f'ignoring report file {report_file!r}: {e}') + else: + _index_report(conn, report, report_file) + + +def _index_report(conn, report, report_file_path): time_start = report['session_info']['time_start'] time_end = report['session_info']['time_end'] - for run_index, run in enumerate(report['runs']): - for test_index, test in enumerate(run['testcases']): - name = test['name'] - hash = test['hash'] - system = test['system'].split(':')[0] - partition = test['system'].split(':')[1] - environment = test['environment'] - run_index = run_index - test_index = test_index - json_file = report_file_path - - cursor.execute( - ( - "INSERT INTO testcases VALUES(?, ?, ?, ?, ?, ?, ?, ?, " - "?, ?)" - ), - ( - name, - hash, - system, - partition, - environment, - time_start, - time_end, - run_index, - test_index, - json_file - ) + for run_idx, run in enumerate(report['runs']): + for test_idx, testcase in enumerate(run['testcases']): + sys, part = testcase['system'], testcase['partition'] + conn.execute( +'''INSERT INTO testcases VALUES(:name, :system, :partition, + :environ, :time_start, :time_end, + :run_index, :test_index, :report_file)''', + { + 'name': testcase['name'], + 'system': sys, + 'partition': part, + 'environ': testcase['environ'], + 'run_index': run_idx, + 'time_start': time_start, + 'time_end': time_end, + 'test_index': test_idx, + 'report_file': report_file_path + } ) -def search_db(name=None, hash=None, system=None, partition=None, - environment=None, time_start=None, time_end=None, - run_index=None, test_index=None): +def fetch_cases_raw(condition): site_config = runtime.runtime().site_config - database_file = osext.expandvars( - site_config.get('general/0/report_database_file') - ) - con = sqlite3.connect(database_file) - cur = con.cursor() - - query = "SELECT test_index, run_index, json_file FROM testcases" - first_condition = False - for condition in ['name', 'hash', 'system', 'partition', 'environment', - 'time_start', 'time_end', 'run_index', 'test_index']: - value = eval(condition) - if value: - if not first_condition: - query += " WHERE " - first_condition = True - else: - query += " AND " - query += f"{condition}='{eval(condition)}'" - - res = cur.execute(query) - results = res.fetchall() - con.close() + prefix = os.path.dirname(osext.expandvars( + site_config.get('general/0/report_file') + )) + with sqlite3.connect(os.path.join(prefix, 'index.db')) as conn: + query = f'SELECT test_index, run_index, report_file FROM testcases where {condition}' + getlogger().debug(query) + results = conn.execute(query).fetchall() # Retrieve files testcases = [] diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 84b3d129cd..c13c4409e7 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -110,6 +110,7 @@ def json(self, force=False): check = t.check partition = check.current_partition entry = { + 'build_jobid': None, 'build_stderr': None, 'build_stdout': None, 'dependencies_actual': [ @@ -120,35 +121,20 @@ def json(self, force=False): 'dependencies_conceptual': [ d[0] for d in t.check.user_deps() ], - 'description': check.descr, - 'display_name': check.display_name, - 'environment': None, 'fail_phase': None, 'fail_reason': None, 'filename': inspect.getfile(type(check)), 'fixture': check.is_fixture(), - 'hash': check.hashcode, - 'jobid': None, 'job_stderr': None, 'job_stdout': None, - 'maintainers': check.maintainers, - 'name': check.name, - 'nodelist': [], - 'outputdir': None, - 'perfvars': None, - 'prefix': check.prefix, 'result': None, - 'stagedir': check.stagedir, 'scheduler': None, - 'system': check.current_system.name, - 'tags': list(check.tags), 'time_compile': t.duration('compile_complete'), 'time_performance': t.duration('performance'), 'time_run': t.duration('run_complete'), 'time_sanity': t.duration('sanity'), 'time_setup': t.duration('setup'), - 'time_total': t.duration('total'), - 'unique_name': check.unique_name + 'time_total': t.duration('total') } # We take partition and environment from the test case and not @@ -156,25 +142,24 @@ def json(self, force=False): # these are not set inside the check. partition = t.testcase.partition environ = t.testcase.environ - entry['system'] = partition.fullname + entry['partition'] = partition.name + entry['environ'] = environ.name entry['scheduler'] = partition.scheduler.registered_name - entry['environment'] = environ.name if check.job: - entry['jobid'] = str(check.job.jobid) entry['job_stderr'] = check.stderr.evaluate() entry['job_stdout'] = check.stdout.evaluate() - entry['nodelist'] = check.job.nodelist or [] if check.build_job: entry['build_stderr'] = check.build_stderr.evaluate() entry['build_stdout'] = check.build_stdout.evaluate() + entry['result'] = t.result if t.failed: num_failures += 1 - entry['result'] = 'failure' elif t.aborted: - entry['result'] = 'aborted' num_aborted += 1 + elif t.skipped: + num_skipped += 1 if t.failed or t.aborted: entry['fail_phase'] = t.failed_stage @@ -186,43 +171,17 @@ def json(self, force=False): 'traceback': t.exc_info[2] } entry['fail_severe'] = errors.is_severe(*t.exc_info) - elif t.skipped: - entry['result'] = 'skipped' - num_skipped += 1 - else: - entry['result'] = 'success' + elif t.succeeded: entry['outputdir'] = check.outputdir - if check.perfvalues: - # Record performance variables - entry['perfvars'] = [] - for key, ref in check.perfvalues.items(): - var = key.split(':')[-1] - val, ref, lower, upper, unit = ref - entry['perfvars'].append({ - 'name': var, - 'reference': ref, - 'thres_lower': lower, - 'thres_upper': upper, - 'unit': unit, - 'value': val - }) - # Add any loggable variables and parameters - entry['check_vars'] = {} - test_cls = type(check) - for name, var in test_cls.var_space.items(): - if var.is_loggable(): - try: - entry['check_vars'][name] = _getattr(check, name) - except AttributeError: - entry['check_vars'][name] = '' - - entry['check_params'] = {} test_cls = type(check) - for name, param in test_cls.param_space.items(): - if param.is_loggable(): - entry['check_params'][name] = _getattr(check, name) + for name, alt_name in test_cls.loggable_attrs(): + key = alt_name if alt_name else name + try: + entry[key] = _getattr(check, name) + except AttributeError: + entry[key] = '' testcases.append(entry) @@ -264,12 +223,12 @@ def _print_failure_info(rec, runid, total_runs): printer.info(line_width * '-') printer.info(f"FAILURE INFO for {rec['display_name']} " f"(run: {runid}/{total_runs})") - printer.info(f" * Description: {rec['description']}") + printer.info(f" * Description: {rec['descr']}") printer.info(f" * System partition: {rec['system']}") - printer.info(f" * Environment: {rec['environment']}") + printer.info(f" * Environment: {rec['environ']}") printer.info(f" * Stage directory: {rec['stagedir']}") printer.info( - f" * Node list: {util.nodelist_abbrev(rec['nodelist'])}" + f" * Node list: {util.nodelist_abbrev(rec['job_nodelist'])}" ) job_type = 'local' if rec['scheduler'] == 'local' else 'batch job' printer.info(f" * Job type: {job_type} (id={rec['jobid']})") @@ -280,8 +239,8 @@ def _print_failure_info(rec, runid, total_runs): printer.info(f" * Maintainers: {rec['maintainers']}") printer.info(f" * Failing phase: {rec['fail_phase']}") if rerun_info and not rec['fixture']: - printer.info(f" * Rerun with '-n /{rec['hash']}" - f" -p {rec['environment']} --system " + printer.info(f" * Rerun with '-n /{rec['hashcode']}" + f" -p {rec['environ']} --system " f"{rec['system']} -r'") msg = rec['fail_reason'] diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index 7c1efca680..e989559476 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -6,10 +6,9 @@ "testcase_type": { "type": "object", "properties": { + "build_jobid": {"type": ["string", "null"]}, "build_stderr": {"type": ["string", "null"]}, "build_stdout": {"type": ["string", "null"]}, - "check_params": {"type": "object"}, - "check_vars": {"type": "object"}, "dependencies_actual": { "type": "array", "items": { @@ -23,9 +22,6 @@ "type": "array", "items": {"type": "string"} }, - "description": {"type": "string"}, - "display_name": {"type": "string"}, - "environment": {"type": ["string", "null"]}, "fail_info": { "type": ["object", "null"], "properties": { @@ -43,65 +39,18 @@ "fail_severe": {"type": "boolean"}, "filename": {"type": "string"}, "fixture": {"type": "boolean"}, - "jobid": {"type": ["string", "null"]}, "job_stderr": {"type": ["string", "null"]}, "job_stdout": {"type": ["string", "null"]}, - "maintainers": { - "type": "array", - "items": {"type": "string"} - }, - "name": {"type": "string"}, - "nodelist": { - "type": "array", - "items": {"type": "string"} - }, - "outputdir": {"type": ["string", "null"]}, - "perfvars": { - "type": ["array", "null"], - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "reference": { - "type": ["number", "null"] - }, - "thres_lower": { - "type": ["number", "null"] - }, - "thres_upper": { - "type": ["number", "null"] - }, - "unit": {"type": ["string", "null"]}, - "value": {"type": "number"} - }, - "required": [ - "name", "reference", - "thres_lower", "thres_upper", - "unit", "value" - ] - } - }, - "prefix": {"type": "string"}, - "result": { - "type": "string", - "enum": ["success", "failure", "aborted", "skipped"] - }, - "scheduler": {"type": ["string", "null"]}, - "stagedir": {"type": ["string", "null"]}, - "system": {"type": "string"}, - "tags": { - "type": "array", - "items": {"type": "string"} - }, + "result": {"type": "string"}, "time_compile": {"type": ["number", "null"]}, "time_performance": {"type": ["number", "null"]}, "time_run": {"type": ["number", "null"]}, "time_sanity": {"type": ["number", "null"]}, "time_setup": {"type": ["number", "null"]}, - "time_total": {"type": ["number", "null"]}, - "unique_name": {"type": "string"} + "time_total": {"type": ["number", "null"]} }, - "required": ["environment", "stagedir", "system", "unique_name"] + "required": ["name", "system", "partition", "environ", + "perfvalues", "result", "fail_reason", "file"] } }, "type": "object", From 4438503bfe230a30966afce1537e62709974b3b2 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 27 May 2024 18:04:12 +0200 Subject: [PATCH 03/69] Performance comparison of past results --- reframe/core/logging.py | 5 +- reframe/core/pipeline.py | 3 +- reframe/core/schedulers/__init__.py | 11 +- reframe/frontend/cli.py | 62 +++-- reframe/frontend/runreport.py | 383 +++++++++++++++++++++++----- reframe/frontend/statistics.py | 56 ++-- reframe/schemas/runreport.json | 29 ++- requirements.txt | 1 + unittests/test_cli.py | 11 +- unittests/test_policies.py | 4 +- unittests/test_schedulers.py | 4 +- 11 files changed, 435 insertions(+), 134 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index e52de9e221..223840f281 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -860,8 +860,9 @@ def log_performance(self, level, task, msg=None, multiline=False): if self.check is None or not self.check.is_performance_check(): return - self.extra['check_partition'] = task.testcase.partition.name - self.extra['check_environ'] = task.testcase.environ.name + _, part, env = task.testcase + self.extra['check_partition'] = part.name + self.extra['check_environ'] = env.name self.extra['check_result'] = task.result fail_reason = what(*task.exc_info) if not task.succeeded else None self.extra['check_fail_reason'] = fail_reason diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 462b103b4b..b532e7b36f 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1517,8 +1517,7 @@ def _job_exitcode(self): @loggable_as('job_nodelist') @property def _job_nodelist(self): - if self.job: - return self.job.nodelist + return self.job.nodelist if self.job else [] def info(self): '''Provide live information for this test. diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 0266e43235..eaeb9125c9 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -180,8 +180,6 @@ def filter_nodes_by_state(nodelist, state): } return nodelist - nodes[part.fullname] = [n.name for n in nodelist] - class Job(jsonext.JSONSerializable, metaclass=JobMeta): @@ -377,7 +375,7 @@ def __init__(self, self._jobid = None self._exitcode = None self._state = None - self._nodelist = None + self._nodelist = [] self._submit_time = None self._completion_time = None @@ -515,7 +513,7 @@ def nodelist(self): This attribute is supported by the ``local``, ``pbs``, ``slurm``, ``squeue``, ``ssh``, and ``torque`` scheduler backends. - This attribute is :class:`None` if no nodes are assigned to the job + This attribute is an empty list if no nodes are assigned to the job yet. The ``squeue`` scheduler backend, i.e., Slurm *without* accounting, @@ -531,7 +529,10 @@ def nodelist(self): .. versionadded:: 2.17 - :type: :class:`List[str]` or :class:`None` + .. versionchanged:: 4.7 + Default value is the empty list. + + :type: :class:`List[str]` ''' return self._nodelist diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f9ed7ccb6c..de5529b1ad 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -13,6 +13,7 @@ import sys import time import traceback +from tabulate import tabulate import reframe.core.config as config import reframe.core.exceptions as errors @@ -28,7 +29,6 @@ import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext import reframe.utility.typecheck as typ - from reframe.frontend.testgenerators import (distribute_tests, getallnodes, repeat_tests, parameterize_tests) @@ -556,7 +556,12 @@ def main(): envvar='RFM_COLORIZE', configvar='general/colorize' ) misc_options.add_argument( - '--performance-report', action='store_true', + '--performance-compare', metavar='CMPSPEC', action='store', + help='Compare past performance results' + ) + misc_options.add_argument( + '--performance-report', action='store', nargs='?', + const='19700101T0000Z:now/last:+job_nodelist/+result', help='Print a report for performance tests' ) misc_options.add_argument( @@ -584,10 +589,6 @@ def main(): '-q', '--quiet', action='count', default=0, help='Decrease verbosity level of output', ) - misc_options.add_argument( - '--fetch-cases', action='store', - help='Experimental: query report' - ) # Options not associated with command-line arguments argparser.add_argument( @@ -731,7 +732,7 @@ def restrict_logging(): ''' - if (options.show_config or options.fetch_cases or + if (options.show_config or options.detect_host_topology or options.describe): logging.getlogger().setLevel(logging.ERROR) return True @@ -925,11 +926,6 @@ def restrict_logging(): autodetect.detect_topology() printer.debug(format_env(options.env_vars)) - if options.fetch_cases: - testcases = runreport.fetch_cases_raw(options.fetch_cases) - print(jsonext.dumps(testcases, indent=2)) - sys.exit(0) - # Setup the check loader if options.restore_session is not None: # We need to load the failed checks only from a list of reports @@ -941,7 +937,12 @@ def restrict_logging(): new=False )] - report = runreport.load_report(*filenames) + try: + report = runreport.load_report(*filenames) + except errors.ReframeError as err: + printer.error(f'failed to load restore session: {err}') + sys.exit(1) + check_search_path = list(report.slice('filename', unique=True)) check_search_recursive = False @@ -1137,12 +1138,11 @@ def _case_failed(t): if not rec: return False - return (rec['result'] == 'failure' or - rec['result'] == 'aborted') + return rec['result'] == 'fail' or rec['result'] == 'abort' testcases = list(filter(_case_failed, testcases)) printer.verbose( - f'Filtering successful test case(s): ' + f'Filtering out successful test case(s): ' f'{len(testcases)} remaining' ) @@ -1290,6 +1290,13 @@ def _sort_testcases(testcases): ) sys.exit(0) + if options.performance_compare: + data, header = runreport.performance_compare_data( + options.performance_compare + ) + print(tabulate(data, headers=header, tablefmt='mixed_grid')) + sys.exit(0) + if not options.run and not options.dry_run: printer.error("No action option specified. Available options:\n" " - `-l'/`-L' for listing\n" @@ -1453,7 +1460,20 @@ def module_unuse(*paths): ) if options.performance_report and not options.dry_run: - printer.info(runner.stats.performance_report()) + try: + data, header = runreport.performance_report_data( + runner.stats.json(), + options.performance_report + ) + except errors.ReframeError as err: + printer.warning( + f'failed to generate performance report: {err}' + ) + else: + print( + tabulate(data, headers=header, tablefmt='mixed_grid') + ) + # printer.info(runner.stats.performance_report()) # Generate the report for this session report_file = os.path.normpath( @@ -1467,7 +1487,9 @@ def module_unuse(*paths): run_stats = runner.stats.json() session_info.update({ 'num_cases': run_stats[0]['num_cases'], - 'num_failures': run_stats[-1]['num_failures'] + 'num_failures': run_stats[-1]['num_failures'], + 'num_aborted': run_stats[-1]['num_aborted'], + 'num_skipped': run_stats[-1]['num_skipped'] }) json_report = { 'session_info': session_info, @@ -1503,9 +1525,9 @@ def module_unuse(*paths): # Index the generated report try: - runreport.index_report(json_report, report_file) + runreport.store_results(json_report, report_file) except Exception as e: - printer.warning(f'failed to index the report: {e}') + printer.warning(f'failed to store results in the database: {e}') raise # Generate the junit xml report for this session diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 1368ea33e9..c1de31cf7f 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -3,23 +3,31 @@ # # SPDX-License-Identifier: BSD-3-Clause +import abc import decimal import functools import glob import json import jsonschema import lxml.etree as etree +import math import os import re import sqlite3 +import statistics +import types +from collections.abc import Hashable +from datetime import datetime, timedelta import reframe as rfm -import reframe.core.exceptions as errors import reframe.core.runtime as runtime import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext +from reframe.core.exceptions import ReframeError from reframe.core.logging import getlogger from reframe.core.warnings import suppress_deprecations +from reframe.utility import nodelist_abbrev + # The schema data version # Major version bumps are expected to break the validation of previous schemas @@ -27,6 +35,13 @@ DATA_VERSION = '4.0' _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') +def _tc_info(tc_entry, name='unique_name'): + name = tc_entry[name] + system = tc_entry['system'] + partition = tc_entry['partition'] + environ = tc_entry['environ'] + return f'{name}@{system}:{partition}+{environ}' + class _RunReport: '''A wrapper to the run report providing some additional functionality''' @@ -40,13 +55,11 @@ def __init__(self, report): self._cases_index = {} for run in self._report['runs']: for tc in run['testcases']: - c, p, e = tc['unique_name'], tc['system'], tc['environment'] - self._cases_index[c, p, e] = tc + self._cases_index[_tc_info(tc)] = tc # Index also the restored cases for tc in self._report['restored_cases']: - c, p, e = tc['unique_name'], tc['system'], tc['environment'] - self._cases_index[c, p, e] = tc + self._cases_index[_tc_info(tc)] = tc def __getitem__(self, key): return self._report[key] @@ -81,12 +94,12 @@ def slice(self, prop, when=None, unique=False): yield val def case(self, check, part, env): - c, p, e = check.unique_name, part.fullname, env.name - ret = self._cases_index.get((c, p, e)) + key = f'{check.unique_name}@{part.fullname}+{env.name}' + ret = self._cases_index.get(key) if ret is None: # Look up the case in the fallback reports for rpt in self._fallbacks: - ret = rpt._cases_index.get((c, p, e)) + ret = rpt._cases_index.get(key) if ret is not None: break @@ -110,7 +123,7 @@ def restore_dangling(self, graph): def _do_restore(self, testcase): tc = self.case(*testcase) if tc is None: - raise errors.ReframeError( + raise ReframeError( f'could not restore testcase {testcase!r}: ' f'not found in the report files' ) @@ -120,7 +133,7 @@ def _do_restore(self, testcase): with open(dump_file) as fp: testcase._check = jsonext.load(fp) except (OSError, json.JSONDecodeError) as e: - raise errors.ReframeError( + raise ReframeError( f'could not restore testcase {testcase!r}') from e @@ -148,10 +161,10 @@ def _load_report(filename): with open(filename) as fp: report = json.load(fp) except OSError as e: - raise errors.ReframeError( + raise ReframeError( f'failed to load report file {filename!r}') from e except json.JSONDecodeError as e: - raise errors.ReframeError( + raise ReframeError( f'report file {filename!r} is not a valid JSON file') from e # Validate the report @@ -166,10 +179,12 @@ def _load_report(filename): except KeyError: found_ver = 'n/a' - raise errors.ReframeError( - f'invalid report {filename!r} ' - f'(required data version: {DATA_VERSION}), found: {found_ver})' - ) + getlogger().verbose(f'JSON validation error: {e}') + raise ReframeError( + f'failed to validate report {filename!r}: {e.args[0]} ' + f'(check report data version: required {DATA_VERSION}, ' + f'found: {found_ver})' + ) from None return _RunReport(report) @@ -204,7 +219,7 @@ def link_latest_report(filename, link_name): os.remove(link_name) create_symlink() else: - raise errors.ReframeError('path exists and is not a symlink') + raise ReframeError('path exists and is not a symlink') def junit_xml_report(json_report): @@ -223,13 +238,13 @@ def junit_xml_report(json_report): 'package': 'reframe', 'tests': str(rfm_run['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), - # XSD schema does not like the timezone format, so we remove it 'timestamp': json_report['session_info']['time_start'][:-5], } ) + etree.SubElement(xml_testsuite, 'properties') for tc in rfm_run['testcases']: - casename = f"{tc['name']} @{tc['system']}+{tc['environ']}" + casename = f'{_tc_info(tc, name="name")}' testcase = etree.SubElement( xml_testsuite, 'testcase', attrib={ @@ -244,8 +259,8 @@ def junit_xml_report(json_report): ) if tc['result'] == 'fail': testcase_msg = etree.SubElement( - testcase, 'fail', attrib={'type': 'fail', - 'message': tc['fail_phase']} + testcase, 'failure', attrib={'type': 'failure', + 'message': tc['fail_phase']} ) testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}" @@ -269,84 +284,320 @@ def get_reports_files(directory): if os.path.isfile(f) and not f.endswith('/latest.json')] -def index_report(report, filename): - prefix = os.path.dirname(filename) - index_file = os.path.join(prefix, 'index.db') - if not os.path.exists(index_file): - getlogger().info('Re-building the report index...') - _build_index(index_file) +def _db_file(): + site_config = runtime.runtime().site_config + prefix = os.path.dirname(osext.expandvars( + site_config.get('general/0/report_file') + )) + filename = os.path.join(prefix, 'results.db') + if not os.path.exists(filename): + # Create subdirs if needed + if prefix: + os.makedirs(prefix, exist_ok=True) + + getlogger().debug(f'Creating the results database in {filename}...') + _db_create(filename) + + return filename + - with sqlite3.connect(index_file) as conn: - _index_report(conn, report, filename) +def store_results(report, report_file): + with sqlite3.connect(_db_file()) as conn: + _db_store_report(conn, report, report_file) -def _build_index(filename): +def _db_create(filename): with sqlite3.connect(filename) as conn: conn.execute( +'''CREATE TABLE IF NOT EXISTS sessions( + id INTEGER PRIMARY KEY, + json_blob TEXT, + report_file TEXT +)''' + ) + conn.execute( '''CREATE TABLE IF NOT EXISTS testcases( name TEXT, system TEXT, partition TEXT, environ TEXT, - time_start TEXT, - time_end TEXT, + job_completion_time_unix REAL, + session_id INTEGER, run_index INTEGER, test_index INTEGER, - report_file TEXT + FOREIGN KEY(session_id) REFERENCES sessions(session_id) )''' ) - prefix = os.path.dirname(filename) - for report_file in glob.iglob(f'{prefix}/*'): - if not os.path.islink(report_file): - try: - report = load_report(report_file) - # except errors.ReframeError as e: - except Exception as e: - getlogger().info(f'ignoring report file {report_file!r}: {e}') - else: - _index_report(conn, report, report_file) - - -def _index_report(conn, report, report_file_path): - time_start = report['session_info']['time_start'] - time_end = report['session_info']['time_end'] + + +def _db_store_report(conn, report, report_file_path): + session_start = report['session_info']['time_start'] for run_idx, run in enumerate(report['runs']): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] + cursor = conn.execute( +'''INSERT INTO sessions VALUES(:session_id, :json_blob, :report_file)''', + {'session_id': None, + 'json_blob': jsonext.dumps(report), + 'report_file': report_file_path}) conn.execute( -'''INSERT INTO testcases VALUES(:name, :system, :partition, - :environ, :time_start, :time_end, - :run_index, :test_index, :report_file)''', +'''INSERT INTO testcases VALUES(:name, :system, :partition, :environ, + :job_completion_time_unix, + :session_id, :run_index, :test_index)''', { 'name': testcase['name'], 'system': sys, 'partition': part, 'environ': testcase['environ'], + 'job_completion_time_unix': testcase[ + 'job_completion_time_unix' + ], + 'session_id': cursor.lastrowid, 'run_index': run_idx, - 'time_start': time_start, - 'time_end': time_end, - 'test_index': test_idx, - 'report_file': report_file_path + 'test_index': test_idx } ) -def fetch_cases_raw(condition): - site_config = runtime.runtime().site_config - prefix = os.path.dirname(osext.expandvars( - site_config.get('general/0/report_file') - )) - with sqlite3.connect(os.path.join(prefix, 'index.db')) as conn: - query = f'SELECT test_index, run_index, report_file FROM testcases where {condition}' +def _fetch_cases_raw(condition): + with sqlite3.connect(_db_file()) as conn: + query = (f'SELECT session_id, run_index, test_index, json_blob FROM ' + f'testcases JOIN sessions ON session_id==id ' + f'WHERE {condition}') getlogger().debug(query) results = conn.execute(query).fetchall() # Retrieve files testcases = [] - for test_index, run_index, json_file in results: - with open(json_file, 'r') as file: - report = json.load(file) - + sessions = {} + for session_id, run_index, test_index, json_blob in results: + report = json.loads(sessions.setdefault(session_id, json_blob)) testcases.append(report['runs'][run_index]['testcases'][test_index]) return testcases + + +def _fetch_cases_time_period(ts_start, ts_end): + return _fetch_cases_raw( + f'(job_completion_time_unix >= {ts_start} AND ' + f'job_completion_time_unix < {ts_end}) ' + 'ORDER BY job_completion_time_unix' + ) + + +def _group_key(groups, testcase): + key = [] + for grp in groups: + val = testcase[grp] + if grp == 'job_nodelist': + # Fold nodelist before adding as a key element + key.append(nodelist_abbrev(val)) + elif not isinstance(val, Hashable): + key.append(str(val)) + else: + key.append(val) + + return tuple(key) + + +def parse_timestamp(s): + now = datetime.now() + def _do_parse(s): + if s == 'now': + return now + + formats = [r'%Y%m%d', r'%Y%m%dT%H%M', + r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] + for fmt in formats: + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + + raise ValueError(f'invalid timestamp: {s}') + + + try: + ts = _do_parse(s) + except ValueError as err: + # Try the relative timestamps + match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) + if not match: + raise err + + ts = _do_parse(match.group('ts')) + amount = int(match.group('amount')) + unit = match.group('unit') + if unit == 'd': + ts += timedelta(days=amount) + elif unit == 'm': + ts += timedelta(minutes=amount) + elif unit == 'h': + ts += timedelta(hours=amount) + elif unit == 's': + ts += timedelta(seconds=amount) + + return ts.timestamp() + + +def _group_testcases(testcases, group_by, extra_cols): + grouped = {} + for tc in testcases: + for pvar, reftuple in tc['perfvalues'].items(): + pvar = pvar.split(':')[-1] + pval, pref, plower, pupper, punit = reftuple + plower = pref * (1 + plower) if plower is not None else -math.inf + pupper = pref * (1 + pupper) if pupper is not None else math.inf + record = { + 'pvar': pvar, + 'pval': pval, + 'plower': plower, + 'pupper': pupper, + 'punit': punit, + **{k: tc[k] for k in group_by + extra_cols if k in tc} + } + key = _group_key(group_by, record) + grouped.setdefault(key, []) + grouped[key].append(record) + + return grouped + +def _aggregate_perf(grouped_testcases, aggr_fn, cols): + other_aggr = _JoinUniqueValues('|') + aggr_data = {} + for key, seq in grouped_testcases.items(): + aggr_data.setdefault(key, {}) + aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq) + for c in cols: + aggr_data[key][c] = other_aggr( + nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c] + for tc in seq + ) + + return aggr_data + +def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, + extra_group_by=None, extra_cols=None): + extra_group_by = extra_group_by or [] + extra_cols = extra_cols or [] + group_by = ['name', 'pvar', 'punit'] + extra_group_by + + grouped_base = _group_testcases(base_testcases, group_by, extra_cols) + grouped_target = _group_testcases(target_testcases, group_by, extra_cols) + pbase = _aggregate_perf(grouped_base, base_fn, extra_cols) + ptarget = _aggregate_perf(grouped_target, target_fn, []) + + # Build the final table data + data = [] + for key, aggr_data in pbase.items(): + pval = aggr_data['pval'] + try: + target_pval = ptarget[key]['pval'] + except KeyError: + pdiff = 'n/a' + else: + pdiff = (pval - target_pval) / target_pval + pdiff = '{:+7.2%}'.format(pdiff) + + name, pvar, punit, *extras = key + line = [name, pvar, pval, punit, pdiff, *extras] + # Add the extra columns + line += [aggr_data[c] for c in extra_cols] + data.append(line) + + return (data, ['name', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols) + +class _Aggregator: + @classmethod + def create(cls, name): + if name == 'first': + return _First() + elif name == 'last': + return _Last() + elif name == 'mean': + return _Mean() + elif name == 'median': + return _Median() + elif name == 'min': + return _Min() + elif name == 'max': + return _Max() + else: + raise ValueError(f'unknown aggregation function: {name!r}') + + + @abc.abstractmethod + def __call__(self, iterable): + pass + +class _First(_Aggregator): + def __call__(self, iterable): + for i, elem in enumerate(iterable): + if i == 0: + return elem + +class _Last(_Aggregator): + def __call__(self, iterable): + if not isinstance(iterable, types.GeneratorType): + return iterable[-1] + + for elem in iterable: + pass + + return elem + + +class _Mean(_Aggregator): + def __call__(self, iterable): + return statistics.mean(iterable) + + +class _Median(_Aggregator): + def __call__(self, iterable): + return statistics.median(iterable) + + +class _Min(_Aggregator): + def __call__(self, iterable): + return min(iterable) + + +class _Max(_Aggregator): + def __call__(self, iterable): + return max(iterable) + +class _JoinUniqueValues(_Aggregator): + def __init__(self, delim): + self.__delim = delim + + def __call__(self, iterable): + unique_vals = {str(elem) for elem in iterable} + return self.__delim.join(unique_vals) + +def performance_report_data(run_stats, report_spec): + period, aggr, cols = report_spec.split('/') + ts_start, ts_end = [parse_timestamp(ts) for ts in period.split(':')] + op, extra_groups = aggr.split(':') + aggr_fn = _Aggregator.create(op) + extra_groups = extra_groups.split('+')[1:] + extra_cols = cols.split('+')[1:] + testcases = run_stats[0]['testcases'] + target_testcases = _fetch_cases_time_period(ts_start, ts_end) + return compare_testcase_data(testcases, target_testcases, _First(), + aggr_fn, extra_groups, extra_cols) + + +def performance_compare_data(spec): + period_base, period_target, aggr, cols = spec.split('/') + base_testcases = _fetch_cases_time_period( + *(parse_timestamp(ts) for ts in period_base.split(':')) + ) + target_testcases = _fetch_cases_time_period( + *(parse_timestamp(ts) for ts in period_target.split(':')) + ) + op, extra_groups = aggr.split(':') + aggr_fn = _Aggregator.create(op) + extra_groups = extra_groups.split('+')[1:] + extra_cols = cols.split('+')[1:] + return compare_testcase_data(base_testcases, target_testcases, aggr_fn, + aggr_fn, extra_groups, extra_cols) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index c13c4409e7..4ad0a4e680 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -7,11 +7,13 @@ import itertools import os import shutil +import time import traceback import reframe.core.runtime as rt import reframe.core.exceptions as errors import reframe.utility as util +from reframe.core.logging import _format_time_rfc3339 from reframe.core.warnings import suppress_deprecations @@ -107,8 +109,10 @@ def json(self, force=False): num_aborted = 0 num_skipped = 0 for t in run: - check = t.check - partition = check.current_partition + # We take partition and environment from the test case and not + # from the check, since if the test fails before `setup()`, + # these are not set inside the check. + check, partition, environ = t.testcase entry = { 'build_jobid': None, 'build_stderr': None, @@ -121,14 +125,18 @@ def json(self, force=False): 'dependencies_conceptual': [ d[0] for d in t.check.user_deps() ], + 'environ': environ.name, 'fail_phase': None, 'fail_reason': None, 'filename': inspect.getfile(type(check)), 'fixture': check.is_fixture(), + 'job_completion_time': None, + 'job_completion_time_unix': None, 'job_stderr': None, 'job_stdout': None, - 'result': None, - 'scheduler': None, + 'partition': partition.name, + 'result': t.result, + 'scheduler': partition.scheduler.registered_name, 'time_compile': t.duration('compile_complete'), 'time_performance': t.duration('performance'), 'time_run': t.duration('run_complete'), @@ -136,15 +144,6 @@ def json(self, force=False): 'time_setup': t.duration('setup'), 'time_total': t.duration('total') } - - # We take partition and environment from the test case and not - # from the check, since if the test fails before `setup()`, - # these are not set inside the check. - partition = t.testcase.partition - environ = t.testcase.environ - entry['partition'] = partition.name - entry['environ'] = environ.name - entry['scheduler'] = partition.scheduler.registered_name if check.job: entry['job_stderr'] = check.stderr.evaluate() entry['job_stdout'] = check.stdout.evaluate() @@ -153,7 +152,6 @@ def json(self, force=False): entry['build_stderr'] = check.build_stderr.evaluate() entry['build_stdout'] = check.build_stdout.evaluate() - entry['result'] = t.result if t.failed: num_failures += 1 elif t.aborted: @@ -177,12 +175,22 @@ def json(self, force=False): # Add any loggable variables and parameters test_cls = type(check) for name, alt_name in test_cls.loggable_attrs(): + if alt_name == 'partition' or alt_name == 'environ': + # We set those from the testcase + continue + key = alt_name if alt_name else name try: entry[key] = _getattr(check, name) except AttributeError: entry[key] = '' + if entry['job_completion_time_unix']: + entry['job_completion_time'] = _format_time_rfc3339( + time.localtime(entry['job_completion_time_unix']), + '%FT%T%:z' + ) + testcases.append(entry) self._run_data.append({ @@ -221,15 +229,14 @@ def _head_n(filename, prefix, num_lines=10): def _print_failure_info(rec, runid, total_runs): printer.info(line_width * '-') - printer.info(f"FAILURE INFO for {rec['display_name']} " + printer.info(f"FAILURE INFO for {rec['name']} " f"(run: {runid}/{total_runs})") printer.info(f" * Description: {rec['descr']}") printer.info(f" * System partition: {rec['system']}") printer.info(f" * Environment: {rec['environ']}") printer.info(f" * Stage directory: {rec['stagedir']}") - printer.info( - f" * Node list: {util.nodelist_abbrev(rec['job_nodelist'])}" - ) + printer.info(f" * Node list: " + f"{util.nodelist_abbrev(rec['job_nodelist'])}") job_type = 'local' if rec['scheduler'] == 'local' else 'batch job' printer.info(f" * Job type: {job_type} (id={rec['jobid']})") printer.info(f" * Dependencies (conceptual): " @@ -269,7 +276,7 @@ def _print_failure_info(rec, runid, total_runs): continue for r in run_info['testcases']: - if r['result'] in {'success', 'aborted', 'skipped'}: + if r['result'] in {'pass', 'abort', 'skip'}: continue _print_failure_info(r, run_no, len(run_report)) @@ -353,11 +360,12 @@ def performance_report(self): for testcases in perf_records.values(): for tc in testcases: - name = tc['display_name'] - hash = tc['hash'] - env = tc['environment'] - part = tc['system'] - lines.append(f'[{name} /{hash} @{part}:{env}]') + name = tc['name'] + hash = tc['hashcode'] + env = tc['environ'] + system = tc['system'] + part = tc['partition'] + lines.append(f'[{name} /{hash} @{system}:{part}+{env}]') for v in interesting_vars: val = tc['check_vars'][v] if val is not None: diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index e989559476..bf60343235 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -22,6 +22,7 @@ "type": "array", "items": {"type": "string"} }, + "environ": {"type": ["string", "null"]}, "fail_info": { "type": ["object", "null"], "properties": { @@ -39,18 +40,29 @@ "fail_severe": {"type": "boolean"}, "filename": {"type": "string"}, "fixture": {"type": "boolean"}, + "job_completion_time": {"type": ["string", "null"]}, + "job_completion_time_unix": {"type": ["number", "null"]}, "job_stderr": {"type": ["string", "null"]}, "job_stdout": {"type": ["string", "null"]}, + "name": {"type": "string"}, + "outputdir": {"type": ["string", "null"]}, + "perfvalues": {"type": "object"}, + "partition": {"type": ["string", "null"]}, "result": {"type": "string"}, + "scheduler": {"type": "string"}, + "system": {"type": "string"}, "time_compile": {"type": ["number", "null"]}, "time_performance": {"type": ["number", "null"]}, "time_run": {"type": ["number", "null"]}, "time_sanity": {"type": ["number", "null"]}, "time_setup": {"type": ["number", "null"]}, - "time_total": {"type": ["number", "null"]} + "time_total": {"type": ["number", "null"]}, + "unique_name": {"type": "string"} }, - "required": ["name", "system", "partition", "environ", - "perfvalues", "result", "fail_reason", "file"] + "required": ["environ", "fail_phase", "fail_reason", "filename", + "job_completion_time_unix", "name", "perfvalues", + "partition", "result", "system", "time_total", + "unique_name"] } }, "type": "object", @@ -72,6 +84,7 @@ "num_cases": {"type": "number"}, "num_failures": {"type": "number"}, "num_aborted": {"type": "number"}, + "num_skipped": {"type": "number"}, "prefix_output": {"type": "string"}, "prefix_stage": {"type": "string"}, "time_elapsed": {"type": "number"}, @@ -81,7 +94,8 @@ "version": {"type": "string"}, "workdir": {"type": "string"} }, - "required": ["data_version"] + "required": ["data_version", "hostname", + "time_elapsed", "time_end", "time_start"] }, "restored_cases": { "type": "array", @@ -92,16 +106,17 @@ "items": { "type": "object", "properties": { + "num_aborted": {"type": "number"}, "num_cases": {"type": "number"}, "num_failures": {"type": "number"}, - "num_aborted": {"type": "number"}, - "runid": {"type": "number"}, + "num_skipped": {"type": "number"}, + "runid": {"typeid": "number"}, "testcases": { "type": "array", "items": {"$ref": "#/defs/testcase_type"} } }, - "required": ["testcases"] + "required": ["num_cases", "num_failures", "testcases"] } } }, diff --git a/requirements.txt b/requirements.txt index 0e9fc5409b..e01146fe10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,6 @@ semver==3.0.2; python_version >= '3.7' setuptools==59.6.0; python_version == '3.6' setuptools==68.0.0; python_version == '3.7' setuptools==72.1.0; python_version >= '3.8' +tabulate==0.9.0 wcwidth==0.2.13 #+pygelf%pygelf==0.4.0 diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 7fc8edd9fd..04fcaad50c 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -421,17 +421,20 @@ def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch): def test_performance_report(run_reframe, run_action): - returncode, stdout, _ = run_reframe( + returncode, stdout, stderr = run_reframe( checkpath=['unittests/resources/checks/frontend_checks.py'], more_options=['-n', '^PerformanceFailureCheck', '--performance-report'], action=run_action ) + if run_action == 'run': - assert r'PERFORMANCE REPORT' in stdout - assert r'perf: 10 Gflop/s' in stdout + assert returncode == 1 else: - assert r'PERFORMANCE REPORT' not in stdout + assert returncode == 0 + + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr def test_skip_system_check_option(run_reframe, run_action): diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 04e0edb78c..5d8efda6d8 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -291,7 +291,7 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): jsonext.dump(report, fp) with pytest.raises(ReframeError, - match=r'invalid report'): + match=r'failed to validate report'): runreport.load_report(tmp_path / 'invalid-version.json') @@ -391,7 +391,7 @@ def assert_reported_skipped(num_skipped): num_reported = 0 for tc in report[0]['testcases']: - if tc['result'] == 'skipped': + if tc['result'] == 'skip': num_reported += 1 assert num_reported == num_skipped diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 609efc8f68..cd1c3413d0 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -476,9 +476,9 @@ def test_prepare_nodes_option_minimal(make_exec_ctx, make_job, slurm_only): def test_submit(make_job, exec_ctx): minimal_job = make_job(sched_access=exec_ctx.access) prepare_job(minimal_job) - assert minimal_job.nodelist is None + assert minimal_job.nodelist == [] submit_job(minimal_job) - assert minimal_job.jobid is not None + assert minimal_job.jobid != [] minimal_job.wait() # Additional scheduler-specific checks From 71c777dbf8dc49d2ccd39420a2ea9d49e7e5ac66 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 18 Jun 2024 16:37:18 +0200 Subject: [PATCH 04/69] Refactor reporting code --- reframe/frontend/cli.py | 117 ++--- reframe/frontend/executors/__init__.py | 50 +- reframe/frontend/printer.py | 161 ++++++ reframe/frontend/reporting/__init__.py | 457 ++++++++++++++++++ .../{runreport.py => reporting/analytics.py} | 429 ++++------------ reframe/frontend/statistics.py | 396 --------------- unittests/test_cli.py | 8 +- unittests/test_policies.py | 122 ++--- 8 files changed, 877 insertions(+), 863 deletions(-) create mode 100644 reframe/frontend/reporting/__init__.py rename reframe/frontend/{runreport.py => reporting/analytics.py} (52%) delete mode 100644 reframe/frontend/statistics.py diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index de5529b1ad..fcd1972717 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -24,7 +24,8 @@ import reframe.frontend.ci as ci import reframe.frontend.dependencies as dependencies import reframe.frontend.filters as filters -import reframe.frontend.runreport as runreport +import reframe.frontend.reporting as reporting +import reframe.frontend.reporting.analytics as analytics import reframe.utility as util import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext @@ -932,18 +933,18 @@ def restrict_logging(): if options.restore_session: filenames = options.restore_session.split(',') else: - filenames = [runreport.next_report_filename( - osext.expandvars(site_config.get('general/0/report_file')), - new=False - )] + filenames = [ + osext.expandvars(site_config.get('general/0/report_file')) + ] try: - report = runreport.load_report(*filenames) + restored_session = reporting.restore_session(*filenames) except errors.ReframeError as err: printer.error(f'failed to load restore session: {err}') sys.exit(1) - check_search_path = list(report.slice('filename', unique=True)) + check_search_path = list(restored_session.slice('filename', + unique=True)) check_search_recursive = False # If `-c` or `-R` are passed explicitly outside the configuration @@ -954,9 +955,7 @@ def restrict_logging(): 'search path set explicitly in the command-line or ' 'the environment' ) - check_search_path = site_config.get( - 'general/0/check_search_path' - ) + check_search_path = site_config.get('general/0/check_search_path') if site_config.is_sticky_option('general/check_search_recursive'): printer.warning( @@ -998,20 +997,20 @@ def print_infoline(param, value): param = param + ':' printer.info(f" {param.ljust(18)} {value}") - session_info = { + report = reporting.RunReport() + report.update_session_info({ 'cmdline': ' '.join(shlex.quote(arg) for arg in sys.argv), 'config_files': rt.site_config.sources, - 'data_version': runreport.DATA_VERSION, - 'hostname': socket.gethostname(), 'log_files': logging.log_files(), 'prefix_output': rt.output_prefix, 'prefix_stage': rt.stage_prefix, 'user': osext.osuser(), 'version': osext.reframe_version(), 'workdir': os.getcwd(), - } + }) # Print command line + session_info = report['session_info'] printer.info(f"[ReFrame Setup]") print_infoline('version', session_info['version']) print_infoline('command', repr(session_info['cmdline'])) @@ -1134,7 +1133,7 @@ def print_infoline(param, value): sys.exit(1) def _case_failed(t): - rec = report.case(*t) + rec = restored_session.case(*t) if not rec: return False @@ -1233,7 +1232,7 @@ def _sort_testcases(testcases): printer.debug('Pruned test DAG') printer.debug(dependencies.format_deps(testgraph)) if options.restore_session is not None: - testgraph, restored_cases = report.restore_dangling(testgraph) + testgraph, restored_cases = restored_session.restore_dangling(testgraph) testcases = dependencies.toposort( testgraph, @@ -1291,10 +1290,10 @@ def _sort_testcases(testcases): sys.exit(0) if options.performance_compare: - data, header = runreport.performance_compare_data( + data, header = reporting.performance_compare( options.performance_compare ) - print(tabulate(data, headers=header, tablefmt='mixed_grid')) + printer(tabulate(data, headers=header, tablefmt='mixed_grid')) sys.exit(0) if not options.run and not options.dry_run: @@ -1436,34 +1435,39 @@ def module_unuse(*paths): ) runner.runall(testcases, restored_cases) finally: + # Build final JSON report time_end = time.time() - session_info['time_end'] = time.strftime( - '%FT%T%z', time.localtime(time_end) - ) - session_info['time_elapsed'] = time_end - time_start + report.update_session_info({ + 'time_end': time.strftime(r'%FT%T%z', + time.localtime(time_end)), + 'time_elapsed': time_end - time_start + }) + report.update_run_stats(runner.stats) + if options.restore_session is not None: + report.update_restored_cases(restored_cases, restored_session) # Print a retry report if we did any retries if options.max_retries and runner.stats.failed(run=0): - printer.info(runner.stats.retry_report()) + printer.retry_report(report) # Print a failure report if we had failures in the last run success = True if runner.stats.failed(): success = False - runner.stats.print_failure_report( - printer, not options.distribute, - options.duration or options.reruns + printer.failure_report( + report, + rerun_info=not options.distribute, + global_stats=options.duration or options.reruns ) if options.failure_stats: - runner.stats.print_failure_stats( - printer, options.duration or options.reruns + printer.failure_stats( + report, global_stats=options.duration or options.reruns ) if options.performance_report and not options.dry_run: try: - data, header = runreport.performance_report_data( - runner.stats.json(), - options.performance_report + data, header = reporting.performance_compare( + options.performance_report, report ) except errors.ReframeError as err: printer.warning( @@ -1483,49 +1487,28 @@ def module_unuse(*paths): if basedir: os.makedirs(basedir, exist_ok=True) - # Build final JSON report - run_stats = runner.stats.json() - session_info.update({ - 'num_cases': run_stats[0]['num_cases'], - 'num_failures': run_stats[-1]['num_failures'], - 'num_aborted': run_stats[-1]['num_aborted'], - 'num_skipped': run_stats[-1]['num_skipped'] - }) - json_report = { - 'session_info': session_info, - 'runs': run_stats, - 'restored_cases': [] - } - if options.restore_session is not None: - for c in restored_cases: - json_report['restored_cases'].append(report.case(*c)) - # Save the report file - report_file = runreport.next_report_filename(report_file) try: - runreport.save_report( - json_report, report_file, - rt.get_option('general/0/compress_report') + default_loc = os.path.dirname( + osext.expandvars(rt.get_default('general/report_file')) ) + report.save( + report_file, + compress=rt.get_option('general/0/compress_report'), + link_to_last=(default_loc == os.path.dirname(report_file)) + ) except OSError as e: printer.warning( f'failed to generate report in {report_file!r}: {e}' ) - else: - default_loc = os.path.dirname( - osext.expandvars(rt.get_default('general/report_file')) + except errors.ReframeError as e: + printer.warning( + f'failed to create symlink to latest report: {e}' ) - if default_loc == os.path.dirname(report_file): - try: - runreport.link_latest_report(report_file, 'latest.json') - except Exception as e: - printer.warning( - f'failed to create symlink to latest report: {e}' - ) - - # Index the generated report + + # Store the generated report for analytics try: - runreport.store_results(json_report, report_file) + analytics.store_report(report, report.filename) except Exception as e: printer.warning(f'failed to store results in the database: {e}') raise @@ -1535,10 +1518,8 @@ def module_unuse(*paths): if junit_report_file: # Expand variables in filename junit_report_file = osext.expandvars(junit_report_file) - junit_xml = runreport.junit_xml_report(json_report) try: - with open(junit_report_file, 'w') as fp: - runreport.junit_dump(junit_xml, fp) + report.save_junit(junit_report_file) except OSError as e: printer.warning( f'failed to generate report in {junit_report_file!r}: ' diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 6c741d6e61..90b81c74c2 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -6,6 +6,7 @@ import abc import contextlib import copy +import itertools import os import signal import sys @@ -24,15 +25,62 @@ ForceExitError, RunSessionTimeout, SkipTestError, + StatisticsError, TaskExit) from reframe.core.schedulers.local import LocalJobScheduler from reframe.frontend.printer import PrettyPrinter -from reframe.frontend.statistics import TestStats ABORT_REASONS = (AssertionError, FailureLimitError, KeyboardInterrupt, ForceExitError, RunSessionTimeout) +class TestStats: + '''Stores test case statistics.''' + + def __init__(self): + # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] + self._alltasks = [[]] + + def add_task(self, task): + current_run = runtime.runtime().current_run + if current_run == len(self._alltasks): + self._alltasks.append([]) + + self._alltasks[current_run].append(task) + + def runs(self): + for runid, tasks in enumerate(self._alltasks): + yield runid, tasks + + def tasks(self, run=-1): + if run is None: + yield from itertools.chain(*self._alltasks) + else: + try: + yield from self._alltasks[run] + except IndexError: + raise StatisticsError(f'no such run: {run}') from None + + def failed(self, run=-1): + return [t for t in self.tasks(run) if t.failed] + + def skipped(self, run=-1): + return [t for t in self.tasks(run) if t.skipped] + + def aborted(self, run=-1): + return [t for t in self.tasks(run) if t.aborted] + + def completed(self, run=-1): + return [t for t in self.tasks(run) if t.completed] + + def num_cases(self, run=-1): + return sum(1 for _ in self.tasks(run)) + + @property + def num_runs(self): + return len(self._alltasks) + + class TestCase: '''A combination of a regression check, a system partition and a programming environment. diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index 789889e3d3..b6530f05bb 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -3,9 +3,16 @@ # # SPDX-License-Identifier: BSD-3-Clause +import os +import shutil import time +import traceback import reframe.core.logging as logging +import reframe.core.runtime as rt import reframe.utility.color as color +from reframe.core.exceptions import SanityError +from reframe.frontend.reporting import format_testcase +from reframe.utility import nodelist_abbrev class PrettyPrinter: @@ -14,6 +21,8 @@ class PrettyPrinter: It takes care of formatting the progress output and adds some more cosmetics to specific levels of messages, such as warnings and errors. + It also takes care of formatting and printing the various reports. + The actual printing is delegated to an internal logger, which is responsible for printing. ''' @@ -84,3 +93,155 @@ def __setattr__(self, attr, value): self.__dict__['colorize'] = value else: super().__setattr__(attr, value) + + def failure_report(self, report, rerun_info=True, global_stats=False): + '''Print a failure report''' + + def _head_n(filename, prefix, num_lines=10): + # filename and prefix are `None` before setup + if filename is None or prefix is None: + return [] + + try: + with open(os.path.join(prefix, filename)) as fp: + lines = [ + f'--- {filename} (first {num_lines} lines) ---' + ] + for i, line in enumerate(fp): + if i < num_lines: + # Remove trailing '\n' + lines.append(line.rstrip()) + + lines += [f'--- {filename} ---'] + except OSError as e: + lines = [f'--- {filename} ({e}) ---'] + + return lines + + def _print_failure_info(rec, runid, total_runs): + self.info(line_width * '-') + self.info(f"FAILURE INFO for {rec['name']} " + f"(run: {runid}/{total_runs})") + self.info(f" * Description: {rec['descr']}") + self.info(f" * System partition: {rec['system']}") + self.info(f" * Environment: {rec['environ']}") + self.info(f" * Stage directory: {rec['stagedir']}") + self.info(f" * Node list: " + f"{nodelist_abbrev(rec['job_nodelist'])}") + job_type = 'local' if rec['scheduler'] == 'local' else 'batch job' + self.info(f" * Job type: {job_type} (id={rec['jobid']})") + self.info(f" * Dependencies (conceptual): " + f"{rec['dependencies_conceptual']}") + self.info(f" * Dependencies (actual): " + f"{rec['dependencies_actual']}") + self.info(f" * Maintainers: {rec['maintainers']}") + self.info(f" * Failing phase: {rec['fail_phase']}") + if rerun_info and not rec['fixture']: + self.info(f" * Rerun with '-n /{rec['hashcode']}" + f" -p {rec['environ']} --system " + f"{rec['system']} -r'") + + msg = rec['fail_reason'] + if isinstance(rec['fail_info']['exc_value'], SanityError): + lines = [msg] + lines += _head_n(rec['job_stdout'], prefix=rec['stagedir']) + lines += _head_n(rec['job_stderr'], prefix=rec['stagedir']) + msg = '\n'.join(lines) + + self.info(f" * Reason: {msg}") + + tb = ''.join(traceback.format_exception( + *rec['fail_info'].values())) + if rec['fail_severe']: + self.info(tb) + else: + self.verbose(tb) + + line_width = shutil.get_terminal_size()[0] + self.info(line_width * '=') + self.info('SUMMARY OF FAILURES') + + for run_no, run_info in enumerate(report['runs'], start=1): + if not global_stats and run_no != len(report['runs']): + continue + + for r in run_info['testcases']: + if r['result'] in {'pass', 'abort', 'skip'}: + continue + + _print_failure_info(r, run_no, len(report['runs'])) + + self.info(line_width * '-') + + def failure_stats(self, report, global_stats=False): + current_run = rt.runtime().current_run + failures = {} + for runid, run_data in enumerate(report['runs']): + if not global_stats and runid != current_run: + continue + + for tc in run_data['testcases']: + info = f'{tc["display_name"]}' + info += f' @{tc["system"]}:{tc["partition"]}+{tc["environ"]}' + + failed_stage = tc['fail_phase'] + failures.setdefault(failed_stage, []) + failures[failed_stage].append(info) + + line_width = shutil.get_terminal_size()[0] + stats_start = line_width * '=' + stats_title = 'FAILURE STATISTICS' + stats_end = line_width * '-' + stats_body = [] + row_format = "{:<13} {:<5} {}" + stats_hline = row_format.format(13*'-', 5*'-', 60*'-') + stats_header = row_format.format('Phase', '#', 'Failing test cases') + if global_stats: + num_tests = report['session_info']['num_cases'] + else: + num_tests = report['runs'][current_run]['num_cases'] + + num_failures = 0 + for fl in failures.values(): + num_failures += len(fl) + + stats_body = [''] + stats_body.append(f'Total number of test cases: {num_tests}') + stats_body.append(f'Total number of failures: {num_failures}') + stats_body.append('') + stats_body.append(stats_header) + stats_body.append(stats_hline) + for p, l in failures.items(): + stats_body.append(row_format.format(p, len(l), l[0])) + for f in l[1:]: + stats_body.append(row_format.format('', '', str(f))) + + if stats_body: + for line in (stats_start, stats_title, *stats_body, stats_end): + self.info(line) + + def retry_report(self, report): + '''Print a report for test retries''' + + if not rt.runtime().current_run: + # Do nothing if no retries + return + + line_width = shutil.get_terminal_size()[0] + lines = [line_width * '='] + lines.append('SUMMARY OF RETRIES') + lines.append(line_width * '-') + messages = {} + for i, run in enumerate(report['runs'][1:], start=1): + for tc in run['testcases']: + # Overwrite entry from previous run if available + tc_info = format_testcase(tc) + messages[tc_info] = ( + f" * Test {tc_info} was retried {i} time(s) and" + f" {'failed' if tc['result'] == 'fail' else 'passed'}." + ) + + for msg in sorted(messages): + lines.append(msg) + + self.info('\n'.join(lines)) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py new file mode 100644 index 0000000000..97f1203949 --- /dev/null +++ b/reframe/frontend/reporting/__init__.py @@ -0,0 +1,457 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import decimal +import functools +import inspect +import json +import jsonschema +import lxml.etree as etree +import os +import re +import socket +import time +from datetime import datetime + +import reframe as rfm +import reframe.utility.jsonext as jsonext +import reframe.utility.osext as osext +from reframe.core.exceptions import ReframeError, what, is_severe +from reframe.core.logging import getlogger, _format_time_rfc3339 +from reframe.core.warnings import suppress_deprecations +from .analytics import (parse_cmp_spec, + fetch_testcases_time_period, + compare_testcase_data) + + +# The schema data version +# Major version bumps are expected to break the validation of previous schemas + +DATA_VERSION = '4.0' +_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') + + +def format_testcase(json, name='unique_name'): + '''Format test case from its json representation''' + name = json[name] + system = json['system'] + partition = json['partition'] + environ = json['environ'] + return f'{name}@{system}:{partition}+{environ}' + + +class _RestoredSessionInfo: + '''A restored session with some additional functionality.''' + + def __init__(self, report): + self._report = report + self._fallbacks = [] # fallback reports + + # Index all runs by test case; if a test case has run multiple times, + # only the last time will be indexed + self._cases_index = {} + for run in self._report['runs']: + for tc in run['testcases']: + self._cases_index[format_testcase(tc)] = tc + + # Index also the restored cases + for tc in self._report['restored_cases']: + self._cases_index[format_testcase(tc)] = tc + + def __getitem__(self, key): + return self._report[key] + + def __getattr__(self, name): + with suppress_deprecations(): + return getattr(self._report, name) + + def add_fallback(self, report): + self._fallbacks.append(report) + + def slice(self, prop, when=None, unique=False): + '''Slice the report on property ``prop``.''' + + if unique: + returned = set() + + for tc in self._report['runs'][-1]['testcases']: + val = tc[prop] + if unique and val in returned: + continue + + if when is None: + if unique: + returned.add(val) + + yield val + elif tc[when[0]] == when[1]: + if unique: + returned.add(val) + + yield val + + def case(self, check, part, env): + key = f'{check.unique_name}@{part.fullname}+{env.name}' + ret = self._cases_index.get(key) + if ret is None: + # Look up the case in the fallback reports + for rpt in self._fallbacks: + ret = rpt._cases_index.get(key) + if ret is not None: + break + + return ret + + def restore_dangling(self, graph): + '''Restore dangling dependencies in graph from the report data. + + Returns the updated graph. + ''' + + restored = [] + for tc, deps in graph.items(): + for d in deps: + if d not in graph: + restored.append(d) + self._do_restore(d) + + return graph, restored + + def _do_restore(self, testcase): + tc = self.case(*testcase) + if tc is None: + raise ReframeError( + f'could not restore testcase {testcase!r}: ' + f'not found in the report files' + ) + + dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json') + try: + with open(dump_file) as fp: + testcase._check = jsonext.load(fp) + except (OSError, json.JSONDecodeError) as e: + raise ReframeError( + f'could not restore testcase {testcase!r}') from e + + +def _expand_report_filename(filepatt, *, newfile): + if '{sessionid}' not in os.fspath(filepatt): + return filepatt + + search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)') + new_id = -1 + basedir = os.path.dirname(filepatt) or '.' + for filename in os.listdir(basedir): + match = re.match(search_patt, filename) + if match: + found_id = int(match.group(1)) + new_id = max(found_id, new_id) + + if newfile: + new_id += 1 + + return filepatt.format(sessionid=new_id) + + +def _restore_session(filename): + filename = _expand_report_filename(filename, newfile=False) + try: + with open(filename) as fp: + report = json.load(fp) + except OSError as e: + raise ReframeError( + f'failed to load report file {filename!r}') from e + except json.JSONDecodeError as e: + raise ReframeError( + f'report file {filename!r} is not a valid JSON file') from e + + # Validate the report + with open(_SCHEMA) as fp: + schema = json.load(fp) + + try: + jsonschema.validate(report, schema) + except jsonschema.ValidationError as e: + try: + found_ver = report['session_info']['data_version'] + except KeyError: + found_ver = 'n/a' + + getlogger().verbose(f'JSON validation error: {e}') + raise ReframeError( + f'failed to validate report {filename!r}: {e.args[0]} ' + f'(check report data version: required {DATA_VERSION}, ' + f'found: {found_ver})' + ) from None + + return _RestoredSessionInfo(report) + + +def restore_session(*filenames): + primary = filenames[0] + restored = _restore_session(primary) + + # Add fallback reports + for f in filenames[1:]: + restored.add_fallback(_restore_session(f)) + + return restored + + +class RunReport: + '''Internal representation of a run report + + This class provides direct access to the underlying report and provides + convenience functions for constructing a new report. + ''' + def __init__(self): + # Initialize the report with the required fields + now = datetime.now().strftime(r'%FT%T%z') + self.__report = { + 'session_info': { + 'data_version': DATA_VERSION, + 'hostname': socket.gethostname(), + 'time_start': now, + 'time_end': now, + 'time_elapsed': 0.0 + }, + 'runs': [], + 'restored_cases': [] + } + self.__filename = None + + @property + def filename(self): + return self.__filename + + def __getattr__(self, name): + return getattr(self.__report, name) + + def __getitem__(self, key): + return self.__report[key] + + def __rfm_json_encode__(self): + return self.__report + + def update_session_info(self, session_info): + self.__report['session_info'].update(session_info) + + def update_restored_cases(self, restored_cases, restored_session): + self.__report['restored_cases'] = [restored_session.case(*c) + for c in restored_cases] + + def update_run_stats(self, stats): + for runid, tasks in stats.runs(): + testcases = [] + num_failures = 0 + num_aborted = 0 + num_skipped = 0 + for t in tasks: + # We take partition and environment from the test case and not + # from the check, since if the test fails before `setup()`, + # these are not set inside the check. + check, partition, environ = t.testcase + entry = { + 'build_jobid': None, + 'build_stderr': None, + 'build_stdout': None, + 'dependencies_actual': [ + (d.check.unique_name, + d.partition.fullname, d.environ.name) + for d in t.testcase.deps + ], + 'dependencies_conceptual': [ + d[0] for d in t.check.user_deps() + ], + 'environ': environ.name, + 'fail_phase': None, + 'fail_reason': None, + 'filename': inspect.getfile(type(check)), + 'fixture': check.is_fixture(), + 'job_completion_time': None, + 'job_completion_time_unix': None, + 'job_stderr': None, + 'job_stdout': None, + 'partition': partition.name, + 'result': t.result, + 'scheduler': partition.scheduler.registered_name, + 'time_compile': t.duration('compile_complete'), + 'time_performance': t.duration('performance'), + 'time_run': t.duration('run_complete'), + 'time_sanity': t.duration('sanity'), + 'time_setup': t.duration('setup'), + 'time_total': t.duration('total') + } + if check.job: + entry['job_stderr'] = check.stderr.evaluate() + entry['job_stdout'] = check.stdout.evaluate() + + if check.build_job: + entry['build_stderr'] = check.build_stderr.evaluate() + entry['build_stdout'] = check.build_stdout.evaluate() + + if t.failed: + num_failures += 1 + elif t.aborted: + num_aborted += 1 + elif t.skipped: + num_skipped += 1 + + if t.failed or t.aborted: + entry['fail_phase'] = t.failed_stage + if t.exc_info is not None: + entry['fail_reason'] = what(*t.exc_info) + entry['fail_info'] = { + 'exc_type': t.exc_info[0], + 'exc_value': t.exc_info[1], + 'traceback': t.exc_info[2] + } + entry['fail_severe'] = is_severe(*t.exc_info) + elif t.succeeded: + entry['outputdir'] = check.outputdir + + # Add any loggable variables and parameters + test_cls = type(check) + for name, alt_name in test_cls.loggable_attrs(): + if alt_name == 'partition' or alt_name == 'environ': + # We set those from the testcase + continue + + key = alt_name if alt_name else name + try: + with suppress_deprecations(): + entry[key] = getattr(check, name) + except AttributeError: + entry[key] = '' + + if entry['job_completion_time_unix']: + entry['job_completion_time'] = _format_time_rfc3339( + time.localtime(entry['job_completion_time_unix']), + '%FT%T%:z' + ) + + testcases.append(entry) + + self.__report['runs'].append({ + 'num_cases': len(tasks), + 'num_failures': num_failures, + 'num_aborted': num_aborted, + 'num_skipped': num_skipped, + 'runid': runid, + 'testcases': testcases + }) + + # Update session info from stats + self.__report['session_info'].update({ + 'num_cases': self.__report['runs'][0]['num_cases'], + 'num_failures': self.__report['runs'][-1]['num_failures'], + 'num_aborted': self.__report['runs'][-1]['num_aborted'], + 'num_skipped': self.__report['runs'][-1]['num_skipped'] + }) + + def save(self, filename, compress=False, link_to_last=True): + filename = _expand_report_filename(filename, newfile=True) + with open(filename, 'w') as fp: + if compress: + jsonext.dump(self.__report, fp) + else: + jsonext.dump(self.__report, fp, indent=2) + fp.write('\n') + + self.__filename = filename + if not link_to_last: + return + + link_name = 'latest.json' + prefix, target_name = os.path.split(filename) + with osext.change_dir(prefix): + create_symlink = functools.partial(os.symlink, + target_name, link_name) + if not os.path.exists(link_name): + create_symlink() + else: + if os.path.islink(link_name): + os.remove(link_name) + create_symlink() + else: + raise ReframeError('path exists and is not a symlink') + + def generate_xml_report(self): + '''Generate a JUnit report from a standard ReFrame JSON report.''' + + report = self.__report + xml_testsuites = etree.Element('testsuites') + for run_id, rfm_run in enumerate(report['runs']): + xml_testsuite = etree.SubElement( + xml_testsuites, 'testsuite', + attrib={ + 'errors': '0', + 'failures': str(rfm_run['num_failures']), + 'hostname': report['session_info']['hostname'], + 'id': str(run_id), + 'name': f'ReFrame run {run_id}', + 'package': 'reframe', + 'tests': str(rfm_run['num_cases']), + 'time': str(report['session_info']['time_elapsed']), + # XSD schema does not like the timezone format, + # so we remove it + 'timestamp': report['session_info']['time_start'][:-5], + } + ) + etree.SubElement(xml_testsuite, 'properties') + for tc in rfm_run['testcases']: + casename = f'{format_testcase(tc, name="name")}' + testcase = etree.SubElement( + xml_testsuite, 'testcase', + attrib={ + 'classname': tc['filename'], + 'name': casename, + + # XSD schema does not like the exponential format and + # since we do not want to impose a fixed width, we pass + # it to `Decimal` to format it automatically. + 'time': str(decimal.Decimal(tc['time_total'] or 0)), + } + ) + if tc['result'] == 'fail': + testcase_msg = etree.SubElement( + testcase, 'failure', attrib={'type': 'failure', + 'message': tc['fail_phase']} + ) + testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}" + + testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out') + testsuite_stdout.text = '' + testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err') + testsuite_stderr.text = '' + + return xml_testsuites + + def save_junit(self, filename): + with open(filename, 'w') as fp: + xml = self.generate_xml_report() + fp.write( + etree.tostring(xml, encoding='utf8', pretty_print=True, + method='xml', xml_declaration=True).decode() + ) + +def performance_compare(cmp, report=None): + match = parse_cmp_spec(cmp) + if match.period_base is None: + if report is None: + raise ValueError('report cannot be `None` ' + 'for current run comparisons') + try: + tcs_base = report['runs'][0]['testcases'] + except IndexError: + tcs_base = [] + + else: + tcs_base = fetch_testcases_time_period(*match.period_base) + + tcs_target = fetch_testcases_time_period(*match.period_target) + return compare_testcase_data(tcs_base, tcs_target, match.aggregator, + match.aggregator, match.extra_groups, + match.extra_cols) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/reporting/analytics.py similarity index 52% rename from reframe/frontend/runreport.py rename to reframe/frontend/reporting/analytics.py index c1de31cf7f..bdba66c21e 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/reporting/analytics.py @@ -4,286 +4,23 @@ # SPDX-License-Identifier: BSD-3-Clause import abc -import decimal -import functools -import glob -import json -import jsonschema -import lxml.etree as etree -import math import os +import math import re import sqlite3 import statistics import types +from collections import namedtuple from collections.abc import Hashable from datetime import datetime, timedelta -import reframe as rfm import reframe.core.runtime as runtime import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext -from reframe.core.exceptions import ReframeError from reframe.core.logging import getlogger -from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev -# The schema data version -# Major version bumps are expected to break the validation of previous schemas - -DATA_VERSION = '4.0' -_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') - -def _tc_info(tc_entry, name='unique_name'): - name = tc_entry[name] - system = tc_entry['system'] - partition = tc_entry['partition'] - environ = tc_entry['environ'] - return f'{name}@{system}:{partition}+{environ}' - - -class _RunReport: - '''A wrapper to the run report providing some additional functionality''' - - def __init__(self, report): - self._report = report - self._fallbacks = [] # fallback reports - - # Index all runs by test case; if a test case has run multiple times, - # only the last time will be indexed - self._cases_index = {} - for run in self._report['runs']: - for tc in run['testcases']: - self._cases_index[_tc_info(tc)] = tc - - # Index also the restored cases - for tc in self._report['restored_cases']: - self._cases_index[_tc_info(tc)] = tc - - def __getitem__(self, key): - return self._report[key] - - def __getattr__(self, name): - with suppress_deprecations(): - return getattr(self._report, name) - - def add_fallback(self, report): - self._fallbacks.append(report) - - def slice(self, prop, when=None, unique=False): - '''Slice the report on property ``prop``.''' - - if unique: - returned = set() - - for tc in self._report['runs'][-1]['testcases']: - val = tc[prop] - if unique and val in returned: - continue - - if when is None: - if unique: - returned.add(val) - - yield val - elif tc[when[0]] == when[1]: - if unique: - returned.add(val) - - yield val - - def case(self, check, part, env): - key = f'{check.unique_name}@{part.fullname}+{env.name}' - ret = self._cases_index.get(key) - if ret is None: - # Look up the case in the fallback reports - for rpt in self._fallbacks: - ret = rpt._cases_index.get(key) - if ret is not None: - break - - return ret - - def restore_dangling(self, graph): - '''Restore dangling dependencies in graph from the report data. - - Returns the updated graph. - ''' - - restored = [] - for tc, deps in graph.items(): - for d in deps: - if d not in graph: - restored.append(d) - self._do_restore(d) - - return graph, restored - - def _do_restore(self, testcase): - tc = self.case(*testcase) - if tc is None: - raise ReframeError( - f'could not restore testcase {testcase!r}: ' - f'not found in the report files' - ) - - dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json') - try: - with open(dump_file) as fp: - testcase._check = jsonext.load(fp) - except (OSError, json.JSONDecodeError) as e: - raise ReframeError( - f'could not restore testcase {testcase!r}') from e - - -def next_report_filename(filepatt, new=True): - if '{sessionid}' not in filepatt: - return filepatt - - search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)') - new_id = -1 - basedir = os.path.dirname(filepatt) or '.' - for filename in os.listdir(basedir): - match = re.match(search_patt, filename) - if match: - found_id = int(match.group(1)) - new_id = max(found_id, new_id) - - if new: - new_id += 1 - - return filepatt.format(sessionid=new_id) - - -def _load_report(filename): - try: - with open(filename) as fp: - report = json.load(fp) - except OSError as e: - raise ReframeError( - f'failed to load report file {filename!r}') from e - except json.JSONDecodeError as e: - raise ReframeError( - f'report file {filename!r} is not a valid JSON file') from e - - # Validate the report - with open(_SCHEMA) as fp: - schema = json.load(fp) - - try: - jsonschema.validate(report, schema) - except jsonschema.ValidationError as e: - try: - found_ver = report['session_info']['data_version'] - except KeyError: - found_ver = 'n/a' - - getlogger().verbose(f'JSON validation error: {e}') - raise ReframeError( - f'failed to validate report {filename!r}: {e.args[0]} ' - f'(check report data version: required {DATA_VERSION}, ' - f'found: {found_ver})' - ) from None - - return _RunReport(report) - - -def load_report(*filenames): - primary = filenames[0] - rpt = _load_report(primary) - - # Add fallback reports - for f in filenames[1:]: - rpt.add_fallback(_load_report(f)) - - return rpt - - -def save_report(report, filename, compress=False): - with open(filename, 'w') as fp: - if compress: - jsonext.dump(report, fp) - else: - jsonext.dump(report, fp, indent=2) - fp.write('\n') - -def link_latest_report(filename, link_name): - prefix, target_name = os.path.split(filename) - with osext.change_dir(prefix): - create_symlink = functools.partial(os.symlink, target_name, link_name) - if not os.path.exists(link_name): - create_symlink() - else: - if os.path.islink(link_name): - os.remove(link_name) - create_symlink() - else: - raise ReframeError('path exists and is not a symlink') - - -def junit_xml_report(json_report): - '''Generate a JUnit report from a standard ReFrame JSON report.''' - - xml_testsuites = etree.Element('testsuites') - for run_id, rfm_run in enumerate(json_report['runs']): - xml_testsuite = etree.SubElement( - xml_testsuites, 'testsuite', - attrib={ - 'errors': '0', - 'failures': str(rfm_run['num_failures']), - 'hostname': json_report['session_info']['hostname'], - 'id': str(run_id), - 'name': f'ReFrame run {run_id}', - 'package': 'reframe', - 'tests': str(rfm_run['num_cases']), - 'time': str(json_report['session_info']['time_elapsed']), - # XSD schema does not like the timezone format, so we remove it - 'timestamp': json_report['session_info']['time_start'][:-5], - } - ) - etree.SubElement(xml_testsuite, 'properties') - for tc in rfm_run['testcases']: - casename = f'{_tc_info(tc, name="name")}' - testcase = etree.SubElement( - xml_testsuite, 'testcase', - attrib={ - 'classname': tc['filename'], - 'name': casename, - - # XSD schema does not like the exponential format and since - # we do not want to impose a fixed width, we pass it to - # `Decimal` to format it automatically. - 'time': str(decimal.Decimal(tc['time_total'] or 0)), - } - ) - if tc['result'] == 'fail': - testcase_msg = etree.SubElement( - testcase, 'failure', attrib={'type': 'failure', - 'message': tc['fail_phase']} - ) - testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}" - - testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out') - testsuite_stdout.text = '' - testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err') - testsuite_stderr.text = '' - - return xml_testsuites - - -def junit_dump(xml, fp): - fp.write( - etree.tostring(xml, encoding='utf8', pretty_print=True, - method='xml', xml_declaration=True).decode() - ) - - -def get_reports_files(directory): - return [f for f in glob.glob(f"{directory}/*") - if os.path.isfile(f) and not f.endswith('/latest.json')] - - def _db_file(): site_config = runtime.runtime().site_config prefix = os.path.dirname(osext.expandvars( @@ -301,11 +38,6 @@ def _db_file(): return filename -def store_results(report, report_file): - with sqlite3.connect(_db_file()) as conn: - _db_store_report(conn, report, report_file) - - def _db_create(filename): with sqlite3.connect(filename) as conn: conn.execute( @@ -331,7 +63,6 @@ def _db_create(filename): def _db_store_report(conn, report, report_file_path): - session_start = report['session_info']['time_start'] for run_idx, run in enumerate(report['runs']): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] @@ -359,7 +90,12 @@ def _db_store_report(conn, report, report_file_path): ) -def _fetch_cases_raw(condition): +def store_report(report, report_file): + with sqlite3.connect(_db_file()) as conn: + _db_store_report(conn, report, report_file) + + +def _fetch_testcases_raw(condition): with sqlite3.connect(_db_file()) as conn: query = (f'SELECT session_id, run_index, test_index, json_blob FROM ' f'testcases JOIN sessions ON session_id==id ' @@ -371,14 +107,14 @@ def _fetch_cases_raw(condition): testcases = [] sessions = {} for session_id, run_index, test_index, json_blob in results: - report = json.loads(sessions.setdefault(session_id, json_blob)) + report = jsonext.loads(sessions.setdefault(session_id, json_blob)) testcases.append(report['runs'][run_index]['testcases'][test_index]) return testcases -def _fetch_cases_time_period(ts_start, ts_end): - return _fetch_cases_raw( +def fetch_testcases_time_period(ts_start, ts_end): + return _fetch_testcases_raw( f'(job_completion_time_unix >= {ts_start} AND ' f'job_completion_time_unix < {ts_end}) ' 'ORDER BY job_completion_time_unix' @@ -400,46 +136,6 @@ def _group_key(groups, testcase): return tuple(key) -def parse_timestamp(s): - now = datetime.now() - def _do_parse(s): - if s == 'now': - return now - - formats = [r'%Y%m%d', r'%Y%m%dT%H%M', - r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] - for fmt in formats: - try: - return datetime.strptime(s, fmt) - except ValueError: - continue - - raise ValueError(f'invalid timestamp: {s}') - - - try: - ts = _do_parse(s) - except ValueError as err: - # Try the relative timestamps - match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) - if not match: - raise err - - ts = _do_parse(match.group('ts')) - amount = int(match.group('amount')) - unit = match.group('unit') - if unit == 'd': - ts += timedelta(days=amount) - elif unit == 'm': - ts += timedelta(minutes=amount) - elif unit == 'h': - ts += timedelta(hours=amount) - elif unit == 's': - ts += timedelta(seconds=amount) - - return ts.timestamp() - - def _group_testcases(testcases, group_by, extra_cols): grouped = {} for tc in testcases: @@ -525,7 +221,6 @@ def create(cls, name): else: raise ValueError(f'unknown aggregation function: {name!r}') - @abc.abstractmethod def __call__(self, iterable): pass @@ -574,26 +269,114 @@ def __call__(self, iterable): unique_vals = {str(elem) for elem in iterable} return self.__delim.join(unique_vals) + +def _parse_timestamp(s): + now = datetime.now() + def _do_parse(s): + if s == 'now': + return now + + formats = [r'%Y%m%d', r'%Y%m%dT%H%M', + r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] + for fmt in formats: + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + + raise ValueError(f'invalid timestamp: {s}') + + try: + ts = _do_parse(s) + except ValueError as err: + # Try the relative timestamps + match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) + if not match: + raise err + + ts = _do_parse(match.group('ts')) + amount = int(match.group('amount')) + unit = match.group('unit') + if unit == 'd': + ts += timedelta(days=amount) + elif unit == 'm': + ts += timedelta(minutes=amount) + elif unit == 'h': + ts += timedelta(hours=amount) + elif unit == 's': + ts += timedelta(seconds=amount) + + return ts.timestamp() + +def _parse_time_period(s): + try: + ts_start, ts_end = s.split(':') + except ValueError: + raise ValueError(f'invalid time period spec: {s}') from None + + return _parse_timestamp(ts_start), _parse_timestamp(ts_end) + +def _parse_extra_cols(s): + try: + extra_cols = s.split('+')[1:] + except (ValueError, IndexError): + raise ValueError(f'invalid extra groups spec: {s}') from None + + return extra_cols + + +def _parse_aggregation(s): + try: + op, extra_groups = s.split(':') + except ValueError: + raise ValueError(f'invalid aggregate function spec: {s}') from None + + return _Aggregator.create(op), _parse_extra_cols(extra_groups) + + +_Match = namedtuple('_Match', ['period_base', 'period_target', + 'aggregator', 'extra_groups', 'extra_cols']) + +def parse_cmp_spec(spec): + parts = spec.split('/') + if len(parts) == 3: + period_base, period_target, aggr, cols = None, *parts + elif len(parts) == 4: + period_base, period_target, aggr, cols = parts + else: + raise ValueError(f'invalid cmp spec: {spec}') + + if period_base is not None: + period_base = _parse_time_period(period_base) + + period_target = _parse_time_period(period_target) + aggr_fn, extra_groups = _parse_aggregation(aggr) + extra_cols = _parse_extra_cols(cols) + return _Match(period_base, period_target, + aggr_fn, extra_groups, extra_cols) + + def performance_report_data(run_stats, report_spec): period, aggr, cols = report_spec.split('/') - ts_start, ts_end = [parse_timestamp(ts) for ts in period.split(':')] + ts_start, ts_end = [_parse_timestamp(ts) for ts in period.split(':')] op, extra_groups = aggr.split(':') aggr_fn = _Aggregator.create(op) extra_groups = extra_groups.split('+')[1:] extra_cols = cols.split('+')[1:] testcases = run_stats[0]['testcases'] - target_testcases = _fetch_cases_time_period(ts_start, ts_end) - return compare_testcase_data(testcases, target_testcases, _First(), + target_testcases = fetch_testcases_time_period(ts_start, ts_end) + return compare_testcase_data(testcases, target_testcases, + _Aggregator.create('first'), aggr_fn, extra_groups, extra_cols) def performance_compare_data(spec): period_base, period_target, aggr, cols = spec.split('/') - base_testcases = _fetch_cases_time_period( - *(parse_timestamp(ts) for ts in period_base.split(':')) + base_testcases = fetch_testcases_time_period( + *(_parse_timestamp(ts) for ts in period_base.split(':')) ) - target_testcases = _fetch_cases_time_period( - *(parse_timestamp(ts) for ts in period_target.split(':')) + target_testcases = fetch_testcases_time_period( + *(_parse_timestamp(ts) for ts in period_target.split(':')) ) op, extra_groups = aggr.split(':') aggr_fn = _Aggregator.create(op) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py deleted file mode 100644 index 4ad0a4e680..0000000000 --- a/reframe/frontend/statistics.py +++ /dev/null @@ -1,396 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import inspect -import itertools -import os -import shutil -import time -import traceback - -import reframe.core.runtime as rt -import reframe.core.exceptions as errors -import reframe.utility as util -from reframe.core.logging import _format_time_rfc3339 -from reframe.core.warnings import suppress_deprecations - - -def _getattr(obj, attr): - with suppress_deprecations(): - return getattr(obj, attr) - - -class TestStats: - '''Stores test case statistics.''' - - def __init__(self): - # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] - self._alltasks = [[]] - - # Data collected for all the runs of this session in JSON format - self._run_data = [] - - def add_task(self, task): - current_run = rt.runtime().current_run - if current_run == len(self._alltasks): - self._alltasks.append([]) - - self._alltasks[current_run].append(task) - - def tasks(self, run=-1): - if run is None: - yield from itertools.chain(*self._alltasks) - else: - try: - yield from self._alltasks[run] - except IndexError: - raise errors.StatisticsError(f'no such run: {run}') from None - - def failed(self, run=-1): - return [t for t in self.tasks(run) if t.failed] - - def skipped(self, run=-1): - return [t for t in self.tasks(run) if t.skipped] - - def aborted(self, run=-1): - return [t for t in self.tasks(run) if t.aborted] - - def completed(self, run=-1): - return [t for t in self.tasks(run) if t.completed] - - def num_cases(self, run=-1): - return sum(1 for _ in self.tasks(run)) - - @property - def num_runs(self): - return len(self._alltasks) - - def retry_report(self): - # Return an empty report if no retries were done. - if not rt.runtime().current_run: - return '' - - line_width = shutil.get_terminal_size()[0] - report = [line_width * '='] - report.append('SUMMARY OF RETRIES') - report.append(line_width * '-') - messages = {} - for run in range(1, len(self._alltasks)): - for t in self.tasks(run): - partition_name = '' - environ_name = '' - if t.check.current_partition: - partition_name = t.check.current_partition.fullname - - if t.check.current_environ: - environ_name = t.check.current_environ.name - - # Overwrite entry from previous run if available - key = f"{t.check.unique_name}:{partition_name}:{environ_name}" - messages[key] = ( - f" * Test {t.check.info()} was retried {run} time(s) and " - f"{'failed' if t.failed else 'passed'}." - ) - - for key in sorted(messages.keys()): - report.append(messages[key]) - - return '\n'.join(report) - - def json(self, force=False): - if not force and self._run_data: - return self._run_data - - for runid, run in enumerate(self._alltasks): - testcases = [] - num_failures = 0 - num_aborted = 0 - num_skipped = 0 - for t in run: - # We take partition and environment from the test case and not - # from the check, since if the test fails before `setup()`, - # these are not set inside the check. - check, partition, environ = t.testcase - entry = { - 'build_jobid': None, - 'build_stderr': None, - 'build_stdout': None, - 'dependencies_actual': [ - (d.check.unique_name, - d.partition.fullname, d.environ.name) - for d in t.testcase.deps - ], - 'dependencies_conceptual': [ - d[0] for d in t.check.user_deps() - ], - 'environ': environ.name, - 'fail_phase': None, - 'fail_reason': None, - 'filename': inspect.getfile(type(check)), - 'fixture': check.is_fixture(), - 'job_completion_time': None, - 'job_completion_time_unix': None, - 'job_stderr': None, - 'job_stdout': None, - 'partition': partition.name, - 'result': t.result, - 'scheduler': partition.scheduler.registered_name, - 'time_compile': t.duration('compile_complete'), - 'time_performance': t.duration('performance'), - 'time_run': t.duration('run_complete'), - 'time_sanity': t.duration('sanity'), - 'time_setup': t.duration('setup'), - 'time_total': t.duration('total') - } - if check.job: - entry['job_stderr'] = check.stderr.evaluate() - entry['job_stdout'] = check.stdout.evaluate() - - if check.build_job: - entry['build_stderr'] = check.build_stderr.evaluate() - entry['build_stdout'] = check.build_stdout.evaluate() - - if t.failed: - num_failures += 1 - elif t.aborted: - num_aborted += 1 - elif t.skipped: - num_skipped += 1 - - if t.failed or t.aborted: - entry['fail_phase'] = t.failed_stage - if t.exc_info is not None: - entry['fail_reason'] = errors.what(*t.exc_info) - entry['fail_info'] = { - 'exc_type': t.exc_info[0], - 'exc_value': t.exc_info[1], - 'traceback': t.exc_info[2] - } - entry['fail_severe'] = errors.is_severe(*t.exc_info) - elif t.succeeded: - entry['outputdir'] = check.outputdir - - # Add any loggable variables and parameters - test_cls = type(check) - for name, alt_name in test_cls.loggable_attrs(): - if alt_name == 'partition' or alt_name == 'environ': - # We set those from the testcase - continue - - key = alt_name if alt_name else name - try: - entry[key] = _getattr(check, name) - except AttributeError: - entry[key] = '' - - if entry['job_completion_time_unix']: - entry['job_completion_time'] = _format_time_rfc3339( - time.localtime(entry['job_completion_time_unix']), - '%FT%T%:z' - ) - - testcases.append(entry) - - self._run_data.append({ - 'num_cases': len(run), - 'num_failures': num_failures, - 'num_aborted': num_aborted, - 'num_skipped': num_skipped, - 'runid': runid, - 'testcases': testcases - }) - - return self._run_data - - def print_failure_report(self, printer, rerun_info=True, - global_stats=False): - def _head_n(filename, prefix, num_lines=10): - # filename and prefix are `None` before setup - if filename is None or prefix is None: - return [] - - try: - with open(os.path.join(prefix, filename)) as fp: - lines = [ - f'--- {filename} (first {num_lines} lines) ---' - ] - for i, line in enumerate(fp): - if i < num_lines: - # Remove trailing '\n' - lines.append(line.rstrip()) - - lines += [f'--- {filename} ---'] - except OSError as e: - lines = [f'--- {filename} ({e}) ---'] - - return lines - - def _print_failure_info(rec, runid, total_runs): - printer.info(line_width * '-') - printer.info(f"FAILURE INFO for {rec['name']} " - f"(run: {runid}/{total_runs})") - printer.info(f" * Description: {rec['descr']}") - printer.info(f" * System partition: {rec['system']}") - printer.info(f" * Environment: {rec['environ']}") - printer.info(f" * Stage directory: {rec['stagedir']}") - printer.info(f" * Node list: " - f"{util.nodelist_abbrev(rec['job_nodelist'])}") - job_type = 'local' if rec['scheduler'] == 'local' else 'batch job' - printer.info(f" * Job type: {job_type} (id={rec['jobid']})") - printer.info(f" * Dependencies (conceptual): " - f"{rec['dependencies_conceptual']}") - printer.info(f" * Dependencies (actual): " - f"{rec['dependencies_actual']}") - printer.info(f" * Maintainers: {rec['maintainers']}") - printer.info(f" * Failing phase: {rec['fail_phase']}") - if rerun_info and not rec['fixture']: - printer.info(f" * Rerun with '-n /{rec['hashcode']}" - f" -p {rec['environ']} --system " - f"{rec['system']} -r'") - - msg = rec['fail_reason'] - if isinstance(rec['fail_info']['exc_value'], errors.SanityError): - lines = [msg] - lines += _head_n(rec['job_stdout'], prefix=rec['stagedir']) - lines += _head_n(rec['job_stderr'], prefix=rec['stagedir']) - msg = '\n'.join(lines) - - printer.info(f" * Reason: {msg}") - - tb = ''.join(traceback.format_exception( - *rec['fail_info'].values())) - if rec['fail_severe']: - printer.info(tb) - else: - printer.verbose(tb) - - line_width = shutil.get_terminal_size()[0] - printer.info(line_width * '=') - printer.info('SUMMARY OF FAILURES') - - run_report = self.json() - for run_no, run_info in enumerate(run_report, start=1): - if not global_stats and run_no != len(run_report): - continue - - for r in run_info['testcases']: - if r['result'] in {'pass', 'abort', 'skip'}: - continue - - _print_failure_info(r, run_no, len(run_report)) - - printer.info(line_width * '-') - - def print_failure_stats(self, printer, global_stats=False): - if global_stats: - runid = None - else: - runid = rt.runtime().current_run - - failures = {} - for tf in (t for t in self.tasks(runid) if t.failed): - check, partition, environ = tf.testcase - info = f'[{check.display_name}]' - if partition: - info += f' @{partition.fullname}' - - if environ: - info += f'+{environ.name}' - - if tf.failed_stage not in failures: - failures[tf.failed_stage] = [] - - failures[tf.failed_stage].append(info) - - line_width = shutil.get_terminal_size()[0] - stats_start = line_width * '=' - stats_title = 'FAILURE STATISTICS' - stats_end = line_width * '-' - stats_body = [] - row_format = "{:<13} {:<5} {}" - stats_hline = row_format.format(13*'-', 5*'-', 60*'-') - stats_header = row_format.format('Phase', '#', 'Failing test cases') - num_tests = self.num_cases(runid) - num_failures = 0 - for fl in failures.values(): - num_failures += len(fl) - - stats_body = [''] - stats_body.append(f'Total number of test cases: {num_tests}') - stats_body.append(f'Total number of failures: {num_failures}') - stats_body.append('') - stats_body.append(stats_header) - stats_body.append(stats_hline) - for p, l in failures.items(): - stats_body.append(row_format.format(p, len(l), l[0])) - for f in l[1:]: - stats_body.append(row_format.format('', '', str(f))) - - if stats_body: - for line in (stats_start, stats_title, *stats_body, stats_end): - printer.info(line) - - def performance_report(self): - width = shutil.get_terminal_size()[0] - lines = ['', width*'=', 'PERFORMANCE REPORT', width*'-'] - - # Collect all the records from performance tests - perf_records = {} - for run in self.json(): - for tc in run['testcases']: - if tc['perfvars']: - key = tc['unique_name'] - perf_records.setdefault(key, []) - perf_records[key].append(tc) - - if not perf_records: - return '' - - interesting_vars = { - 'num_cpus_per_task', - 'num_gpus_per_node', - 'num_tasks', - 'num_tasks_per_core', - 'num_tasks_per_node', - 'num_tasks_per_socket', - 'use_multithreading' - } - - for testcases in perf_records.values(): - for tc in testcases: - name = tc['name'] - hash = tc['hashcode'] - env = tc['environ'] - system = tc['system'] - part = tc['partition'] - lines.append(f'[{name} /{hash} @{system}:{part}+{env}]') - for v in interesting_vars: - val = tc['check_vars'][v] - if val is not None: - lines.append(f' {v}: {val}') - - lines.append(' performance:') - for v in tc['perfvars']: - name = v['name'] - val = v['value'] - ref = v['reference'] - unit = v['unit'] - lthr = v['thres_lower'] - uthr = v['thres_upper'] - if lthr is not None: - lthr *= 100 - else: - lthr = '-inf' - - if uthr is not None: - uthr *= 100 - else: - uthr = 'inf' - - lines.append(f' - {name}: {val} {unit} ' - f'(r: {ref} {unit} l: {lthr}% u: +{uthr}%)') - - lines.append(width*'-') - return '\n'.join(lines) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 04fcaad50c..c1a44fd494 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -17,7 +17,7 @@ import reframe.core.environments as env import reframe.core.logging as logging import reframe.core.runtime as rt -import reframe.frontend.runreport as runreport +import reframe.frontend.reporting as reporting import reframe.utility.osext as osext import unittests.utility as test_util from reframe import INSTALL_PREFIX @@ -168,7 +168,7 @@ def test_check_restore_session_failed(run_reframe, tmp_path): checkpath=[], more_options=['--restore-session', '--failed'] ) - report = runreport.load_report(f'{tmp_path}/.reframe/reports/latest.json') + report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json') assert set(report.slice('name', when=('fail_phase', 'sanity'))) == {'T2'} assert set(report.slice('name', when=('fail_phase', 'startup'))) == {'T7', 'T9'} @@ -188,7 +188,7 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path): checkpath=[], more_options=['--restore-session', '-n', 'T1'] ) - report = runreport.load_report(f'{tmp_path}/.reframe/reports/latest.json') + report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json') assert report['runs'][-1]['num_cases'] == 1 assert report['runs'][-1]['testcases'][0]['name'] == 'T1' @@ -926,7 +926,7 @@ def test_failure_stats(run_reframe, run_action): else: assert returncode != 0 assert r'FAILURE STATISTICS' in stdout - assert r'sanity 1 [SanityFailureCheck' in stdout + assert r'sanity 1 SanityFailureCheck' in stdout def test_maxfail_option(run_reframe): diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 5d8efda6d8..265f6aa52f 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -19,7 +19,7 @@ import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors import reframe.frontend.executors.policies as policies -import reframe.frontend.runreport as runreport +import reframe.frontend.reporting as reporting import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext import reframe.utility.sanity as sn @@ -33,6 +33,7 @@ RunSessionTimeout, TaskDependencyError) from reframe.frontend.loader import RegressionCheckLoader +from reframe.frontend.reporting import RunReport from unittests.resources.checks.hellocheck import HelloTest from unittests.resources.checks.frontend_checks import ( BadSetupCheck, @@ -214,31 +215,33 @@ def _validate_junit_report(report): schema.assert_(report) -def _generate_runreport(run_stats, time_start, time_end): - return { - 'session_info': { - 'cmdline': ' '.join(sys.argv), - 'config_files': rt.runtime().site_config.sources, - 'data_version': runreport.DATA_VERSION, - 'hostname': socket.gethostname(), - 'num_cases': run_stats[0]['num_cases'], - 'num_failures': run_stats[-1]['num_failures'], - 'prefix_output': rt.runtime().output_prefix, - 'prefix_stage': rt.runtime().stage_prefix, - 'time_elapsed': time_end - time_start, - 'time_end': time.strftime( - '%FT%T%z', time.localtime(time_end), - ), - 'time_start': time.strftime( - '%FT%T%z', time.localtime(time_start), - ), - 'user': osext.osuser(), - 'version': osext.reframe_version(), - 'workdir': os.getcwd() - }, - 'restored_cases': [], - 'runs': run_stats - } +def _generate_runreport(run_stats, time_start=None, time_end=None): + report = RunReport() + report.update_session_info({ + 'cmdline': ' '.join(sys.argv), + 'config_files': rt.runtime().site_config.sources, + 'data_version': reporting.DATA_VERSION, + 'hostname': socket.gethostname(), + 'prefix_output': rt.runtime().output_prefix, + 'prefix_stage': rt.runtime().stage_prefix, + 'user': osext.osuser(), + 'version': osext.reframe_version(), + 'workdir': os.getcwd() + }) + if time_start and time_end: + time_elapsed = time_end - time_start + time_start = time.strftime(r'%FT%T%z', time.localtime(time_start)) + time_end = time.strftime(r'%FT%T%z', time.localtime(time_end)) + report.update_session_info({ + 'time_elapsed': time_elapsed, + 'time_end': time_end, + 'time_start': time_start + }) + + if run_stats: + report.update_run_stats(run_stats) + + return report def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): @@ -254,28 +257,23 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): assert 1 == num_failures_stage(runner, 'performance') assert 1 == num_failures_stage(runner, 'cleanup') - # Create a run report and validate it - report = _generate_runreport(runner.stats.json(), *tm.timestamps()) - # We dump the report first, in order to get any object conversions right - report_file = tmp_path / 'report.json' - with open(report_file, 'w') as fp: - jsonext.dump(report, fp) + report = _generate_runreport(runner.stats, *tm.timestamps()) + report.save(tmp_path / 'report.json') # We explicitly set `time_total` to `None` in the last test case, in order - # to test the proper handling of `None`.` + # to test the proper handling of `None`. report['runs'][0]['testcases'][-1]['time_total'] = None # Validate the junit report - xml_report = runreport.junit_xml_report(report) - _validate_junit_report(xml_report) + _validate_junit_report(report.generate_xml_report()) - # Read and validate the report using the runreport module - runreport.load_report(report_file) + # Read and validate the report using the `reporting` module + reporting.restore_session(tmp_path / 'report.json') # Try to load a non-existent report with pytest.raises(ReframeError, match='failed to load report file'): - runreport.load_report(tmp_path / 'does_not_exist.json') + reporting.restore_session(tmp_path / 'does_not_exist.json') # Generate an invalid JSON with open(tmp_path / 'invalid.json', 'w') as fp: @@ -283,16 +281,14 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): fp.write('invalid') with pytest.raises(ReframeError, match=r'is not a valid JSON file'): - runreport.load_report(tmp_path / 'invalid.json') + reporting.restore_session(tmp_path / 'invalid.json') # Generate a report that does not comply to the schema del report['session_info']['data_version'] - with open(tmp_path / 'invalid-version.json', 'w') as fp: - jsonext.dump(report, fp) - + report.save(tmp_path / 'invalid-version.json') with pytest.raises(ReframeError, match=r'failed to validate report'): - runreport.load_report(tmp_path / 'invalid-version.json') + reporting.restore_session(tmp_path / 'invalid-version.json') def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx): @@ -386,11 +382,11 @@ def test_runall_skip_tests(make_runner, make_cases, runner.runall(cases) def assert_reported_skipped(num_skipped): - report = runner.stats.json() - assert report[0]['num_skipped'] == num_skipped + run_report = _generate_runreport(runner.stats)['runs'] + assert run_report[0]['num_skipped'] == num_skipped num_reported = 0 - for tc in report[0]['testcases']: + for tc in run_report[0]['testcases']: if tc['result'] == 'skip': num_reported += 1 @@ -456,9 +452,6 @@ def test_retries_bad_check(make_runner, make_cases, common_exec_ctx): assert runner.max_retries == rt.runtime().current_run assert 2 == len(runner.stats.failed()) - # Ensure that the report does not raise any exception - runner.stats.retry_report() - def test_retries_good_check(make_runner, make_cases, common_exec_ctx): runner = make_runner(max_retries=2) @@ -911,11 +904,9 @@ def report_file(make_runner, dep_cases, common_exec_ctx, tmp_path): with timer() as tm: runner.runall(dep_cases) - report = _generate_runreport(runner.stats.json(), *tm.timestamps()) filename = tmp_path / 'report.json' - with open(filename, 'w') as fp: - jsonext.dump(report, fp) - + report = _generate_runreport(runner.stats, *tm.timestamps()) + report.save(filename) return filename @@ -928,7 +919,7 @@ def test_restore_session(report_file, make_runner, ) # Restore the required test cases - report = runreport.load_report(report_file) + report = reporting.restore_session(report_file) testgraph, restored_cases = report.restore_dangling(testgraph) assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} @@ -938,29 +929,18 @@ def test_restore_session(report_file, make_runner, with timer() as tm: runner.runall(selected, restored_cases) - new_report = _generate_runreport(runner.stats.json(), *tm.timestamps()) + new_report = _generate_runreport(runner.stats, *tm.timestamps()) assert new_report['runs'][0]['num_cases'] == 1 assert new_report['runs'][0]['testcases'][0]['name'] == 'T1' # Generate an empty report and load it as primary with the original report # as a fallback, in order to test if the dependencies are still resolved # correctly - empty_report = tmp_path / 'empty.json' - - with open(empty_report, 'w') as fp: - empty_run = [ - { - 'num_cases': 0, - 'num_failures': 0, - 'num_aborted': 0, - 'num_skipped': 0, - 'runid': 0, - 'testcases': [] - } - ] - jsonext.dump(_generate_runreport(empty_run, *tm.timestamps()), fp) - - report2 = runreport.load_report(empty_report, report_file) + empty_report = _generate_runreport(None, *tm.timestamps()) + empty_report_file = tmp_path / 'empty.json' + empty_report.save(empty_report_file) + + report2 = reporting.restore_session(empty_report_file, report_file) restored_cases = report2.restore_dangling(testgraph)[1] assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} From e73e7d69c87453db7223719819897eb804080b77 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 28 Jun 2024 17:04:28 +0200 Subject: [PATCH 05/69] Support for selecting sessions explicitly --- reframe/frontend/cli.py | 36 +++++----- reframe/frontend/reporting/__init__.py | 27 +++++--- reframe/frontend/reporting/analytics.py | 89 +++++++++++++++---------- reframe/schemas/runreport.json | 2 + unittests/test_policies.py | 9 +-- 5 files changed, 92 insertions(+), 71 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index fcd1972717..f1f0fad523 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -233,6 +233,9 @@ def main(): testgen_options = argparser.add_argument_group( 'Options for generating tests dynamically' ) + reporting_options = argparser.add_argument_group( + 'Options related to results reporting' + ) misc_options = argparser.add_argument_group('Miscellaneous options') # Output directory options @@ -535,6 +538,17 @@ def main(): help='Repeat selected tests N times' ) + # Reporting options + reporting_options.add_argument( + '--performance-compare', metavar='CMPSPEC', action='store', + help='Compare past performance results' + ) + reporting_options.add_argument( + '--performance-report', action='store', nargs='?', + const='19700101T0000Z:now/last:+job_nodelist/+result', + help='Print a report for performance tests' + ) + # Miscellaneous options misc_options.add_argument( '-C', '--config-file', action='append', metavar='FILE', @@ -556,15 +570,6 @@ def main(): help='Disable coloring of output', envvar='RFM_COLORIZE', configvar='general/colorize' ) - misc_options.add_argument( - '--performance-compare', metavar='CMPSPEC', action='store', - help='Compare past performance results' - ) - misc_options.add_argument( - '--performance-report', action='store', nargs='?', - const='19700101T0000Z:now/last:+job_nodelist/+result', - help='Print a report for performance tests' - ) misc_options.add_argument( '--show-config', action='store', nargs='?', const='all', metavar='PARAM', @@ -1430,18 +1435,11 @@ def module_unuse(*paths): options.maxfail, options.reruns, options.duration) try: time_start = time.time() - session_info['time_start'] = time.strftime( - '%FT%T%z', time.localtime(time_start), - ) runner.runall(testcases, restored_cases) finally: # Build final JSON report time_end = time.time() - report.update_session_info({ - 'time_end': time.strftime(r'%FT%T%z', - time.localtime(time_end)), - 'time_elapsed': time_end - time_start - }) + report.update_timestamps(time_start, time_end) report.update_run_stats(runner.stats) if options.restore_session is not None: report.update_restored_cases(restored_cases, restored_session) @@ -1508,10 +1506,12 @@ def module_unuse(*paths): # Store the generated report for analytics try: - analytics.store_report(report, report.filename) + sess_uuid = analytics.store_report(report, report.filename) except Exception as e: printer.warning(f'failed to store results in the database: {e}') raise + else: + printer.info(f'Current session stored with UUID: {sess_uuid}') # Generate the junit xml report for this session junit_report_file = rt.get_option('general/0/report_junit') diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 97f1203949..d3dd344328 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -13,7 +13,6 @@ import re import socket import time -from datetime import datetime import reframe as rfm import reframe.utility.jsonext as jsonext @@ -208,19 +207,17 @@ class RunReport: ''' def __init__(self): # Initialize the report with the required fields - now = datetime.now().strftime(r'%FT%T%z') + self.__filename = None self.__report = { 'session_info': { 'data_version': DATA_VERSION, - 'hostname': socket.gethostname(), - 'time_start': now, - 'time_end': now, - 'time_elapsed': 0.0 + 'hostname': socket.gethostname() }, 'runs': [], 'restored_cases': [] } - self.__filename = None + now = time.time() + self.update_timestamps(now, now) @property def filename(self): @@ -236,12 +233,26 @@ def __rfm_json_encode__(self): return self.__report def update_session_info(self, session_info): - self.__report['session_info'].update(session_info) + # Remove timestamps + for key, val in session_info.items(): + if not key.startswith('time_'): + self.__report['session_info'][key] = val def update_restored_cases(self, restored_cases, restored_session): self.__report['restored_cases'] = [restored_session.case(*c) for c in restored_cases] + def update_timestamps(self, ts_start, ts_end): + fmt = r'%FT%T%z' + self.__report['session_info'].update({ + 'time_start': time.strftime(fmt, time.localtime(ts_start)), + 'time_start_unix': ts_start, + 'time_end': time.strftime(fmt, time.localtime(ts_end)), + 'time_end_unix': ts_end, + 'time_elapsed': ts_end - ts_start + }) + print(self.__report['session_info']) + def update_run_stats(self, stats): for runid, tasks in stats.runs(): testcases = [] diff --git a/reframe/frontend/reporting/analytics.py b/reframe/frontend/reporting/analytics.py index bdba66c21e..8cf3d77057 100644 --- a/reframe/frontend/reporting/analytics.py +++ b/reframe/frontend/reporting/analytics.py @@ -13,6 +13,7 @@ from collections import namedtuple from collections.abc import Hashable from datetime import datetime, timedelta +from numbers import Number import reframe.core.runtime as runtime import reframe.utility.jsonext as jsonext @@ -43,6 +44,9 @@ def _db_create(filename): conn.execute( '''CREATE TABLE IF NOT EXISTS sessions( id INTEGER PRIMARY KEY, + session_uuid TEXT, + session_start_unix REAL, + session_end_unix REAL, json_blob TEXT, report_file TEXT )''' @@ -63,12 +67,22 @@ def _db_create(filename): def _db_store_report(conn, report, report_file_path): + session_start_unix = report['session_info']['time_start_unix'] + session_end_unix = report['session_info']['time_end_unix'] + session_uuid = datetime.fromtimestamp(session_start_unix).strftime( + r'%Y%m%dT%H%M%S%z' + ) for run_idx, run in enumerate(report['runs']): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] cursor = conn.execute( -'''INSERT INTO sessions VALUES(:session_id, :json_blob, :report_file)''', +'''INSERT INTO sessions VALUES(:session_id, :session_uuid, + :session_start_unix, :session_end_unix, + :json_blob, :report_file)''', {'session_id': None, + 'session_uuid': session_uuid, + 'session_start_unix': session_start_unix, + 'session_end_unix': session_end_unix, 'json_blob': jsonext.dumps(report), 'report_file': report_file_path}) conn.execute( @@ -89,10 +103,12 @@ def _db_store_report(conn, report, report_file_path): } ) + return session_uuid + def store_report(report, report_file): with sqlite3.connect(_db_file()) as conn: - _db_store_report(conn, report, report_file) + return _db_store_report(conn, report, report_file) def _fetch_testcases_raw(condition): @@ -113,10 +129,22 @@ def _fetch_testcases_raw(condition): return testcases +def _fetch_session_time_period(condition): + with sqlite3.connect(_db_file()) as conn: + query = (f'SELECT session_start_unix, session_end_unix FROM sessions ' + f'WHERE {condition} LIMIT 1') + getlogger().debug(query) + results = conn.execute(query).fetchall() + if results: + return results[0] + + return None, None + + def fetch_testcases_time_period(ts_start, ts_end): return _fetch_testcases_raw( f'(job_completion_time_unix >= {ts_start} AND ' - f'job_completion_time_unix < {ts_end}) ' + f'job_completion_time_unix <= {ts_end}) ' 'ORDER BY job_completion_time_unix' ) @@ -203,6 +231,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, return (data, ['name', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols) + class _Aggregator: @classmethod def create(cls, name): @@ -271,6 +300,9 @@ def __call__(self, iterable): def _parse_timestamp(s): + if isinstance(s, Number): + return s + now = datetime.now() def _do_parse(s): if s == 'now': @@ -309,10 +341,23 @@ def _do_parse(s): return ts.timestamp() def _parse_time_period(s): - try: - ts_start, ts_end = s.split(':') - except ValueError: - raise ValueError(f'invalid time period spec: {s}') from None + if s.startswith('^'): + # Retrieve the period of a full session + try: + session_uuid = s[1:] + except IndexError: + raise ValueError(f'invalid session uuid: {s}') from None + else: + ts_start, ts_end = _fetch_session_time_period( + f'session_uuid == "{session_uuid}"' + ) + if not ts_start or not ts_end: + raise ValueError(f'no such session: {session_uuid}') + else: + try: + ts_start, ts_end = s.split(':') + except ValueError: + raise ValueError(f'invalid time period spec: {s}') from None return _parse_timestamp(ts_start), _parse_timestamp(ts_end) @@ -354,33 +399,3 @@ def parse_cmp_spec(spec): extra_cols = _parse_extra_cols(cols) return _Match(period_base, period_target, aggr_fn, extra_groups, extra_cols) - - -def performance_report_data(run_stats, report_spec): - period, aggr, cols = report_spec.split('/') - ts_start, ts_end = [_parse_timestamp(ts) for ts in period.split(':')] - op, extra_groups = aggr.split(':') - aggr_fn = _Aggregator.create(op) - extra_groups = extra_groups.split('+')[1:] - extra_cols = cols.split('+')[1:] - testcases = run_stats[0]['testcases'] - target_testcases = fetch_testcases_time_period(ts_start, ts_end) - return compare_testcase_data(testcases, target_testcases, - _Aggregator.create('first'), - aggr_fn, extra_groups, extra_cols) - - -def performance_compare_data(spec): - period_base, period_target, aggr, cols = spec.split('/') - base_testcases = fetch_testcases_time_period( - *(_parse_timestamp(ts) for ts in period_base.split(':')) - ) - target_testcases = fetch_testcases_time_period( - *(_parse_timestamp(ts) for ts in period_target.split(':')) - ) - op, extra_groups = aggr.split(':') - aggr_fn = _Aggregator.create(op) - extra_groups = extra_groups.split('+')[1:] - extra_cols = cols.split('+')[1:] - return compare_testcase_data(base_testcases, target_testcases, aggr_fn, - aggr_fn, extra_groups, extra_cols) diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index bf60343235..85a902e6ec 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -89,7 +89,9 @@ "prefix_stage": {"type": "string"}, "time_elapsed": {"type": "number"}, "time_end": {"type": "string"}, + "time_end_unix": {"type": "number"}, "time_start": {"type": "string"}, + "time_start_unix": {"type": "number"}, "user": {"type": "string"}, "version": {"type": "string"}, "workdir": {"type": "string"} diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 265f6aa52f..ced716424b 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -229,14 +229,7 @@ def _generate_runreport(run_stats, time_start=None, time_end=None): 'workdir': os.getcwd() }) if time_start and time_end: - time_elapsed = time_end - time_start - time_start = time.strftime(r'%FT%T%z', time.localtime(time_start)) - time_end = time.strftime(r'%FT%T%z', time.localtime(time_end)) - report.update_session_info({ - 'time_elapsed': time_elapsed, - 'time_end': time_end, - 'time_start': time_start - }) + report.update_timestamps(time_start, time_end) if run_stats: report.update_run_stats(run_stats) From e6afa06626106055ec546f1587e5338f624410a7 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 29 Jun 2024 00:56:43 +0200 Subject: [PATCH 06/69] Fix unit tests --- reframe/frontend/reporting/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index d3dd344328..366587f2b3 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -251,7 +251,6 @@ def update_timestamps(self, ts_start, ts_end): 'time_end_unix': ts_end, 'time_elapsed': ts_end - ts_start }) - print(self.__report['session_info']) def update_run_stats(self, stats): for runid, tasks in stats.runs(): @@ -427,11 +426,13 @@ def generate_xml_report(self): } ) if tc['result'] == 'fail': + fail_phase = tc['fail_phase'] + fail_reason = tc['fail_reason'] testcase_msg = etree.SubElement( testcase, 'failure', attrib={'type': 'failure', - 'message': tc['fail_phase']} + 'message': fail_phase} ) - testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}" + testcase_msg.text = f"{tc['fail_phase']}: {fail_reason}" testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out') testsuite_stdout.text = '' @@ -448,6 +449,7 @@ def save_junit(self, filename): method='xml', xml_declaration=True).decode() ) + def performance_compare(cmp, report=None): match = parse_cmp_spec(cmp) if match.period_base is None: From 7a307885bcb4f709e36a91826f12bdb210b7a063 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 29 Jun 2024 01:47:31 +0200 Subject: [PATCH 07/69] Fix coding style issues --- reframe/core/logging.py | 4 +- reframe/core/pipeline.py | 13 ++-- reframe/core/schedulers/__init__.py | 4 +- reframe/frontend/cli.py | 40 ++++++------ reframe/frontend/executors/__init__.py | 10 +-- reframe/frontend/printer.py | 12 ++-- reframe/frontend/reporting/analytics.py | 82 ++++++++++++++----------- unittests/test_cli.py | 10 ++- unittests/test_policies.py | 3 +- unittests/test_schedulers.py | 82 ++++++++++++------------- 10 files changed, 136 insertions(+), 124 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 223840f281..d91b067fbe 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -433,7 +433,7 @@ def _create_file_handler(site_config, config_prefix): def _create_filelog_handler(site_config, config_prefix): basedir = os.path.abspath(os.path.join( - site_config.get(f'systems/0/prefix'), + site_config.get('systems/0/prefix'), osext.expandvars(site_config.get(f'{config_prefix}/basedir')) )) prefix = osext.expandvars(site_config.get(f'{config_prefix}/prefix')) @@ -581,7 +581,7 @@ def _create_httpjson_handler(site_config, config_prefix): def _record_to_json(record, extras, ignore_keys): def _can_send(key): - return not key.startswith('_') and not key in ignore_keys + return not key.startswith('_') and key not in ignore_keys def _sanitize(s): return re.sub(r'\W', '_', s) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index b532e7b36f..f928a406f5 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -46,7 +46,8 @@ class _NoRuntime(ContainerPlatform): - '''Proxy container runtime for storing container platform info early enough. + '''Proxy container runtime for storing container platform info early + enough. This will be replaced by the framework with a concrete implementation based on the current partition info. @@ -847,8 +848,8 @@ def pipeline_hooks(cls): #: .. deprecated:: 4.0.0 #: Please use :attr:`env_vars` instead. variables = deprecate(variable(alias=env_vars), - f"the use of 'variables' is deprecated; " - f"please use 'env_vars' instead") + "the use of 'variables' is deprecated; " + "please use 'env_vars' instead") #: Time limit for this test. #: @@ -1704,7 +1705,7 @@ def _setup_build_job(self, **job_opts): ) def _setup_run_job(self, **job_opts): - self._job = self._create_job(f'run', self.local, **job_opts) + self._job = self._create_job('run', self.local, **job_opts) def _setup_container_platform(self): try: @@ -2216,7 +2217,7 @@ def check_performance(self): if perf_patterns is not None and self.perf_variables: raise ReframeSyntaxError( - f"you cannot mix 'perf_patterns' and 'perf_variables' syntax" + "you cannot mix 'perf_patterns' and 'perf_variables' syntax" ) # Convert `perf_patterns` to `perf_variables` @@ -2364,7 +2365,7 @@ def cleanup(self, remove_files=False): aliased = os.path.samefile(self._stagedir, self._outputdir) if aliased: self.logger.debug( - f'outputdir and stagedir are the same; copying skipped' + 'outputdir and stagedir are the same; copying skipped' ) else: self._copy_to_outputdir() diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index eaeb9125c9..a8565a99bc 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -555,7 +555,7 @@ def prepare(self, commands, environs=None, prepare_cmds=None, strict_flex=False, **gen_opts): environs = environs or [] if self.num_tasks is not None and self.num_tasks <= 0: - getlogger().debug(f'[F] Flexible node allocation requested') + getlogger().debug('[F] Flexible node allocation requested') num_tasks_per_node = self.num_tasks_per_node or 1 min_num_tasks = (-self.num_tasks if self.num_tasks else num_tasks_per_node) @@ -636,7 +636,7 @@ def finished(self): return done def __eq__(self, other): - return type(self) == type(other) and self.jobid == other.jobid + return type(self) is type(other) and self.jobid == other.jobid def __hash__(self): return hash(self.jobid) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f1f0fad523..268caf8dc9 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -9,7 +9,6 @@ import os import random import shlex -import socket import sys import time import traceback @@ -90,7 +89,8 @@ def dep_lines(u, *, prefix, depth=0, lines=None, printed=None, else: fmt_fixt_vars = '' - name_info = f'{u.check.display_name}{fmt_fixt_vars} /{u.check.hashcode}' + name_info = (f'{u.check.display_name}{fmt_fixt_vars} ' + f'/{u.check.hashcode}') tc_info = '' details = '' if concretized: @@ -98,7 +98,8 @@ def dep_lines(u, *, prefix, depth=0, lines=None, printed=None, location = inspect.getfile(type(u.check)) if detailed: - details = f' [variant: {u.check.variant_num}, file: {location!r}]' + details = (f' [variant: {u.check.variant_num}, ' + f'file: {location!r}]') lines.append(f'{prefix}^{name_info}{tc_info}{details}') @@ -1016,12 +1017,12 @@ def print_infoline(param, value): # Print command line session_info = report['session_info'] - printer.info(f"[ReFrame Setup]") + printer.info('[ReFrame Setup]') print_infoline('version', session_info['version']) print_infoline('command', repr(session_info['cmdline'])) print_infoline( - f"launched by", - f"{session_info['user'] or ''}@{session_info['hostname']}" + 'launched by', + f'{session_info["user"] or ""}@{session_info["hostname"]}' ) print_infoline('working directory', repr(session_info['workdir'])) print_infoline( @@ -1237,7 +1238,9 @@ def _sort_testcases(testcases): printer.debug('Pruned test DAG') printer.debug(dependencies.format_deps(testgraph)) if options.restore_session is not None: - testgraph, restored_cases = restored_session.restore_dangling(testgraph) + testgraph, restored_cases = restored_session.restore_dangling( + testgraph + ) testcases = dependencies.toposort( testgraph, @@ -1320,7 +1323,7 @@ def _sort_testcases(testcases): # Load the environment for the current system try: - printer.debug(f'Loading environment for current system') + printer.debug('Loading environment for current system') runtime.loadenv(rt.system.preload_environ) except errors.EnvironError as e: printer.error("failed to load current system's environment; " @@ -1332,14 +1335,14 @@ def module_use(*paths): try: rt.modules_system.searchpath_add(*paths) except errors.EnvironError as e: - printer.warning(f'could not add module paths correctly') + printer.warning('could not add module paths correctly') printer.debug(str(e)) def module_unuse(*paths): try: rt.modules_system.searchpath_remove(*paths) except errors.EnvironError as e: - printer.warning(f'could not remove module paths correctly') + printer.warning('could not remove module paths correctly') printer.debug(str(e)) printer.debug('(Un)using module paths from command line') @@ -1417,18 +1420,19 @@ def module_unuse(*paths): exec_policy.sched_options = parsed_job_options if options.maxfail < 0: raise errors.CommandLineError( - f'--maxfail should be a non-negative integer: ' + '--maxfail should be a non-negative integer: ' f'{options.maxfail}' ) if options.reruns and options.duration: raise errors.CommandLineError( - f"'--reruns' option cannot be combined with '--duration'" + "'--reruns' option cannot be combined with '--duration'" ) if options.reruns < 0: raise errors.CommandLineError( - f"'--reruns' should be a non-negative integer: {options.reruns}" + "'--reruns' should be a non-negative integer: " + f"{options.reruns}" ) runner = Runner(exec_policy, printer, options.max_retries, @@ -1494,7 +1498,7 @@ def module_unuse(*paths): report_file, compress=rt.get_option('general/0/compress_report'), link_to_last=(default_loc == os.path.dirname(report_file)) - ) + ) except OSError as e: printer.warning( f'failed to generate report in {report_file!r}: {e}' @@ -1508,8 +1512,9 @@ def module_unuse(*paths): try: sess_uuid = analytics.store_report(report, report.filename) except Exception as e: - printer.warning(f'failed to store results in the database: {e}') - raise + printer.warning( + f'failed to store results in the database: {e}' + ) else: printer.info(f'Current session stored with UUID: {sess_uuid}') @@ -1551,9 +1556,8 @@ def module_unuse(*paths): finally: try: logging.getprofiler().exit_region() # region: 'test processing' - log_files = logging.log_files() if site_config.get('general/0/save_log_files'): - log_files = logging.save_log_files(rt.output_prefix) + logging.save_log_files(rt.output_prefix) except OSError as e: printer.error(f'could not save log file: {e}') sys.exit(1) diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 90b81c74c2..8bf3f58117 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -163,12 +163,10 @@ def generate_testcases(checks, prepare=False): '''Generate concrete test cases from checks. If `prepare` is true then each of the cases will also be prepared for - being sent to the test pipeline. Note that setting this to true may slow down - the test case generation. - + being sent to the test pipeline. Note that setting this to true may slow + down the test case generation. ''' - rt = runtime.runtime() cases = [] for c in checks: valid_comb = runtime.valid_sysenv_comb(c.valid_systems, @@ -410,7 +408,6 @@ def temp_dry_run(check): with temp_dry_run(self.check): return fn(*args, **kwargs) - @logging.time_function def setup(self, *args, **kwargs): self.testcase.prepare() @@ -647,9 +644,6 @@ def runall(self, testcases, restored_cases=None): runid = None if self._global_stats else -1 num_aborted = len(self._stats.aborted(runid)) num_failures = len(self._stats.failed(runid)) - num_completed = len(self._stats.completed(runid)) - num_skipped = len(self._stats.skipped(runid)) - num_tasks = self._stats.num_cases(runid) if num_failures > 0: status = 'FAILED' elif num_aborted > 0: diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index b6530f05bb..d94444fade 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -121,25 +121,25 @@ def _head_n(filename, prefix, num_lines=10): def _print_failure_info(rec, runid, total_runs): self.info(line_width * '-') self.info(f"FAILURE INFO for {rec['name']} " - f"(run: {runid}/{total_runs})") + f"(run: {runid}/{total_runs})") self.info(f" * Description: {rec['descr']}") self.info(f" * System partition: {rec['system']}") self.info(f" * Environment: {rec['environ']}") self.info(f" * Stage directory: {rec['stagedir']}") self.info(f" * Node list: " - f"{nodelist_abbrev(rec['job_nodelist'])}") + f"{nodelist_abbrev(rec['job_nodelist'])}") job_type = 'local' if rec['scheduler'] == 'local' else 'batch job' self.info(f" * Job type: {job_type} (id={rec['jobid']})") self.info(f" * Dependencies (conceptual): " - f"{rec['dependencies_conceptual']}") + f"{rec['dependencies_conceptual']}") self.info(f" * Dependencies (actual): " - f"{rec['dependencies_actual']}") + f"{rec['dependencies_actual']}") self.info(f" * Maintainers: {rec['maintainers']}") self.info(f" * Failing phase: {rec['fail_phase']}") if rerun_info and not rec['fixture']: self.info(f" * Rerun with '-n /{rec['hashcode']}" - f" -p {rec['environ']} --system " - f"{rec['system']} -r'") + f" -p {rec['environ']} --system " + f"{rec['system']} -r'") msg = rec['fail_reason'] if isinstance(rec['fail_info']['exc_value'], SanityError): diff --git a/reframe/frontend/reporting/analytics.py b/reframe/frontend/reporting/analytics.py index 8cf3d77057..d8d79da25d 100644 --- a/reframe/frontend/reporting/analytics.py +++ b/reframe/frontend/reporting/analytics.py @@ -41,29 +41,24 @@ def _db_file(): def _db_create(filename): with sqlite3.connect(filename) as conn: - conn.execute( -'''CREATE TABLE IF NOT EXISTS sessions( - id INTEGER PRIMARY KEY, - session_uuid TEXT, - session_start_unix REAL, - session_end_unix REAL, - json_blob TEXT, - report_file TEXT -)''' - ) - conn.execute( -'''CREATE TABLE IF NOT EXISTS testcases( - name TEXT, - system TEXT, - partition TEXT, - environ TEXT, - job_completion_time_unix REAL, - session_id INTEGER, - run_index INTEGER, - test_index INTEGER, - FOREIGN KEY(session_id) REFERENCES sessions(session_id) -)''' - ) + conn.execute('CREATE TABLE IF NOT EXISTS sessions(' + 'id INTEGER PRIMARY KEY, ' + 'session_uuid TEXT, ' + 'session_start_unix REAL, ' + 'session_end_unix REAL, ' + 'json_blob TEXT, ' + 'report_file TEXT)') + conn.execute('CREATE TABLE IF NOT EXISTS testcases(' + 'name TEXT,' + 'system TEXT,' + 'partition TEXT,' + 'environ TEXT,' + 'job_completion_time_unix REAL,' + 'session_id INTEGER,' + 'run_index INTEGER,' + 'test_index INTEGER,' + 'FOREIGN KEY(session_id) ' + 'REFERENCES sessions(session_id))') def _db_store_report(conn, report, report_file_path): @@ -76,19 +71,24 @@ def _db_store_report(conn, report, report_file_path): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] cursor = conn.execute( -'''INSERT INTO sessions VALUES(:session_id, :session_uuid, - :session_start_unix, :session_end_unix, - :json_blob, :report_file)''', - {'session_id': None, - 'session_uuid': session_uuid, - 'session_start_unix': session_start_unix, - 'session_end_unix': session_end_unix, - 'json_blob': jsonext.dumps(report), - 'report_file': report_file_path}) + 'INSERT INTO sessions VALUES(' + ':session_id, :session_uuid, ' + ':session_start_unix, :session_end_unix, ' + ':json_blob, :report_file)', + { + 'session_id': None, + 'session_uuid': session_uuid, + 'session_start_unix': session_start_unix, + 'session_end_unix': session_end_unix, + 'json_blob': jsonext.dumps(report), + 'report_file': report_file_path + } + ) conn.execute( -'''INSERT INTO testcases VALUES(:name, :system, :partition, :environ, - :job_completion_time_unix, - :session_id, :run_index, :test_index)''', + 'INSERT INTO testcases VALUES(' + ':name, :system, :partition, :environ, ' + ':job_completion_time_unix, ' + ':session_id, :run_index, :test_index)', { 'name': testcase['name'], 'system': sys, @@ -186,6 +186,7 @@ def _group_testcases(testcases, group_by, extra_cols): return grouped + def _aggregate_perf(grouped_testcases, aggr_fn, cols): other_aggr = _JoinUniqueValues('|') aggr_data = {} @@ -200,6 +201,7 @@ def _aggregate_perf(grouped_testcases, aggr_fn, cols): return aggr_data + def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, extra_group_by=None, extra_cols=None): extra_group_by = extra_group_by or [] @@ -229,7 +231,8 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, line += [aggr_data[c] for c in extra_cols] data.append(line) - return (data, ['name', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols) + return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] + + extra_group_by + extra_cols)) class _Aggregator: @@ -254,12 +257,14 @@ def create(cls, name): def __call__(self, iterable): pass + class _First(_Aggregator): def __call__(self, iterable): for i, elem in enumerate(iterable): if i == 0: return elem + class _Last(_Aggregator): def __call__(self, iterable): if not isinstance(iterable, types.GeneratorType): @@ -290,6 +295,7 @@ class _Max(_Aggregator): def __call__(self, iterable): return max(iterable) + class _JoinUniqueValues(_Aggregator): def __init__(self, delim): self.__delim = delim @@ -304,6 +310,7 @@ def _parse_timestamp(s): return s now = datetime.now() + def _do_parse(s): if s == 'now': return now @@ -340,6 +347,7 @@ def _do_parse(s): return ts.timestamp() + def _parse_time_period(s): if s.startswith('^'): # Retrieve the period of a full session @@ -361,6 +369,7 @@ def _parse_time_period(s): return _parse_timestamp(ts_start), _parse_timestamp(ts_end) + def _parse_extra_cols(s): try: extra_cols = s.split('+')[1:] @@ -382,6 +391,7 @@ def _parse_aggregation(s): _Match = namedtuple('_Match', ['period_base', 'period_target', 'aggregator', 'extra_groups', 'extra_cols']) + def parse_cmp_spec(spec): parts = spec.split('/') if len(parts) == 3: diff --git a/unittests/test_cli.py b/unittests/test_cli.py index c1a44fd494..0f9f10880d 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -168,7 +168,9 @@ def test_check_restore_session_failed(run_reframe, tmp_path): checkpath=[], more_options=['--restore-session', '--failed'] ) - report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json') + report = reporting.restore_session( + f'{tmp_path}/.reframe/reports/latest.json' + ) assert set(report.slice('name', when=('fail_phase', 'sanity'))) == {'T2'} assert set(report.slice('name', when=('fail_phase', 'startup'))) == {'T7', 'T9'} @@ -188,7 +190,9 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path): checkpath=[], more_options=['--restore-session', '-n', 'T1'] ) - report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json') + report = reporting.restore_session( + f'{tmp_path}/.reframe/reports/latest.json' + ) assert report['runs'][-1]['num_cases'] == 1 assert report['runs'][-1]['testcases'][0]['name'] == 'T1' @@ -201,7 +205,7 @@ def test_check_restore_session_check_search_path(run_reframe, tmp_path): checkpath=['unittests/resources/checks_unlisted/deps_complex.py'] ) returncode, stdout, _ = run_reframe( - checkpath=[f'foo/'], + checkpath=['foo/'], more_options=['--restore-session', '-n', 'T1', '-R'], action='list' ) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index ced716424b..66ed64389b 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -1322,7 +1322,8 @@ def test_perf_logging_lazy(make_runner, make_exec_ctx, lazy_perf_test, testcases = executors.generate_testcases([lazy_perf_test]) _assert_no_logging_error(runner.runall, testcases) - logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_LazyPerfTest.log' + logfile = (tmp_path / 'perflogs' / 'generic' / 'default' / + '_LazyPerfTest.log') assert os.path.exists(logfile) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index cd1c3413d0..b2da80bcfb 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -158,23 +158,23 @@ def assert_job_script_sanity(job): def _expected_lsf_directives(job): return set([ - f'#BSUB -J testjob', + '#BSUB -J testjob', f'#BSUB -o {job.stdout}', f'#BSUB -e {job.stderr}', f'#BSUB -nnodes {job.num_tasks // job.num_tasks_per_node}', f'#BSUB -W {int(job.time_limit // 60)}', f'#BSUB -R "affinity[core({job.num_cpus_per_task})]"', - f'#BSUB -x', - f'#BSUB --account=spam', - f'#BSUB --gres=gpu:4', - f'#DW jobdw capacity=100GB', - f'#DW stage_in source=/foo', + '#BSUB -x', + '#BSUB --account=spam', + '#BSUB --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo', ]) def _expected_lsf_directives_minimal(job): return set([ - f'#BSUB -J testjob', + '#BSUB -J testjob', f'#BSUB -o {job.stdout}', f'#BSUB -e {job.stderr}', f'#BSUB -n {job.num_tasks}' @@ -183,7 +183,7 @@ def _expected_lsf_directives_minimal(job): def _expected_lsf_directives_no_tasks(job): return set([ - f'#BSUB -J testjob', + '#BSUB -J testjob', f'#BSUB -o {job.stdout}', f'#BSUB -e {job.stderr}' ]) @@ -202,24 +202,22 @@ def _expected_flux_directives_no_tasks(job): def _expected_sge_directives(job): - num_nodes = job.num_tasks // job.num_tasks_per_node - num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node return set([ - f'#$ -N "testjob"', - f'#$ -l h_rt=0:5:0', + '#$ -N "testjob"', + '#$ -l h_rt=0:5:0', f'#$ -o {job.stdout}', f'#$ -e {job.stderr}', f'#$ -wd {job.workdir}', - f'#$ --gres=gpu:4', - f'#$ --account=spam', - f'#DW jobdw capacity=100GB', - f'#DW stage_in source=/foo' + '#$ --gres=gpu:4', + '#$ --account=spam', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' ]) def _expected_sge_directives_minimal(job): return set([ - f'#$ -N "testjob"', + '#$ -N "testjob"', f'#$ -o {job.stdout}', f'#$ -e {job.stderr}', f'#$ -wd {job.workdir}' @@ -276,24 +274,24 @@ def _expected_pbs_directives(job): num_nodes = job.num_tasks // job.num_tasks_per_node num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node return set([ - f'#PBS -N testjob', - f'#PBS -l walltime=0:5:0', + '#PBS -N testjob', + '#PBS -l walltime=0:5:0', f'#PBS -o {job.stdout}', f'#PBS -e {job.stderr}', f'#PBS -l select={num_nodes}:mpiprocs={job.num_tasks_per_node}:ncpus={num_cpus_per_node}:mem=100GB:cpu_type=haswell', # noqa: E501 - f'#PBS --account=spam', - f'#PBS --gres=gpu:4', - f'#DW jobdw capacity=100GB', - f'#DW stage_in source=/foo' + '#PBS --account=spam', + '#PBS --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' ]) def _expected_pbs_directives_minimal(job): return set([ - f'#PBS -N testjob', + '#PBS -N testjob', f'#PBS -o {job.stdout}', f'#PBS -e {job.stderr}', - f'#PBS -l select=1:mpiprocs=1:ncpus=1' + '#PBS -l select=1:mpiprocs=1:ncpus=1' ]) @@ -304,25 +302,25 @@ def _expected_torque_directives(job): num_nodes = job.num_tasks // job.num_tasks_per_node num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node return set([ - f'#PBS -N testjob', - f'#PBS -l walltime=0:5:0', + '#PBS -N testjob', + '#PBS -l walltime=0:5:0', f'#PBS -o {job.stdout}', f'#PBS -e {job.stderr}', f'#PBS -l nodes={num_nodes}:ppn={num_cpus_per_node}:haswell', - f'#PBS -l mem=100GB', - f'#PBS --account=spam', - f'#PBS --gres=gpu:4', - f'#DW jobdw capacity=100GB', - f'#DW stage_in source=/foo' + '#PBS -l mem=100GB', + '#PBS --account=spam', + '#PBS --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' ]) def _expected_torque_directives_minimal(job): return set([ - f'#PBS -N testjob', + '#PBS -N testjob', f'#PBS -o {job.stdout}', f'#PBS -e {job.stderr}', - f'#PBS -l nodes=1:ppn=1' + '#PBS -l nodes=1:ppn=1' ]) @@ -333,23 +331,23 @@ def _expected_oar_directives(job): num_nodes = job.num_tasks // job.num_tasks_per_node num_tasks_per_node = job.num_tasks_per_node return set([ - f'#OAR -n "testjob"', + '#OAR -n "testjob"', f'#OAR -O {job.stdout}', f'#OAR -E {job.stderr}', f'#OAR -l /host={num_nodes}/core={num_tasks_per_node},walltime=0:5:0', - f'#OAR --account=spam', - f'#OAR --gres=gpu:4', - f'#DW jobdw capacity=100GB', - f'#DW stage_in source=/foo' + '#OAR --account=spam', + '#OAR --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' ]) def _expected_oar_directives_minimal(job): return set([ - f'#OAR -n "testjob"', + '#OAR -n "testjob"', f'#OAR -O {job.stdout}', f'#OAR -E {job.stderr}', - f'#OAR -l /host=1/core=1' + '#OAR -l /host=1/core=1' ]) @@ -652,7 +650,7 @@ def test_sched_access_in_submit(make_job): job.scheduler._sched_access_in_submit = True if job.scheduler.registered_name in ('flux', 'local', 'ssh'): - pytest.skip(f'not relevant for this scheduler backend') + pytest.skip('not relevant for this scheduler backend') prepare_job(job) with open(job.script_filename) as fp: From fe36ed5a44cc894a024b8422156a7eaab0934740 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 10:20:12 +0200 Subject: [PATCH 08/69] Fix `tabulate` requirement for Python 3.6 --- requirements.txt | 3 ++- setup.cfg | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e01146fe10..e6df345eb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ semver==3.0.2; python_version >= '3.7' setuptools==59.6.0; python_version == '3.6' setuptools==68.0.0; python_version == '3.7' setuptools==72.1.0; python_version >= '3.8' -tabulate==0.9.0 +tabulate==0.8.10; python_version == '3.6' +tabulate==0.9.0; python_version >= '3.7' wcwidth==0.2.13 #+pygelf%pygelf==0.4.0 diff --git a/setup.cfg b/setup.cfg index cbbf1b17e4..5ae3959849 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,8 @@ install_requires = requests <= 2.27.1; python_version == '3.6' semver semver <= 2.13.0; python_version == '3.6' + tabulate + tabulate <= 0.8.10; python_version == '3.6' [options.packages.find] include = reframe,reframe.*,hpctestlib.* From f56d1ed3103c9f67ac91735bdff6434199ee7901 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 11:48:08 +0200 Subject: [PATCH 09/69] Add an API around results storage and merge `analytics` with `reporting` --- reframe/frontend/cli.py | 3 +- reframe/frontend/reporting/__init__.py | 292 ++++++++++++++++- reframe/frontend/reporting/analytics.py | 411 ------------------------ 3 files changed, 287 insertions(+), 419 deletions(-) delete mode 100644 reframe/frontend/reporting/analytics.py diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 268caf8dc9..a7be337061 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -24,7 +24,6 @@ import reframe.frontend.dependencies as dependencies import reframe.frontend.filters as filters import reframe.frontend.reporting as reporting -import reframe.frontend.reporting.analytics as analytics import reframe.utility as util import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext @@ -1510,7 +1509,7 @@ def module_unuse(*paths): # Store the generated report for analytics try: - sess_uuid = analytics.store_report(report, report.filename) + sess_uuid = report.store() except Exception as e: printer.warning( f'failed to store results in the database: {e}' diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 366587f2b3..6a96d73afb 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -3,16 +3,24 @@ # # SPDX-License-Identifier: BSD-3-Clause +import abc import decimal import functools import inspect import json import jsonschema import lxml.etree as etree +import math import os import re import socket +import statistics import time +import types +from collections import namedtuple +from collections.abc import Hashable +from datetime import datetime, timedelta +from numbers import Number import reframe as rfm import reframe.utility.jsonext as jsonext @@ -20,10 +28,8 @@ from reframe.core.exceptions import ReframeError, what, is_severe from reframe.core.logging import getlogger, _format_time_rfc3339 from reframe.core.warnings import suppress_deprecations -from .analytics import (parse_cmp_spec, - fetch_testcases_time_period, - compare_testcase_data) - +from reframe.utility import nodelist_abbrev +from .storage import StorageBackend # The schema data version # Major version bumps are expected to break the validation of previous schemas @@ -41,6 +47,9 @@ def format_testcase(json, name='unique_name'): return f'{name}@{system}:{partition}+{environ}' +_ReportStorage = StorageBackend.create('sqlite') + + class _RestoredSessionInfo: '''A restored session with some additional functionality.''' @@ -388,6 +397,11 @@ def save(self, filename, compress=False, link_to_last=True): else: raise ReframeError('path exists and is not a symlink') + def store(self): + '''Store the report in the results storage.''' + + return _ReportStorage.store(self, self.filename) + def generate_xml_report(self): '''Generate a JUnit report from a standard ReFrame JSON report.''' @@ -450,6 +464,268 @@ def save_junit(self, filename): ) +def _group_key(groups, testcase): + key = [] + for grp in groups: + val = testcase[grp] + if grp == 'job_nodelist': + # Fold nodelist before adding as a key element + key.append(nodelist_abbrev(val)) + elif not isinstance(val, Hashable): + key.append(str(val)) + else: + key.append(val) + + return tuple(key) + + +def _group_testcases(testcases, group_by, extra_cols): + grouped = {} + for tc in testcases: + for pvar, reftuple in tc['perfvalues'].items(): + pvar = pvar.split(':')[-1] + pval, pref, plower, pupper, punit = reftuple + plower = pref * (1 + plower) if plower is not None else -math.inf + pupper = pref * (1 + pupper) if pupper is not None else math.inf + record = { + 'pvar': pvar, + 'pval': pval, + 'plower': plower, + 'pupper': pupper, + 'punit': punit, + **{k: tc[k] for k in group_by + extra_cols if k in tc} + } + key = _group_key(group_by, record) + grouped.setdefault(key, []) + grouped[key].append(record) + + return grouped + + +def _aggregate_perf(grouped_testcases, aggr_fn, cols): + other_aggr = _JoinUniqueValues('|') + aggr_data = {} + for key, seq in grouped_testcases.items(): + aggr_data.setdefault(key, {}) + aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq) + for c in cols: + aggr_data[key][c] = other_aggr( + nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c] + for tc in seq + ) + + return aggr_data + + +def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, + extra_group_by=None, extra_cols=None): + extra_group_by = extra_group_by or [] + extra_cols = extra_cols or [] + group_by = ['name', 'pvar', 'punit'] + extra_group_by + + grouped_base = _group_testcases(base_testcases, group_by, extra_cols) + grouped_target = _group_testcases(target_testcases, group_by, extra_cols) + pbase = _aggregate_perf(grouped_base, base_fn, extra_cols) + ptarget = _aggregate_perf(grouped_target, target_fn, []) + + # Build the final table data + data = [] + for key, aggr_data in pbase.items(): + pval = aggr_data['pval'] + try: + target_pval = ptarget[key]['pval'] + except KeyError: + pdiff = 'n/a' + else: + pdiff = (pval - target_pval) / target_pval + pdiff = '{:+7.2%}'.format(pdiff) + + name, pvar, punit, *extras = key + line = [name, pvar, pval, punit, pdiff, *extras] + # Add the extra columns + line += [aggr_data[c] for c in extra_cols] + data.append(line) + + return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] + + extra_group_by + extra_cols)) + + +class _Aggregator: + @classmethod + def create(cls, name): + if name == 'first': + return _First() + elif name == 'last': + return _Last() + elif name == 'mean': + return _Mean() + elif name == 'median': + return _Median() + elif name == 'min': + return _Min() + elif name == 'max': + return _Max() + else: + raise ValueError(f'unknown aggregation function: {name!r}') + + @abc.abstractmethod + def __call__(self, iterable): + pass + + +class _First(_Aggregator): + def __call__(self, iterable): + for i, elem in enumerate(iterable): + if i == 0: + return elem + + +class _Last(_Aggregator): + def __call__(self, iterable): + if not isinstance(iterable, types.GeneratorType): + return iterable[-1] + + for elem in iterable: + pass + + return elem + + +class _Mean(_Aggregator): + def __call__(self, iterable): + return statistics.mean(iterable) + + +class _Median(_Aggregator): + def __call__(self, iterable): + return statistics.median(iterable) + + +class _Min(_Aggregator): + def __call__(self, iterable): + return min(iterable) + + +class _Max(_Aggregator): + def __call__(self, iterable): + return max(iterable) + + +class _JoinUniqueValues(_Aggregator): + def __init__(self, delim): + self.__delim = delim + + def __call__(self, iterable): + unique_vals = {str(elem) for elem in iterable} + return self.__delim.join(unique_vals) + + +def _parse_timestamp(s): + if isinstance(s, Number): + return s + + now = datetime.now() + + def _do_parse(s): + if s == 'now': + return now + + formats = [r'%Y%m%d', r'%Y%m%dT%H%M', + r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] + for fmt in formats: + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + + raise ValueError(f'invalid timestamp: {s}') + + try: + ts = _do_parse(s) + except ValueError as err: + # Try the relative timestamps + match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) + if not match: + raise err + + ts = _do_parse(match.group('ts')) + amount = int(match.group('amount')) + unit = match.group('unit') + if unit == 'd': + ts += timedelta(days=amount) + elif unit == 'm': + ts += timedelta(minutes=amount) + elif unit == 'h': + ts += timedelta(hours=amount) + elif unit == 's': + ts += timedelta(seconds=amount) + + return ts.timestamp() + + +def _parse_time_period(s): + if s.startswith('^'): + # Retrieve the period of a full session + try: + session_uuid = s[1:] + except IndexError: + raise ValueError(f'invalid session uuid: {s}') from None + else: + ts_start, ts_end = _ReportStorage.fetch_session_time_period( + f'session_uuid == "{session_uuid}"' + ) + if not ts_start or not ts_end: + raise ValueError(f'no such session: {session_uuid}') + else: + try: + ts_start, ts_end = s.split(':') + except ValueError: + raise ValueError(f'invalid time period spec: {s}') from None + + return _parse_timestamp(ts_start), _parse_timestamp(ts_end) + + +def _parse_extra_cols(s): + try: + extra_cols = s.split('+')[1:] + except (ValueError, IndexError): + raise ValueError(f'invalid extra groups spec: {s}') from None + + return extra_cols + + +def _parse_aggregation(s): + try: + op, extra_groups = s.split(':') + except ValueError: + raise ValueError(f'invalid aggregate function spec: {s}') from None + + return _Aggregator.create(op), _parse_extra_cols(extra_groups) + + +_Match = namedtuple('_Match', ['period_base', 'period_target', + 'aggregator', 'extra_groups', 'extra_cols']) + + +def parse_cmp_spec(spec): + parts = spec.split('/') + if len(parts) == 3: + period_base, period_target, aggr, cols = None, *parts + elif len(parts) == 4: + period_base, period_target, aggr, cols = parts + else: + raise ValueError(f'invalid cmp spec: {spec}') + + if period_base is not None: + period_base = _parse_time_period(period_base) + + period_target = _parse_time_period(period_target) + aggr_fn, extra_groups = _parse_aggregation(aggr) + extra_cols = _parse_extra_cols(cols) + return _Match(period_base, period_target, + aggr_fn, extra_groups, extra_cols) + + def performance_compare(cmp, report=None): match = parse_cmp_spec(cmp) if match.period_base is None: @@ -462,9 +738,13 @@ def performance_compare(cmp, report=None): tcs_base = [] else: - tcs_base = fetch_testcases_time_period(*match.period_base) + tcs_base = _ReportStorage.fetch_testcases_time_period( + *match.period_base + ) - tcs_target = fetch_testcases_time_period(*match.period_target) + tcs_target = _ReportStorage.fetch_testcases_time_period( + *match.period_target + ) return compare_testcase_data(tcs_base, tcs_target, match.aggregator, match.aggregator, match.extra_groups, match.extra_cols) diff --git a/reframe/frontend/reporting/analytics.py b/reframe/frontend/reporting/analytics.py deleted file mode 100644 index d8d79da25d..0000000000 --- a/reframe/frontend/reporting/analytics.py +++ /dev/null @@ -1,411 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import abc -import os -import math -import re -import sqlite3 -import statistics -import types -from collections import namedtuple -from collections.abc import Hashable -from datetime import datetime, timedelta -from numbers import Number - -import reframe.core.runtime as runtime -import reframe.utility.jsonext as jsonext -import reframe.utility.osext as osext -from reframe.core.logging import getlogger -from reframe.utility import nodelist_abbrev - - -def _db_file(): - site_config = runtime.runtime().site_config - prefix = os.path.dirname(osext.expandvars( - site_config.get('general/0/report_file') - )) - filename = os.path.join(prefix, 'results.db') - if not os.path.exists(filename): - # Create subdirs if needed - if prefix: - os.makedirs(prefix, exist_ok=True) - - getlogger().debug(f'Creating the results database in {filename}...') - _db_create(filename) - - return filename - - -def _db_create(filename): - with sqlite3.connect(filename) as conn: - conn.execute('CREATE TABLE IF NOT EXISTS sessions(' - 'id INTEGER PRIMARY KEY, ' - 'session_uuid TEXT, ' - 'session_start_unix REAL, ' - 'session_end_unix REAL, ' - 'json_blob TEXT, ' - 'report_file TEXT)') - conn.execute('CREATE TABLE IF NOT EXISTS testcases(' - 'name TEXT,' - 'system TEXT,' - 'partition TEXT,' - 'environ TEXT,' - 'job_completion_time_unix REAL,' - 'session_id INTEGER,' - 'run_index INTEGER,' - 'test_index INTEGER,' - 'FOREIGN KEY(session_id) ' - 'REFERENCES sessions(session_id))') - - -def _db_store_report(conn, report, report_file_path): - session_start_unix = report['session_info']['time_start_unix'] - session_end_unix = report['session_info']['time_end_unix'] - session_uuid = datetime.fromtimestamp(session_start_unix).strftime( - r'%Y%m%dT%H%M%S%z' - ) - for run_idx, run in enumerate(report['runs']): - for test_idx, testcase in enumerate(run['testcases']): - sys, part = testcase['system'], testcase['partition'] - cursor = conn.execute( - 'INSERT INTO sessions VALUES(' - ':session_id, :session_uuid, ' - ':session_start_unix, :session_end_unix, ' - ':json_blob, :report_file)', - { - 'session_id': None, - 'session_uuid': session_uuid, - 'session_start_unix': session_start_unix, - 'session_end_unix': session_end_unix, - 'json_blob': jsonext.dumps(report), - 'report_file': report_file_path - } - ) - conn.execute( - 'INSERT INTO testcases VALUES(' - ':name, :system, :partition, :environ, ' - ':job_completion_time_unix, ' - ':session_id, :run_index, :test_index)', - { - 'name': testcase['name'], - 'system': sys, - 'partition': part, - 'environ': testcase['environ'], - 'job_completion_time_unix': testcase[ - 'job_completion_time_unix' - ], - 'session_id': cursor.lastrowid, - 'run_index': run_idx, - 'test_index': test_idx - } - ) - - return session_uuid - - -def store_report(report, report_file): - with sqlite3.connect(_db_file()) as conn: - return _db_store_report(conn, report, report_file) - - -def _fetch_testcases_raw(condition): - with sqlite3.connect(_db_file()) as conn: - query = (f'SELECT session_id, run_index, test_index, json_blob FROM ' - f'testcases JOIN sessions ON session_id==id ' - f'WHERE {condition}') - getlogger().debug(query) - results = conn.execute(query).fetchall() - - # Retrieve files - testcases = [] - sessions = {} - for session_id, run_index, test_index, json_blob in results: - report = jsonext.loads(sessions.setdefault(session_id, json_blob)) - testcases.append(report['runs'][run_index]['testcases'][test_index]) - - return testcases - - -def _fetch_session_time_period(condition): - with sqlite3.connect(_db_file()) as conn: - query = (f'SELECT session_start_unix, session_end_unix FROM sessions ' - f'WHERE {condition} LIMIT 1') - getlogger().debug(query) - results = conn.execute(query).fetchall() - if results: - return results[0] - - return None, None - - -def fetch_testcases_time_period(ts_start, ts_end): - return _fetch_testcases_raw( - f'(job_completion_time_unix >= {ts_start} AND ' - f'job_completion_time_unix <= {ts_end}) ' - 'ORDER BY job_completion_time_unix' - ) - - -def _group_key(groups, testcase): - key = [] - for grp in groups: - val = testcase[grp] - if grp == 'job_nodelist': - # Fold nodelist before adding as a key element - key.append(nodelist_abbrev(val)) - elif not isinstance(val, Hashable): - key.append(str(val)) - else: - key.append(val) - - return tuple(key) - - -def _group_testcases(testcases, group_by, extra_cols): - grouped = {} - for tc in testcases: - for pvar, reftuple in tc['perfvalues'].items(): - pvar = pvar.split(':')[-1] - pval, pref, plower, pupper, punit = reftuple - plower = pref * (1 + plower) if plower is not None else -math.inf - pupper = pref * (1 + pupper) if pupper is not None else math.inf - record = { - 'pvar': pvar, - 'pval': pval, - 'plower': plower, - 'pupper': pupper, - 'punit': punit, - **{k: tc[k] for k in group_by + extra_cols if k in tc} - } - key = _group_key(group_by, record) - grouped.setdefault(key, []) - grouped[key].append(record) - - return grouped - - -def _aggregate_perf(grouped_testcases, aggr_fn, cols): - other_aggr = _JoinUniqueValues('|') - aggr_data = {} - for key, seq in grouped_testcases.items(): - aggr_data.setdefault(key, {}) - aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq) - for c in cols: - aggr_data[key][c] = other_aggr( - nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c] - for tc in seq - ) - - return aggr_data - - -def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, - extra_group_by=None, extra_cols=None): - extra_group_by = extra_group_by or [] - extra_cols = extra_cols or [] - group_by = ['name', 'pvar', 'punit'] + extra_group_by - - grouped_base = _group_testcases(base_testcases, group_by, extra_cols) - grouped_target = _group_testcases(target_testcases, group_by, extra_cols) - pbase = _aggregate_perf(grouped_base, base_fn, extra_cols) - ptarget = _aggregate_perf(grouped_target, target_fn, []) - - # Build the final table data - data = [] - for key, aggr_data in pbase.items(): - pval = aggr_data['pval'] - try: - target_pval = ptarget[key]['pval'] - except KeyError: - pdiff = 'n/a' - else: - pdiff = (pval - target_pval) / target_pval - pdiff = '{:+7.2%}'.format(pdiff) - - name, pvar, punit, *extras = key - line = [name, pvar, pval, punit, pdiff, *extras] - # Add the extra columns - line += [aggr_data[c] for c in extra_cols] - data.append(line) - - return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] + - extra_group_by + extra_cols)) - - -class _Aggregator: - @classmethod - def create(cls, name): - if name == 'first': - return _First() - elif name == 'last': - return _Last() - elif name == 'mean': - return _Mean() - elif name == 'median': - return _Median() - elif name == 'min': - return _Min() - elif name == 'max': - return _Max() - else: - raise ValueError(f'unknown aggregation function: {name!r}') - - @abc.abstractmethod - def __call__(self, iterable): - pass - - -class _First(_Aggregator): - def __call__(self, iterable): - for i, elem in enumerate(iterable): - if i == 0: - return elem - - -class _Last(_Aggregator): - def __call__(self, iterable): - if not isinstance(iterable, types.GeneratorType): - return iterable[-1] - - for elem in iterable: - pass - - return elem - - -class _Mean(_Aggregator): - def __call__(self, iterable): - return statistics.mean(iterable) - - -class _Median(_Aggregator): - def __call__(self, iterable): - return statistics.median(iterable) - - -class _Min(_Aggregator): - def __call__(self, iterable): - return min(iterable) - - -class _Max(_Aggregator): - def __call__(self, iterable): - return max(iterable) - - -class _JoinUniqueValues(_Aggregator): - def __init__(self, delim): - self.__delim = delim - - def __call__(self, iterable): - unique_vals = {str(elem) for elem in iterable} - return self.__delim.join(unique_vals) - - -def _parse_timestamp(s): - if isinstance(s, Number): - return s - - now = datetime.now() - - def _do_parse(s): - if s == 'now': - return now - - formats = [r'%Y%m%d', r'%Y%m%dT%H%M', - r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] - for fmt in formats: - try: - return datetime.strptime(s, fmt) - except ValueError: - continue - - raise ValueError(f'invalid timestamp: {s}') - - try: - ts = _do_parse(s) - except ValueError as err: - # Try the relative timestamps - match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) - if not match: - raise err - - ts = _do_parse(match.group('ts')) - amount = int(match.group('amount')) - unit = match.group('unit') - if unit == 'd': - ts += timedelta(days=amount) - elif unit == 'm': - ts += timedelta(minutes=amount) - elif unit == 'h': - ts += timedelta(hours=amount) - elif unit == 's': - ts += timedelta(seconds=amount) - - return ts.timestamp() - - -def _parse_time_period(s): - if s.startswith('^'): - # Retrieve the period of a full session - try: - session_uuid = s[1:] - except IndexError: - raise ValueError(f'invalid session uuid: {s}') from None - else: - ts_start, ts_end = _fetch_session_time_period( - f'session_uuid == "{session_uuid}"' - ) - if not ts_start or not ts_end: - raise ValueError(f'no such session: {session_uuid}') - else: - try: - ts_start, ts_end = s.split(':') - except ValueError: - raise ValueError(f'invalid time period spec: {s}') from None - - return _parse_timestamp(ts_start), _parse_timestamp(ts_end) - - -def _parse_extra_cols(s): - try: - extra_cols = s.split('+')[1:] - except (ValueError, IndexError): - raise ValueError(f'invalid extra groups spec: {s}') from None - - return extra_cols - - -def _parse_aggregation(s): - try: - op, extra_groups = s.split(':') - except ValueError: - raise ValueError(f'invalid aggregate function spec: {s}') from None - - return _Aggregator.create(op), _parse_extra_cols(extra_groups) - - -_Match = namedtuple('_Match', ['period_base', 'period_target', - 'aggregator', 'extra_groups', 'extra_cols']) - - -def parse_cmp_spec(spec): - parts = spec.split('/') - if len(parts) == 3: - period_base, period_target, aggr, cols = None, *parts - elif len(parts) == 4: - period_base, period_target, aggr, cols = parts - else: - raise ValueError(f'invalid cmp spec: {spec}') - - if period_base is not None: - period_base = _parse_time_period(period_base) - - period_target = _parse_time_period(period_target) - aggr_fn, extra_groups = _parse_aggregation(aggr) - extra_cols = _parse_extra_cols(cols) - return _Match(period_base, period_target, - aggr_fn, extra_groups, extra_cols) From ed4f728c3d0a0f9d7af647529cbf3bbce8643ac1 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 16:22:21 +0200 Subject: [PATCH 10/69] Add missing file --- reframe/frontend/reporting/storage.py | 174 ++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 reframe/frontend/reporting/storage.py diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py new file mode 100644 index 0000000000..66cea655c8 --- /dev/null +++ b/reframe/frontend/reporting/storage.py @@ -0,0 +1,174 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import abc +import os +import sqlite3 +from datetime import datetime + +import reframe.utility.jsonext as jsonext +import reframe.utility.osext as osext +from reframe.core.logging import getlogger +from reframe.core.runtime import runtime + + +class StorageBackend: + '''Abstract class that represents the results backend storage''' + + @classmethod + def create(cls, backend, *args, **kwargs): + '''Factory method for creating storage backends''' + if backend == 'sqlite': + return _SqliteStorage(*args, **kwargs) + else: + raise NotImplementedError + + @classmethod + def default(cls): + '''Return default storage backend''' + return cls.create('sqlite') + + @abc.abstractmethod + def store(self, report, report_file): + '''Store the given report''' + + @abc.abstractmethod + def fetch_session_time_period(self, session_uuid): + '''Fetch the time period from specific session''' + + @abc.abstractmethod + def fetch_testcases_time_period(self, ts_start, ts_end): + '''Fetch all test cases from specified period''' + + +class _SqliteStorage(StorageBackend): + def __init__(self): + site_config = runtime().site_config + prefix = os.path.dirname(osext.expandvars( + site_config.get('general/0/report_file') + )) + self.__db_file = os.path.join(prefix, 'results.db') + + def _db_file(self): + prefix = os.path.dirname(self.__db_file) + if not os.path.exists(self.__db_file): + # Create subdirs if needed + if prefix: + os.makedirs(prefix, exist_ok=True) + + self._db_create() + + return self.__db_file + + def _db_create(self): + clsname = type(self).__name__ + getlogger().debug( + f'{clsname}: creating results database in {self.__db_file}...' + ) + with sqlite3.connect(self.__db_file) as conn: + conn.execute('CREATE TABLE IF NOT EXISTS sessions(' + 'id INTEGER PRIMARY KEY, ' + 'session_uuid TEXT, ' + 'session_start_unix REAL, ' + 'session_end_unix REAL, ' + 'json_blob TEXT, ' + 'report_file TEXT)') + conn.execute('CREATE TABLE IF NOT EXISTS testcases(' + 'name TEXT,' + 'system TEXT,' + 'partition TEXT,' + 'environ TEXT,' + 'job_completion_time_unix REAL,' + 'session_id INTEGER,' + 'run_index INTEGER,' + 'test_index INTEGER,' + 'FOREIGN KEY(session_id) ' + 'REFERENCES sessions(session_id))') + + def _db_store_report(self, conn, report, report_file_path): + session_start_unix = report['session_info']['time_start_unix'] + session_end_unix = report['session_info']['time_end_unix'] + session_uuid = datetime.fromtimestamp(session_start_unix).strftime( + r'%Y%m%dT%H%M%S%z' + ) + for run_idx, run in enumerate(report['runs']): + for test_idx, testcase in enumerate(run['testcases']): + sys, part = testcase['system'], testcase['partition'] + cursor = conn.execute( + 'INSERT INTO sessions VALUES(' + ':session_id, :session_uuid, ' + ':session_start_unix, :session_end_unix, ' + ':json_blob, :report_file)', + { + 'session_id': None, + 'session_uuid': session_uuid, + 'session_start_unix': session_start_unix, + 'session_end_unix': session_end_unix, + 'json_blob': jsonext.dumps(report), + 'report_file': report_file_path + } + ) + conn.execute( + 'INSERT INTO testcases VALUES(' + ':name, :system, :partition, :environ, ' + ':job_completion_time_unix, ' + ':session_id, :run_index, :test_index)', + { + 'name': testcase['name'], + 'system': sys, + 'partition': part, + 'environ': testcase['environ'], + 'job_completion_time_unix': testcase[ + 'job_completion_time_unix' + ], + 'session_id': cursor.lastrowid, + 'run_index': run_idx, + 'test_index': test_idx + } + ) + + return session_uuid + + def store(self,report, report_file): + with sqlite3.connect(self._db_file()) as conn: + return self._db_store_report(conn, report, report_file) + + def _fetch_testcases_raw(self, condition): + with sqlite3.connect(self._db_file()) as conn: + query = (f'SELECT session_id, run_index, test_index, json_blob FROM ' + f'testcases JOIN sessions ON session_id==id ' + f'WHERE {condition}') + getlogger().debug(query) + results = conn.execute(query).fetchall() + + # Retrieve files + testcases = [] + sessions = {} + for session_id, run_index, test_index, json_blob in results: + report = jsonext.loads(sessions.setdefault(session_id, json_blob)) + testcases.append(report['runs'][run_index]['testcases'][test_index]) + + return testcases + + def fetch_session_time_period(self, session_uuid): + with sqlite3.connect(self._db_file()) as conn: + query = ('SELECT session_start_unix, session_end_unix ' + f'FROM sessions WHERE session_uuid == "{session_uuid}" ' + 'LIMIT 1') + getlogger().debug(query) + results = conn.execute(query).fetchall() + if results: + return results[0] + + return None, None + + def fetch_testcases_time_period(self, ts_start, ts_end): + return self._fetch_testcases_raw( + f'(job_completion_time_unix >= {ts_start} AND ' + f'job_completion_time_unix <= {ts_end}) ' + 'ORDER BY job_completion_time_unix' + ) + + From f68a092209b5d88b4e672385e78d95fc50f47d39 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 12:04:54 +0200 Subject: [PATCH 11/69] Enhance printer to print tabulated data --- reframe/frontend/cli.py | 11 ++++------- reframe/frontend/printer.py | 7 +++++++ reframe/frontend/reporting/__init__.py | 12 ++++++------ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index a7be337061..fe8d7d449a 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -12,7 +12,6 @@ import sys import time import traceback -from tabulate import tabulate import reframe.core.config as config import reframe.core.exceptions as errors @@ -1297,10 +1296,9 @@ def _sort_testcases(testcases): sys.exit(0) if options.performance_compare: - data, header = reporting.performance_compare( - options.performance_compare + printer.table( + *reporting.performance_compare(options.performance_compare) ) - printer(tabulate(data, headers=header, tablefmt='mixed_grid')) sys.exit(0) if not options.run and not options.dry_run: @@ -1475,9 +1473,8 @@ def module_unuse(*paths): f'failed to generate performance report: {err}' ) else: - print( - tabulate(data, headers=header, tablefmt='mixed_grid') - ) + printer.table(data, header) + # printer.info(runner.stats.performance_report()) # Generate the report for this session diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index d94444fade..6dac37713c 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -7,6 +7,8 @@ import shutil import time import traceback +from tabulate import tabulate + import reframe.core.logging as logging import reframe.core.runtime as rt import reframe.utility.color as color @@ -245,3 +247,8 @@ def retry_report(self, report): lines.append(msg) self.info('\n'.join(lines)) + + def table(self, data, header): + '''Print a table''' + + self.info(tabulate(data, headers=header, tablefmt='mixed_grid')) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 6a96d73afb..85b50e0cc8 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -47,7 +47,7 @@ def format_testcase(json, name='unique_name'): return f'{name}@{system}:{partition}+{environ}' -_ReportStorage = StorageBackend.create('sqlite') +_ReportStorage = functools.partial(StorageBackend.create, backend='sqlite') class _RestoredSessionInfo: @@ -400,7 +400,7 @@ def save(self, filename, compress=False, link_to_last=True): def store(self): '''Store the report in the results storage.''' - return _ReportStorage.store(self, self.filename) + return _ReportStorage().store(self, self.filename) def generate_xml_report(self): '''Generate a JUnit report from a standard ReFrame JSON report.''' @@ -671,8 +671,8 @@ def _parse_time_period(s): except IndexError: raise ValueError(f'invalid session uuid: {s}') from None else: - ts_start, ts_end = _ReportStorage.fetch_session_time_period( - f'session_uuid == "{session_uuid}"' + ts_start, ts_end = _ReportStorage().fetch_session_time_period( + session_uuid ) if not ts_start or not ts_end: raise ValueError(f'no such session: {session_uuid}') @@ -738,11 +738,11 @@ def performance_compare(cmp, report=None): tcs_base = [] else: - tcs_base = _ReportStorage.fetch_testcases_time_period( + tcs_base = _ReportStorage().fetch_testcases_time_period( *match.period_base ) - tcs_target = _ReportStorage.fetch_testcases_time_period( + tcs_target = _ReportStorage().fetch_testcases_time_period( *match.period_target ) return compare_testcase_data(tcs_base, tcs_target, match.aggregator, From 77aac04d4b7818900f91f6353b7048b224240c00 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 15:49:43 +0200 Subject: [PATCH 12/69] Move comparison spec code to a separate utility package --- reframe/frontend/reporting/__init__.py | 184 +----------------------- reframe/frontend/reporting/utility.py | 192 +++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 182 deletions(-) create mode 100644 reframe/frontend/reporting/utility.py diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 85b50e0cc8..7092861350 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -14,13 +14,8 @@ import os import re import socket -import statistics import time -import types -from collections import namedtuple from collections.abc import Hashable -from datetime import datetime, timedelta -from numbers import Number import reframe as rfm import reframe.utility.jsonext as jsonext @@ -30,6 +25,7 @@ from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev from .storage import StorageBackend +from .utility import Aggregator, parse_cmp_spec # The schema data version # Major version bumps are expected to break the validation of previous schemas @@ -503,7 +499,7 @@ def _group_testcases(testcases, group_by, extra_cols): def _aggregate_perf(grouped_testcases, aggr_fn, cols): - other_aggr = _JoinUniqueValues('|') + other_aggr = Aggregator.create('join_uniq', '|') aggr_data = {} for key, seq in grouped_testcases.items(): aggr_data.setdefault(key, {}) @@ -550,182 +546,6 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, extra_group_by + extra_cols)) -class _Aggregator: - @classmethod - def create(cls, name): - if name == 'first': - return _First() - elif name == 'last': - return _Last() - elif name == 'mean': - return _Mean() - elif name == 'median': - return _Median() - elif name == 'min': - return _Min() - elif name == 'max': - return _Max() - else: - raise ValueError(f'unknown aggregation function: {name!r}') - - @abc.abstractmethod - def __call__(self, iterable): - pass - - -class _First(_Aggregator): - def __call__(self, iterable): - for i, elem in enumerate(iterable): - if i == 0: - return elem - - -class _Last(_Aggregator): - def __call__(self, iterable): - if not isinstance(iterable, types.GeneratorType): - return iterable[-1] - - for elem in iterable: - pass - - return elem - - -class _Mean(_Aggregator): - def __call__(self, iterable): - return statistics.mean(iterable) - - -class _Median(_Aggregator): - def __call__(self, iterable): - return statistics.median(iterable) - - -class _Min(_Aggregator): - def __call__(self, iterable): - return min(iterable) - - -class _Max(_Aggregator): - def __call__(self, iterable): - return max(iterable) - - -class _JoinUniqueValues(_Aggregator): - def __init__(self, delim): - self.__delim = delim - - def __call__(self, iterable): - unique_vals = {str(elem) for elem in iterable} - return self.__delim.join(unique_vals) - - -def _parse_timestamp(s): - if isinstance(s, Number): - return s - - now = datetime.now() - - def _do_parse(s): - if s == 'now': - return now - - formats = [r'%Y%m%d', r'%Y%m%dT%H%M', - r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] - for fmt in formats: - try: - return datetime.strptime(s, fmt) - except ValueError: - continue - - raise ValueError(f'invalid timestamp: {s}') - - try: - ts = _do_parse(s) - except ValueError as err: - # Try the relative timestamps - match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) - if not match: - raise err - - ts = _do_parse(match.group('ts')) - amount = int(match.group('amount')) - unit = match.group('unit') - if unit == 'd': - ts += timedelta(days=amount) - elif unit == 'm': - ts += timedelta(minutes=amount) - elif unit == 'h': - ts += timedelta(hours=amount) - elif unit == 's': - ts += timedelta(seconds=amount) - - return ts.timestamp() - - -def _parse_time_period(s): - if s.startswith('^'): - # Retrieve the period of a full session - try: - session_uuid = s[1:] - except IndexError: - raise ValueError(f'invalid session uuid: {s}') from None - else: - ts_start, ts_end = _ReportStorage().fetch_session_time_period( - session_uuid - ) - if not ts_start or not ts_end: - raise ValueError(f'no such session: {session_uuid}') - else: - try: - ts_start, ts_end = s.split(':') - except ValueError: - raise ValueError(f'invalid time period spec: {s}') from None - - return _parse_timestamp(ts_start), _parse_timestamp(ts_end) - - -def _parse_extra_cols(s): - try: - extra_cols = s.split('+')[1:] - except (ValueError, IndexError): - raise ValueError(f'invalid extra groups spec: {s}') from None - - return extra_cols - - -def _parse_aggregation(s): - try: - op, extra_groups = s.split(':') - except ValueError: - raise ValueError(f'invalid aggregate function spec: {s}') from None - - return _Aggregator.create(op), _parse_extra_cols(extra_groups) - - -_Match = namedtuple('_Match', ['period_base', 'period_target', - 'aggregator', 'extra_groups', 'extra_cols']) - - -def parse_cmp_spec(spec): - parts = spec.split('/') - if len(parts) == 3: - period_base, period_target, aggr, cols = None, *parts - elif len(parts) == 4: - period_base, period_target, aggr, cols = parts - else: - raise ValueError(f'invalid cmp spec: {spec}') - - if period_base is not None: - period_base = _parse_time_period(period_base) - - period_target = _parse_time_period(period_target) - aggr_fn, extra_groups = _parse_aggregation(aggr) - extra_cols = _parse_extra_cols(cols) - return _Match(period_base, period_target, - aggr_fn, extra_groups, extra_cols) - - def performance_compare(cmp, report=None): match = parse_cmp_spec(cmp) if match.period_base is None: diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py new file mode 100644 index 0000000000..9cbc1e6666 --- /dev/null +++ b/reframe/frontend/reporting/utility.py @@ -0,0 +1,192 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import abc +import re +import statistics +import types +from collections import namedtuple +from datetime import datetime, timedelta +from numbers import Number +from .storage import StorageBackend + +class Aggregator: + @classmethod + def create(cls, name, *args, **kwargs): + if name == 'first': + return AggrFirst(*args, **kwargs) + elif name == 'last': + return AggrLast(*args, **kwargs) + elif name == 'mean': + return AggrMean(*args, **kwargs) + elif name == 'median': + return AggrMedian(*args, **kwargs) + elif name == 'min': + return AggrMin(*args, **kwargs) + elif name == 'max': + return AggrMax(*args, **kwargs) + elif name == 'join_uniq': + return AggrJoinUniqueValues(*args, **kwargs) + else: + raise ValueError(f'unknown aggregation function: {name!r}') + + @abc.abstractmethod + def __call__(self, iterable): + pass + + +class AggrFirst(Aggregator): + def __call__(self, iterable): + for i, elem in enumerate(iterable): + if i == 0: + return elem + + +class AggrLast(Aggregator): + def __call__(self, iterable): + if not isinstance(iterable, types.GeneratorType): + return iterable[-1] + + for elem in iterable: + pass + + return elem + + +class AggrMean(Aggregator): + def __call__(self, iterable): + return statistics.mean(iterable) + + +class AggrMedian(Aggregator): + def __call__(self, iterable): + return statistics.median(iterable) + + +class AggrMin(Aggregator): + def __call__(self, iterable): + return min(iterable) + + +class AggrMax(Aggregator): + def __call__(self, iterable): + return max(iterable) + + +class AggrJoinUniqueValues(Aggregator): + def __init__(self, delim): + self.__delim = delim + + def __call__(self, iterable): + unique_vals = {str(elem) for elem in iterable} + return self.__delim.join(unique_vals) + + +def _parse_timestamp(s): + if isinstance(s, Number): + return s + + now = datetime.now() + + def _do_parse(s): + if s == 'now': + return now + + formats = [r'%Y%m%d', r'%Y%m%dT%H%M', + r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z'] + for fmt in formats: + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + + raise ValueError(f'invalid timestamp: {s}') + + try: + ts = _do_parse(s) + except ValueError as err: + # Try the relative timestamps + match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) + if not match: + raise err + + ts = _do_parse(match.group('ts')) + amount = int(match.group('amount')) + unit = match.group('unit') + if unit == 'd': + ts += timedelta(days=amount) + elif unit == 'm': + ts += timedelta(minutes=amount) + elif unit == 'h': + ts += timedelta(hours=amount) + elif unit == 's': + ts += timedelta(seconds=amount) + + return ts.timestamp() + + +def _parse_time_period(s): + if s.startswith('^'): + # Retrieve the period of a full session + try: + session_uuid = s[1:] + except IndexError: + raise ValueError(f'invalid session uuid: {s}') from None + else: + backend = StorageBackend('sqlite') + ts_start, ts_end = backend.fetch_session_time_period( + session_uuid + ) + if not ts_start or not ts_end: + raise ValueError(f'no such session: {session_uuid}') + else: + try: + ts_start, ts_end = s.split(':') + except ValueError: + raise ValueError(f'invalid time period spec: {s}') from None + + return _parse_timestamp(ts_start), _parse_timestamp(ts_end) + + +def _parse_extra_cols(s): + try: + extra_cols = s.split('+')[1:] + except (ValueError, IndexError): + raise ValueError(f'invalid extra groups spec: {s}') from None + + return extra_cols + + +def _parse_aggregation(s): + try: + op, extra_groups = s.split(':') + except ValueError: + raise ValueError(f'invalid aggregate function spec: {s}') from None + + return Aggregator.create(op), _parse_extra_cols(extra_groups) + + +_Match = namedtuple('_Match', ['period_base', 'period_target', + 'aggregator', 'extra_groups', 'extra_cols']) + + +def parse_cmp_spec(spec): + parts = spec.split('/') + if len(parts) == 3: + period_base, period_target, aggr, cols = None, *parts + elif len(parts) == 4: + period_base, period_target, aggr, cols = parts + else: + raise ValueError(f'invalid cmp spec: {spec}') + + if period_base is not None: + period_base = _parse_time_period(period_base) + + period_target = _parse_time_period(period_target) + aggr_fn, extra_groups = _parse_aggregation(aggr) + extra_cols = _parse_extra_cols(cols) + return _Match(period_base, period_target, + aggr_fn, extra_groups, extra_cols) + From 0b52501c2ad91fbc50a032e62d53e563c68b5c0e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 15:56:03 +0200 Subject: [PATCH 13/69] Improve access to default storage backend --- reframe/frontend/reporting/__init__.py | 9 +++------ reframe/frontend/reporting/utility.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 7092861350..28b580fb45 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -43,9 +43,6 @@ def format_testcase(json, name='unique_name'): return f'{name}@{system}:{partition}+{environ}' -_ReportStorage = functools.partial(StorageBackend.create, backend='sqlite') - - class _RestoredSessionInfo: '''A restored session with some additional functionality.''' @@ -396,7 +393,7 @@ def save(self, filename, compress=False, link_to_last=True): def store(self): '''Store the report in the results storage.''' - return _ReportStorage().store(self, self.filename) + return StorageBackend.default().store(self, self.filename) def generate_xml_report(self): '''Generate a JUnit report from a standard ReFrame JSON report.''' @@ -558,11 +555,11 @@ def performance_compare(cmp, report=None): tcs_base = [] else: - tcs_base = _ReportStorage().fetch_testcases_time_period( + tcs_base = StorageBackend.default().fetch_testcases_time_period( *match.period_base ) - tcs_target = _ReportStorage().fetch_testcases_time_period( + tcs_target = StorageBackend.default().fetch_testcases_time_period( *match.period_target ) return compare_testcase_data(tcs_base, tcs_target, match.aggregator, diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 9cbc1e6666..2d9a839968 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -135,7 +135,7 @@ def _parse_time_period(s): except IndexError: raise ValueError(f'invalid session uuid: {s}') from None else: - backend = StorageBackend('sqlite') + backend = StorageBackend.default() ts_start, ts_end = backend.fetch_session_time_period( session_uuid ) From 5a5b3812538dea7cfca69a2b753a89c9cc817a79 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 16:11:10 +0200 Subject: [PATCH 14/69] Add configuration option for specifying the perf. report spec --- reframe/frontend/cli.py | 8 ++++++-- reframe/schemas/config.json | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index fe8d7d449a..b6f38674f8 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -544,7 +544,11 @@ def main(): ) reporting_options.add_argument( '--performance-report', action='store', nargs='?', - const='19700101T0000Z:now/last:+job_nodelist/+result', + # a non-empty (unused) token to ensure that option will be set + # even if no argument is passed + const='', + configvar='general/0/perf_report_spec', + envvar='RFM_PERF_REPORT_SPEC', help='Print a report for performance tests' ) @@ -1466,7 +1470,7 @@ def module_unuse(*paths): if options.performance_report and not options.dry_run: try: data, header = reporting.performance_compare( - options.performance_report, report + rt.get_option('general/0/perf_report_spec'), report ) except errors.ReframeError as err: printer.warning( diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 2dcbec053c..bcea54e185 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -514,6 +514,7 @@ "non_default_craype": {"type": "boolean"}, "dump_pipeline_progress": {"type": "boolean"}, "perf_info_level": {"$ref": "#/defs/loglevel"}, + "perf_report_spec": {"type": "string"}, "pipeline_timeout": {"type": ["number", "null"]}, "purge_environment": {"type": "boolean"}, "remote_detect": {"type": "boolean"}, @@ -567,6 +568,7 @@ "general/module_mappings": [], "general/non_default_craype": false, "general/perf_info_level": "info", + "general/perf_report_spec": "19700101T0000Z:now/last:+job_nodelist/+result", "general/purge_environment": false, "general/remote_detect": false, "general/remote_workdir": ".", From 864917ccf73807949b9825b7eec16bd8907044f3 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 17:16:56 +0200 Subject: [PATCH 15/69] Remove unused import --- reframe/frontend/reporting/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 28b580fb45..bde0857e63 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -import abc import decimal import functools import inspect From 3137f15044205a70749111cdd626f8a6822a0ed0 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 1 Jul 2024 17:30:11 +0200 Subject: [PATCH 16/69] Python 3.6 compatible default value for `perf_report_spec` --- reframe/schemas/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index bcea54e185..3a9d6d80f5 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -568,7 +568,7 @@ "general/module_mappings": [], "general/non_default_craype": false, "general/perf_info_level": "info", - "general/perf_report_spec": "19700101T0000Z:now/last:+job_nodelist/+result", + "general/perf_report_spec": "19700101T0000+0000:now/last:+job_nodelist/+result", "general/purge_environment": false, "general/remote_detect": false, "general/remote_workdir": ".", From 15dd76cf0d8a6c4a1e5a88fe30d63e873f79c7af Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 2 Jul 2024 10:18:55 +0200 Subject: [PATCH 17/69] Fix coding style issues --- reframe/frontend/reporting/storage.py | 46 +++++++++++++-------------- reframe/frontend/reporting/utility.py | 2 +- setup.cfg | 3 +- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 66cea655c8..0422539713 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -69,23 +69,23 @@ def _db_create(self): ) with sqlite3.connect(self.__db_file) as conn: conn.execute('CREATE TABLE IF NOT EXISTS sessions(' - 'id INTEGER PRIMARY KEY, ' - 'session_uuid TEXT, ' - 'session_start_unix REAL, ' - 'session_end_unix REAL, ' - 'json_blob TEXT, ' - 'report_file TEXT)') + 'id INTEGER PRIMARY KEY, ' + 'session_uuid TEXT, ' + 'session_start_unix REAL, ' + 'session_end_unix REAL, ' + 'json_blob TEXT, ' + 'report_file TEXT)') conn.execute('CREATE TABLE IF NOT EXISTS testcases(' - 'name TEXT,' - 'system TEXT,' - 'partition TEXT,' - 'environ TEXT,' - 'job_completion_time_unix REAL,' - 'session_id INTEGER,' - 'run_index INTEGER,' - 'test_index INTEGER,' - 'FOREIGN KEY(session_id) ' - 'REFERENCES sessions(session_id))') + 'name TEXT,' + 'system TEXT,' + 'partition TEXT,' + 'environ TEXT,' + 'job_completion_time_unix REAL,' + 'session_id INTEGER,' + 'run_index INTEGER,' + 'test_index INTEGER,' + 'FOREIGN KEY(session_id) ' + 'REFERENCES sessions(session_id))') def _db_store_report(self, conn, report, report_file_path): session_start_unix = report['session_info']['time_start_unix'] @@ -131,15 +131,15 @@ def _db_store_report(self, conn, report, report_file_path): return session_uuid - def store(self,report, report_file): + def store(self, report, report_file): with sqlite3.connect(self._db_file()) as conn: return self._db_store_report(conn, report, report_file) def _fetch_testcases_raw(self, condition): with sqlite3.connect(self._db_file()) as conn: - query = (f'SELECT session_id, run_index, test_index, json_blob FROM ' - f'testcases JOIN sessions ON session_id==id ' - f'WHERE {condition}') + query = ('SELECT session_id, run_index, test_index, json_blob ' + 'FROM testcases JOIN sessions ON session_id==id ' + f'WHERE {condition}') getlogger().debug(query) results = conn.execute(query).fetchall() @@ -148,7 +148,9 @@ def _fetch_testcases_raw(self, condition): sessions = {} for session_id, run_index, test_index, json_blob in results: report = jsonext.loads(sessions.setdefault(session_id, json_blob)) - testcases.append(report['runs'][run_index]['testcases'][test_index]) + testcases.append( + report['runs'][run_index]['testcases'][test_index] + ) return testcases @@ -170,5 +172,3 @@ def fetch_testcases_time_period(self, ts_start, ts_end): f'job_completion_time_unix <= {ts_end}) ' 'ORDER BY job_completion_time_unix' ) - - diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 2d9a839968..05cc0cf337 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -12,6 +12,7 @@ from numbers import Number from .storage import StorageBackend + class Aggregator: @classmethod def create(cls, name, *args, **kwargs): @@ -189,4 +190,3 @@ def parse_cmp_spec(spec): extra_cols = _parse_extra_cols(cols) return _Match(period_base, period_target, aggr_fn, extra_groups, extra_cols) - diff --git a/setup.cfg b/setup.cfg index 5ae3959849..974e38855d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,4 +47,5 @@ include = reframe,reframe.*,hpctestlib.* reframe = schemas/* [flake8] -ignore = E129,E221,E226,E241,E402,E272,E741,E742,E743,W504 +extend-ignore = E129,E221,E226,E241,E402,E272,E741,E742,E743,F821,W504 +exclude = .git,__pycache__,docs/conf.py,external From 2ca850c855dd624321f9cd1a23b6aa2e810efe46 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 2 Jul 2024 11:54:06 +0200 Subject: [PATCH 18/69] Improve formatting of performance report --- reframe/frontend/cli.py | 4 +--- reframe/frontend/printer.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index b6f38674f8..9d55d34541 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1477,9 +1477,7 @@ def module_unuse(*paths): f'failed to generate performance report: {err}' ) else: - printer.table(data, header) - - # printer.info(runner.stats.performance_report()) + printer.performance_report(data, header) # Generate the report for this session report_file = os.path.normpath( diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index 6dac37713c..2409637a20 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -159,9 +159,8 @@ def _print_failure_info(rec, runid, total_runs): else: self.verbose(tb) - line_width = shutil.get_terminal_size()[0] - self.info(line_width * '=') - self.info('SUMMARY OF FAILURES') + line_width = min(80, shutil.get_terminal_size()[0]) + self.info(' SUMMARY OF FAILURES '.center(line_width, '=')) for run_no, run_info in enumerate(report['runs'], start=1): if not global_stats and run_no != len(report['runs']): @@ -248,6 +247,14 @@ def retry_report(self, report): self.info('\n'.join(lines)) + def performance_report(self, data, header): + width = min(80, shutil.get_terminal_size()[0]) + self.info('') + self.info(' PERFORMANCE REPORT '.center(width, '=')) + self.info('') + self.table(data, header) + self.info('') + def table(self, data, header): '''Print a table''' From 9d6d1bba6bb2dbfec441a21a9195fac75295765c Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 8 Jul 2024 17:15:19 +0300 Subject: [PATCH 19/69] Improve error handling and fix handling of `perf_report_spec` --- reframe/frontend/argparse.py | 2 +- reframe/frontend/cli.py | 28 ++++++++++++++------------ reframe/frontend/reporting/__init__.py | 7 +++++-- reframe/schemas/config.json | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py index e6c37b7484..82c86af1a3 100644 --- a/reframe/frontend/argparse.py +++ b/reframe/frontend/argparse.py @@ -127,7 +127,7 @@ def update_config(self, site_config): def __repr__(self): return (f'{type(self).__name__}({self.__namespace!r}, ' - '{self.__option_map})') + f'{self.__option_map})') class _ArgumentHolder: diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 9d55d34541..8c0058bc09 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -293,10 +293,9 @@ def main(): envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) output_options.add_argument( - '--timestamp', action='store', nargs='?', const='%y%m%dT%H%M%S%z', - metavar='TIMEFMT', + '--timestamp', action='store', nargs='?', metavar='TIMEFMT', help=('Append a timestamp to the output and stage directory prefixes ' - '(default: "%%FT%%T")'), + r'(default: "%y%m%dT%H%M%S%z")'), envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' ) @@ -544,12 +543,10 @@ def main(): ) reporting_options.add_argument( '--performance-report', action='store', nargs='?', - # a non-empty (unused) token to ensure that option will be set - # even if no argument is passed - const='', - configvar='general/0/perf_report_spec', + configvar='general/perf_report_spec', envvar='RFM_PERF_REPORT_SPEC', - help='Print a report for performance tests' + help=('Print a report for performance tests ' + '(default: "19700101T0000+0000:now/last:+job_nodelist/+result")') ) # Miscellaneous options @@ -1300,10 +1297,15 @@ def _sort_testcases(testcases): sys.exit(0) if options.performance_compare: - printer.table( - *reporting.performance_compare(options.performance_compare) - ) - sys.exit(0) + try: + printer.table( + *reporting.performance_compare(options.performance_compare) + ) + except (errors.ReframeError, ValueError) as err: + printer.error(f'failed to generate performance report: {err}') + sys.exit(1) + else: + sys.exit(0) if not options.run and not options.dry_run: printer.error("No action option specified. Available options:\n" @@ -1472,7 +1474,7 @@ def module_unuse(*paths): data, header = reporting.performance_compare( rt.get_option('general/0/perf_report_spec'), report ) - except errors.ReframeError as err: + except (errors.ReframeError, ValueError) as err: printer.warning( f'failed to generate performance report: {err}' ) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index bde0857e63..e808dd4bf6 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -529,8 +529,11 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, except KeyError: pdiff = 'n/a' else: - pdiff = (pval - target_pval) / target_pval - pdiff = '{:+7.2%}'.format(pdiff) + if pval is None or target_pval is None: + pdiff = 'n/a' + else: + pdiff = (pval - target_pval) / target_pval + pdiff = '{:+7.2%}'.format(pdiff) name, pvar, punit, *extras = key line = [name, pvar, pval, punit, pdiff, *extras] diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 3a9d6d80f5..51189b2811 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -578,7 +578,7 @@ "general/resolve_module_conflicts": true, "general/save_log_files": false, "general/target_systems": ["*"], - "general/timestamp_dirs": "", + "general/timestamp_dirs": "%y%m%dT%H%M%S%z", "general/trap_job_errors": false, "general/unload_modules": [], "general/use_login_shell": false, From 0266495cc33eb5664734c9bfcb2477b8ec7463c3 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 8 Jul 2024 19:39:19 +0300 Subject: [PATCH 20/69] Fix handling of `const` optional argument in `ArgumentParser` --- reframe/core/runtime.py | 13 +++++++++---- reframe/frontend/argparse.py | 25 +++++++++++++++++++++++-- reframe/frontend/cli.py | 7 +++++-- unittests/test_argparser.py | 19 +++++++------------ 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index 784ba7c750..24d754b991 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -27,11 +27,12 @@ class RuntimeContext: .. versionadded:: 2.13 ''' - def __init__(self, site_config): + def __init__(self, site_config, *, use_timestamps=False): self._site_config = site_config self._system = System.create(site_config) self._current_run = 0 self._timestamp = time.localtime() + self._use_timestamps = use_timestamps def _makedir(self, *dirs, wipeout=False): ret = os.path.join(*dirs) @@ -110,7 +111,11 @@ def perflogdir(self): @property def timestamp(self): - timefmt = self.site_config.get('general/0/timestamp_dirs') + if self._use_timestamps: + timefmt = self.site_config.get('general/0/timestamp_dirs') + else: + timefmt = '' + return time.strftime(timefmt, self._timestamp) @property @@ -192,11 +197,11 @@ def get_default(self, option): _runtime_context = None -def init_runtime(site_config): +def init_runtime(site_config, **kwargs): global _runtime_context if _runtime_context is None: - _runtime_context = RuntimeContext(site_config) + _runtime_context = RuntimeContext(site_config, **kwargs) def runtime(): diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py index 82c86af1a3..ec6c667142 100644 --- a/reframe/frontend/argparse.py +++ b/reframe/frontend/argparse.py @@ -40,6 +40,24 @@ # that essentially associate environment variables with configuration # arguments, without having to define a corresponding command line option. +class _Undefined: + pass + + +# We use a special value for denoting const values that are to be set from the +# configuration default. This placeholder must be used as the `const` argument +# for options with `nargs='?'`. The underlying `ArugmentParser` will use the +# `const` value as if it were supplied from the command-line thus fooling our +# machinery of environment variables and configuration options overriding any +# defaults. For this reason, we use a unique placeholder so that we can +# distinguish whether this value is a default or actually supplied from the +# command-line. +CONST_DEFAULT = _Undefined() + + +def _undefined(val): + return val is None or val is CONST_DEFAULT + class _Namespace: def __init__(self, namespace, option_map): @@ -76,7 +94,10 @@ def __getattr__(self, name): return ret envvar, _, action, arg_type, default = self.__option_map[name] - if ret is None and envvar is not None: + if ret is CONST_DEFAULT: + default = CONST_DEFAULT + + if _undefined(ret) and envvar is not None: # Try the environment variable envvar, *delim = envvar.split(maxsplit=2) delim = delim[0] if delim else ',' @@ -120,7 +141,7 @@ def update_config(self, site_config): errors.append(e) continue - if value is not None: + if not _undefined(value): site_config.add_sticky_option(confvar, value) return errors diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 8c0058bc09..faa0a40674 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -294,8 +294,9 @@ def main(): ) output_options.add_argument( '--timestamp', action='store', nargs='?', metavar='TIMEFMT', + const=argparse.CONST_DEFAULT, help=('Append a timestamp to the output and stage directory prefixes ' - r'(default: "%y%m%dT%H%M%S%z")'), + '(default: "%%y%%m%%dT%%H%%M%%S%%z")'), envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' ) @@ -543,6 +544,7 @@ def main(): ) reporting_options.add_argument( '--performance-report', action='store', nargs='?', + const=argparse.CONST_DEFAULT, configvar='general/perf_report_spec', envvar='RFM_PERF_REPORT_SPEC', help=('Print a report for performance tests ' @@ -856,7 +858,8 @@ def restrict_logging(): try: printer.debug('Initializing runtime') - runtime.init_runtime(site_config) + runtime.init_runtime(site_config, + use_timestamps=options.timestamp is not None) except errors.ConfigError as e: printer.error(f'failed to initialize runtime: {e}') printer.info(logfiles_message()) diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py index 6647ed1e19..cd6fffea21 100644 --- a/unittests/test_argparser.py +++ b/unittests/test_argparser.py @@ -7,7 +7,7 @@ import reframe.core.runtime as rt import unittests.utility as test_util -from reframe.frontend.argparse import ArgumentParser +from reframe.frontend.argparse import ArgumentParser, CONST_DEFAULT @pytest.fixture @@ -118,7 +118,7 @@ def extended_parser(): default='bar' ) foo_options.add_argument( - '--timestamp', action='store', + '--timestamp', action='store', nargs='?', const=CONST_DEFAULT, envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' ) foo_options.add_argument( @@ -143,17 +143,14 @@ def extended_parser(): def test_option_precedence(default_exec_ctx, extended_parser): with rt.temp_environment(env_vars={ - 'RFM_TIMESTAMP': '%F', 'RFM_NON_DEFAULT_CRAYPE': 'yes', 'RFM_MODULES_PRELOAD': 'a,b,c', 'RFM_CHECK_SEARCH_PATH': 'x:y:z' }): - options = extended_parser.parse_args( - ['--timestamp=%FT%T', '--nocolor'] - ) + options = extended_parser.parse_args(['--nocolor', '--timestamp']) assert options.recursive is None - assert options.timestamp == '%FT%T' + assert options.timestamp is CONST_DEFAULT assert options.non_default_craype is True assert options.config_file is None assert options.prefix is None @@ -165,19 +162,17 @@ def test_option_precedence(default_exec_ctx, extended_parser): def test_option_with_config(default_exec_ctx, extended_parser, tmp_path): with rt.temp_environment(env_vars={ - 'RFM_TIMESTAMP': '%F', + 'RFM_TIMESTAMP_DIRS': r'%F', 'RFM_NON_DEFAULT_CRAYPE': 'yes', 'RFM_MODULES_PRELOAD': 'a,b,c', 'RFM_KEEP_STAGE_FILES': 'no', 'RFM_GIT_TIMEOUT': '0.3' }): site_config = rt.runtime().site_config - options = extended_parser.parse_args( - ['--timestamp=%FT%T', '--nocolor'] - ) + options = extended_parser.parse_args(['--nocolor', '--timestamp']) options.update_config(site_config) assert site_config.get('general/0/check_search_recursive') is False - assert site_config.get('general/0/timestamp_dirs') == '%FT%T' + assert site_config.get('general/0/timestamp_dirs') == r'%F' assert site_config.get('general/0/non_default_craype') is True assert site_config.get('systems/0/prefix') == str(tmp_path) assert site_config.get('general/0/colorize') is False From 9b68376fb8747b8446b5cbbb069adf5fe82191c7 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 10 Jul 2024 15:46:47 +0300 Subject: [PATCH 21/69] Synchronize storing to the DB and saving the report file --- reframe/frontend/reporting/__init__.py | 12 +++++++++-- reframe/frontend/reporting/storage.py | 28 +++++++++++++------------- requirements.txt | 3 +++ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index e808dd4bf6..b8e0d63538 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -15,6 +15,7 @@ import socket import time from collections.abc import Hashable +from filelock import FileLock import reframe as rfm import reframe.utility.jsonext as jsonext @@ -362,7 +363,7 @@ def update_run_stats(self, stats): 'num_skipped': self.__report['runs'][-1]['num_skipped'] }) - def save(self, filename, compress=False, link_to_last=True): + def _save(self, filename, compress, link_to_last): filename = _expand_report_filename(filename, newfile=True) with open(filename, 'w') as fp: if compress: @@ -389,10 +390,17 @@ def save(self, filename, compress=False, link_to_last=True): else: raise ReframeError('path exists and is not a symlink') + def save(self, filename, compress=False, link_to_last=True): + prefix = os.path.dirname(filename) or '.' + with FileLock(os.path.join(prefix, '.report.lock')): + self._save(filename, compress, link_to_last) + def store(self): '''Store the report in the results storage.''' - return StorageBackend.default().store(self, self.filename) + prefix = os.path.dirname(self.filename) or '.' + with FileLock(os.path.join(prefix, '.db.lock')): + return StorageBackend.default().store(self, self.filename) def generate_xml_report(self): '''Generate a JUnit report from a standard ReFrame JSON report.''' diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 0422539713..af99b6ff7b 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -93,23 +93,23 @@ def _db_store_report(self, conn, report, report_file_path): session_uuid = datetime.fromtimestamp(session_start_unix).strftime( r'%Y%m%dT%H%M%S%z' ) + cursor = conn.execute( + 'INSERT INTO sessions VALUES(' + ':session_id, :session_uuid, ' + ':session_start_unix, :session_end_unix, ' + ':json_blob, :report_file)', + { + 'session_id': None, + 'session_uuid': session_uuid, + 'session_start_unix': session_start_unix, + 'session_end_unix': session_end_unix, + 'json_blob': jsonext.dumps(report), + 'report_file': report_file_path + } + ) for run_idx, run in enumerate(report['runs']): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] - cursor = conn.execute( - 'INSERT INTO sessions VALUES(' - ':session_id, :session_uuid, ' - ':session_start_unix, :session_end_unix, ' - ':json_blob, :report_file)', - { - 'session_id': None, - 'session_uuid': session_uuid, - 'session_start_unix': session_start_unix, - 'session_end_unix': session_end_unix, - 'json_blob': jsonext.dumps(report), - 'report_file': report_file_path - } - ) conn.execute( 'INSERT INTO testcases VALUES(' ':name, :system, :partition, :environ, ' diff --git a/requirements.txt b/requirements.txt index e6df345eb9..790dd8cc60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ archspec==0.2.4 argcomplete==3.1.2; python_version < '3.8' argcomplete==3.4.0; python_version >= '3.8' +filelock==3.4.1; python_version == '3.6' +filelock==3.12.2; python_version == '3.7' +filelock==3.15.4; python_version >= '3.8' importlib_metadata==4.0.1; python_version < '3.8' jsonschema==3.2.0 lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64' From 1108fcf97aedd47b509144daf11a28cfacaa59b5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 10 Jul 2024 16:28:22 +0300 Subject: [PATCH 22/69] Use session ID as UUID --- reframe/frontend/reporting/storage.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index af99b6ff7b..c626ff8f1a 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -70,7 +70,6 @@ def _db_create(self): with sqlite3.connect(self.__db_file) as conn: conn.execute('CREATE TABLE IF NOT EXISTS sessions(' 'id INTEGER PRIMARY KEY, ' - 'session_uuid TEXT, ' 'session_start_unix REAL, ' 'session_end_unix REAL, ' 'json_blob TEXT, ' @@ -90,23 +89,19 @@ def _db_create(self): def _db_store_report(self, conn, report, report_file_path): session_start_unix = report['session_info']['time_start_unix'] session_end_unix = report['session_info']['time_end_unix'] - session_uuid = datetime.fromtimestamp(session_start_unix).strftime( - r'%Y%m%dT%H%M%S%z' - ) cursor = conn.execute( 'INSERT INTO sessions VALUES(' - ':session_id, :session_uuid, ' - ':session_start_unix, :session_end_unix, ' + ':session_id, :session_start_unix, :session_end_unix, ' ':json_blob, :report_file)', { 'session_id': None, - 'session_uuid': session_uuid, 'session_start_unix': session_start_unix, 'session_end_unix': session_end_unix, 'json_blob': jsonext.dumps(report), 'report_file': report_file_path } ) + session_id = cursor.lastrowid for run_idx, run in enumerate(report['runs']): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] @@ -123,13 +118,13 @@ def _db_store_report(self, conn, report, report_file_path): 'job_completion_time_unix': testcase[ 'job_completion_time_unix' ], - 'session_id': cursor.lastrowid, + 'session_id': session_id, 'run_index': run_idx, 'test_index': test_idx } ) - return session_uuid + return session_id def store(self, report, report_file): with sqlite3.connect(self._db_file()) as conn: @@ -157,7 +152,7 @@ def _fetch_testcases_raw(self, condition): def fetch_session_time_period(self, session_uuid): with sqlite3.connect(self._db_file()) as conn: query = ('SELECT session_start_unix, session_end_unix ' - f'FROM sessions WHERE session_uuid == "{session_uuid}" ' + f'FROM sessions WHERE id == "{session_uuid}" ' 'LIMIT 1') getlogger().debug(query) results = conn.execute(query).fetchall() From 3df63f23d3bf7aeda376ceb136b2e84ab615a699 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 10 Jul 2024 16:36:49 +0300 Subject: [PATCH 23/69] Remove unused import + fix `setup.cfg` --- reframe/frontend/reporting/storage.py | 1 - setup.cfg | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index c626ff8f1a..089b19c55f 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -6,7 +6,6 @@ import abc import os import sqlite3 -from datetime import datetime import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext diff --git a/setup.cfg b/setup.cfg index 974e38855d..3dc33868e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,9 @@ install_requires = archspec >= 0.2.4 argcomplete argcomplete <= 3.1.2; python_version < '3.8' + filelock + filelock<=3.12.2; python_version == '3.7' + filelock<=3.4.1; python_version == '3.6' jsonschema lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64' lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64' From aa75c377a1f43ac45162b4a08f4bfba41bcbcd63 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 10 Jul 2024 20:47:00 +0300 Subject: [PATCH 24/69] Group all actions in a required mutually exclusive option group --- reframe/frontend/argparse.py | 44 +++++++++++++++++++++++++++++++--- reframe/frontend/cli.py | 46 ++++++++++++------------------------ unittests/test_argparser.py | 7 ++++++ unittests/test_cli.py | 13 ++++++---- 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py index ec6c667142..93a4070f36 100644 --- a/reframe/frontend/argparse.py +++ b/reframe/frontend/argparse.py @@ -170,6 +170,12 @@ def __getattr__(self, name): return getattr(self._holder, name) + def __setattr__(self, name, value): + if name.startswith('_'): + super().__setattr__(name, value) + else: + setattr(self._holder, name, value) + def add_argument(self, *flags, **kwargs): try: opt_name = kwargs['dest'] @@ -254,6 +260,14 @@ def add_argument_group(self, *args, **kwargs): self._groups.append(group) return group + def add_mutually_exclusive_group(self, *args, **kwargs): + group = _ArgumentGroup( + self._holder.add_mutually_exclusive_group(*args, **kwargs), + self._option_map + ) + self._groups.append(group) + return group + def _resolve_attr(self, attr, namespaces): for ns in namespaces: if ns is None: @@ -269,7 +283,7 @@ def _update_defaults(self): for g in self._groups: self._defaults.__dict__.update(g._defaults.__dict__) - def parse_args(self, args=None, namespace=None): + def parse_args(self, args=None, namespace=None, suppress_required=False): '''Convert argument strings to objects and return them as attributes of a namespace. @@ -281,7 +295,30 @@ def parse_args(self, args=None, namespace=None): for it will be looked up first in `namespace` and if not found there, it will be assigned the default value as specified in its corresponding `add_argument()` call. If no default value was specified either, the - attribute will be set to `None`.''' + attribute will be set to `None`. + + If `suppress_required` is true, required mutually-exclusive groups will + be treated as optional for this parsing operation. + ''' + + class suppress_required_groups: + '''Temporarily suppress required groups if `suppress_required` + is true.''' + def __init__(this): + this._changed_grp = [] + + def __enter__(this): + if suppress_required: + for grp in self._groups: + if hasattr(grp, 'required') and grp.required: + this._changed_grp.append(grp) + grp.required = False + + return this + + def __exit__(this, *args, **kwargs): + for grp in this._changed_grp: + grp.required = True # Enable auto-completion argcomplete.autocomplete(self._holder) @@ -291,7 +328,8 @@ def parse_args(self, args=None, namespace=None): # newly parsed options to completely override any options defined in # namespace. The implementation of `argparse.ArgumentParser` does not # do this in options with an 'append' action. - options = self._holder.parse_args(args, None) + with suppress_required_groups(): + options = self._holder.parse_args(args, None) # Check if namespace refers to our namespace and take the cmd options # namespace suitable for ArgumentParser diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index faa0a40674..0c03ceccf7 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -211,6 +211,7 @@ def calc_verbosity(site_config, quiesce): def main(): # Setup command line options argparser = argparse.ArgumentParser() + action_options = argparser.add_mutually_exclusive_group(required=True) output_options = argparser.add_argument_group( 'Options controlling ReFrame output' ) @@ -220,9 +221,6 @@ def main(): select_options = argparser.add_argument_group( 'Options for selecting checks' ) - action_options = argparser.add_argument_group( - 'Options controlling actions' - ) run_options = argparser.add_argument_group( 'Options controlling the execution of checks' ) @@ -374,7 +372,6 @@ def main(): help=('Generate into FILE a Gitlab CI pipeline ' 'for the selected tests and exit'), ) - action_options.add_argument( '--describe', action='store_true', help='Give full details on the selected tests' @@ -400,6 +397,18 @@ def main(): '--dry-run', action='store_true', help='Dry run the tests without submitting them for execution' ) + action_options.add_argument( + '--performance-compare', metavar='CMPSPEC', action='store', + help='Compare past performance results' + ) + action_options.add_argument( + '--show-config', action='store', nargs='?', const='all', + metavar='PARAM', + help='Print the value of configuration parameter PARAM and exit' + ) + action_options.add_argument( + '-V', '--version', action='version', version=osext.reframe_version() + ) # Run options run_options.add_argument( @@ -538,10 +547,6 @@ def main(): ) # Reporting options - reporting_options.add_argument( - '--performance-compare', metavar='CMPSPEC', action='store', - help='Compare past performance results' - ) reporting_options.add_argument( '--performance-report', action='store', nargs='?', const=argparse.CONST_DEFAULT, @@ -572,22 +577,10 @@ def main(): help='Disable coloring of output', envvar='RFM_COLORIZE', configvar='general/colorize' ) - misc_options.add_argument( - '--show-config', action='store', nargs='?', const='all', - metavar='PARAM', - help='Print the value of configuration parameter PARAM and exit' - ) - misc_options.add_argument( - '--index-db', action='store_true', - help='Index old job reports in the database', - ) misc_options.add_argument( '--system', action='store', help='Load configuration for SYSTEM', envvar='RFM_SYSTEM' ) - misc_options.add_argument( - '-V', '--version', action='version', version=osext.reframe_version() - ) misc_options.add_argument( '-v', '--verbose', action='count', help='Increase verbosity level of output', @@ -838,7 +831,8 @@ def restrict_logging(): itertools.chain.from_iterable(shlex.split(m) for m in mode_args)) # Parse the mode's options and reparse the command-line - options = argparser.parse_args(mode_args) + options = argparser.parse_args(mode_args, + suppress_required=True) options = argparser.parse_args(namespace=options.cmd_options) options.update_config(site_config) @@ -1310,16 +1304,6 @@ def _sort_testcases(testcases): else: sys.exit(0) - if not options.run and not options.dry_run: - printer.error("No action option specified. Available options:\n" - " - `-l'/`-L' for listing\n" - " - `-r' for running\n" - " - `--dry-run' for dry running\n" - " - `--list-tags' for listing unique test tags\n" - " - `--ci-generate' for generating a CI pipeline\n" - f"Try `{argparser.prog} -h' for more options.") - sys.exit(1) - # Manipulate ReFrame's environment if site_config.get('general/0/purge_environment'): rt.modules_system.unload_all() diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py index cd6fffea21..6be36f3836 100644 --- a/unittests/test_argparser.py +++ b/unittests/test_argparser.py @@ -203,3 +203,10 @@ def test_envvar_option(default_exec_ctx, extended_parser): def test_envvar_option_default_val(default_exec_ctx, extended_parser): options = extended_parser.parse_args([]) assert options.env_option == 'bar' + + +def test_suppress_required(argparser): + group = argparser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo', action='store_true') + group.add_argument('--bar', action='store_true') + argparser.parse_args([], suppress_required=True) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 0f9f10880d..15cc95ca7b 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -528,6 +528,7 @@ def test_execution_modes(run_reframe, run_action): returncode, stdout, stderr = run_reframe( mode='unittest', action=run_action ) + assert returncode == 0 assert 'Traceback' not in stdout assert 'Traceback' not in stderr assert 'FAILED' not in stdout @@ -747,7 +748,8 @@ def test_show_config_all(run_reframe): # Just make sure that this option does not make the frontend crash returncode, stdout, stderr = run_reframe( more_options=['--show-config'], - system='testsys' + system='testsys', + action=None ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -758,7 +760,8 @@ def test_show_config_param(run_reframe): # Just make sure that this option does not make the frontend crash returncode, stdout, stderr = run_reframe( more_options=['--show-config=systems'], - system='testsys' + system='testsys', + action=None ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -769,7 +772,8 @@ def test_show_config_unknown_param(run_reframe): # Just make sure that this option does not make the frontend crash returncode, stdout, stderr = run_reframe( more_options=['--show-config=foo'], - system='testsys' + system='testsys', + action=None ) assert 'no such configuration parameter found' in stdout assert 'Traceback' not in stdout @@ -780,7 +784,8 @@ def test_show_config_unknown_param(run_reframe): def test_show_config_null_param(run_reframe): returncode, stdout, stderr = run_reframe( more_options=['--show-config=general/report_junit'], - system='testsys' + system='testsys', + action=None ) assert 'null' in stdout assert 'Traceback' not in stdout From 8df09cec51dac132ee590909ee51bc54f1a79cb2 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 10 Jul 2024 23:58:54 +0300 Subject: [PATCH 25/69] Add `--detect-host-topology` to actions --- reframe/frontend/cli.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 0c03ceccf7..342b167701 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -376,6 +376,16 @@ def main(): '--describe', action='store_true', help='Give full details on the selected tests' ) + action_options.add_argument( + '--detect-host-topology', metavar='FILE', action='store', + nargs='?', const='-', + help=('Detect the local host topology and exit, ' + 'optionally saving it in FILE') + ) + action_options.add_argument( + '--dry-run', action='store_true', + help='Dry run the tests without submitting them for execution' + ) action_options.add_argument( '-L', '--list-detailed', nargs='?', const='T', choices=['C', 'T'], help=('List the selected tests (T) or the concretized test cases (C) ' @@ -389,18 +399,14 @@ def main(): '--list-tags', action='store_true', help='List the unique tags found in the selected tests and exit' ) - action_options.add_argument( - '-r', '--run', action='store_true', - help='Run the selected checks' - ) - action_options.add_argument( - '--dry-run', action='store_true', - help='Dry run the tests without submitting them for execution' - ) action_options.add_argument( '--performance-compare', metavar='CMPSPEC', action='store', help='Compare past performance results' ) + action_options.add_argument( + '-r', '--run', action='store_true', + help='Run the selected checks' + ) action_options.add_argument( '--show-config', action='store', nargs='?', const='all', metavar='PARAM', @@ -563,12 +569,6 @@ def main(): help='Set configuration file', envvar='RFM_CONFIG_FILES :' ) - misc_options.add_argument( - '--detect-host-topology', metavar='FILE', action='store', - nargs='?', const='-', - help=('Detect the local host topology and exit, ' - 'optionally saving it in FILE') - ) misc_options.add_argument( '--failure-stats', action='store_true', help='Print failure statistics' ) From 81a9513b4bb83eb0cdbf4ebb107f855277009653 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 11 Jul 2024 00:36:10 +0300 Subject: [PATCH 26/69] Fix unit tests --- unittests/test_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 15cc95ca7b..6585dae0f6 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1116,7 +1116,8 @@ def test_detect_host_topology(run_reframe): from reframe.utility.cpuinfo import cpuinfo returncode, stdout, stderr = run_reframe( - more_options=['--detect-host-topology'] + more_options=['--detect-host-topology'], + action=None ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -1129,7 +1130,8 @@ def test_detect_host_topology_file(run_reframe, tmp_path): topo_file = tmp_path / 'topo.json' returncode, stdout, stderr = run_reframe( - more_options=[f'--detect-host-topology={topo_file}'] + more_options=[f'--detect-host-topology={topo_file}'], + action=None ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr From 5f6e12d491f61589aa9b6eb15ced3ed6b4fab42a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 11 Jul 2024 13:35:27 +0300 Subject: [PATCH 27/69] Fetch test cases from session exclusively --- reframe/frontend/reporting/__init__.py | 20 +++++++++++++++----- reframe/frontend/reporting/storage.py | 16 ++++++++++++++++ reframe/frontend/reporting/utility.py | 23 ++++++++++++++++------- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index b8e0d63538..70c44372fd 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -555,7 +555,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, def performance_compare(cmp, report=None): match = parse_cmp_spec(cmp) - if match.period_base is None: + if match.period_base is None and match.session_base is None: if report is None: raise ValueError('report cannot be `None` ' 'for current run comparisons') @@ -564,14 +564,24 @@ def performance_compare(cmp, report=None): except IndexError: tcs_base = [] - else: + elif match.period_base is not None: tcs_base = StorageBackend.default().fetch_testcases_time_period( *match.period_base ) + else: + tcs_base = StorageBackend.default().fetch_testcases_from_session( + match.session_base + ) + + if match.period_target: + tcs_target = StorageBackend.default().fetch_testcases_time_period( + *match.period_target + ) + else: + tcs_target = StorageBackend.default().fetch_testcases_from_session( + match.session_target + ) - tcs_target = StorageBackend.default().fetch_testcases_time_period( - *match.period_target - ) return compare_testcase_data(tcs_base, tcs_target, match.aggregator, match.aggregator, match.extra_groups, match.extra_cols) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 089b19c55f..ba97c37e76 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -166,3 +166,19 @@ def fetch_testcases_time_period(self, ts_start, ts_end): f'job_completion_time_unix <= {ts_end}) ' 'ORDER BY job_completion_time_unix' ) + + def fetch_testcases_from_session(self, session_uuid): + with sqlite3.connect(self._db_file()) as conn: + query = f'SELECT json_blob from sessions WHERE id=={session_uuid}' + getlogger().debug(query) + results = conn.execute(query).fetchall() + + if not results: + return [] + + testcases = [] + session_info = jsonext.loads(results[0][0]) + for run in session_info['runs']: + testcases += run['testcases'] + + return testcases diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 05cc0cf337..6f2ba477af 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -169,11 +169,22 @@ def _parse_aggregation(s): return Aggregator.create(op), _parse_extra_cols(extra_groups) -_Match = namedtuple('_Match', ['period_base', 'period_target', - 'aggregator', 'extra_groups', 'extra_cols']) +_Match = namedtuple('_Match', + ['period_base', 'period_target', + 'session_base', 'session_target', + 'aggregator', 'extra_groups', 'extra_cols']) def parse_cmp_spec(spec): + def _parse_period_spec(s): + if s is None: + return None, None + + if s.startswith('^'): + return s[1:], None + + return None, _parse_time_period(s) + parts = spec.split('/') if len(parts) == 3: period_base, period_target, aggr, cols = None, *parts @@ -182,11 +193,9 @@ def parse_cmp_spec(spec): else: raise ValueError(f'invalid cmp spec: {spec}') - if period_base is not None: - period_base = _parse_time_period(period_base) - - period_target = _parse_time_period(period_target) + session_base, period_base = _parse_period_spec(period_base) + session_target, period_target = _parse_period_spec(period_target) aggr_fn, extra_groups = _parse_aggregation(aggr) extra_cols = _parse_extra_cols(cols) - return _Match(period_base, period_target, + return _Match(period_base, period_target, session_base, session_target, aggr_fn, extra_groups, extra_cols) From 13504563bb61fe6d36fe0669ae8df00207fa6d91 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 11 Jul 2024 15:53:17 +0300 Subject: [PATCH 28/69] Allow grouping testcasesby runid and session_id` --- reframe/frontend/reporting/storage.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index ba97c37e76..abeb4e6b87 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -129,6 +129,13 @@ def store(self, report, report_file): with sqlite3.connect(self._db_file()) as conn: return self._db_store_report(conn, report, report_file) + def _enrich_testcase(self, tc, session_uuid, runid): + '''Enrich testcase record with more information''' + + tc['runid'] = runid + tc['session_uuid'] = session_uuid + return tc + def _fetch_testcases_raw(self, condition): with sqlite3.connect(self._db_file()) as conn: query = ('SELECT session_id, run_index, test_index, json_blob ' @@ -142,9 +149,10 @@ def _fetch_testcases_raw(self, condition): sessions = {} for session_id, run_index, test_index, json_blob in results: report = jsonext.loads(sessions.setdefault(session_id, json_blob)) - testcases.append( - report['runs'][run_index]['testcases'][test_index] - ) + testcases.append(self._enrich_testcase( + report['runs'][run_index]['testcases'][test_index], + session_id, run_index + )) return testcases @@ -176,9 +184,9 @@ def fetch_testcases_from_session(self, session_uuid): if not results: return [] - testcases = [] session_info = jsonext.loads(results[0][0]) - for run in session_info['runs']: - testcases += run['testcases'] - - return testcases + return [ + self._enrich_testcase(tc, session_uuid, runid) + for runid, run in enumerate(session_info['runs']) + for tc in run['testcases'] + ] From 92f91362a03b52bbae65a10b34eed1d6984de934 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 11 Jul 2024 18:50:23 +0300 Subject: [PATCH 29/69] Use proper UUIDs as session IDs --- reframe/frontend/reporting/__init__.py | 7 +++- reframe/frontend/reporting/storage.py | 54 +++++++++++--------------- reframe/schemas/runreport.json | 5 ++- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 70c44372fd..05a85c4f1b 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -14,6 +14,7 @@ import re import socket import time +import uuid from collections.abc import Hashable from filelock import FileLock @@ -213,7 +214,8 @@ def __init__(self): self.__report = { 'session_info': { 'data_version': DATA_VERSION, - 'hostname': socket.gethostname() + 'hostname': socket.gethostname(), + 'uuid': str(uuid.uuid4()) }, 'runs': [], 'restored_cases': [] @@ -255,6 +257,7 @@ def update_timestamps(self, ts_start, ts_end): }) def update_run_stats(self, stats): + session_uuid = self.__report['session_info']['uuid'] for runid, tasks in stats.runs(): testcases = [] num_failures = 0 @@ -288,7 +291,9 @@ def update_run_stats(self, stats): 'job_stdout': None, 'partition': partition.name, 'result': t.result, + 'runid': runid, 'scheduler': partition.scheduler.registered_name, + 'session_uuid': session_uuid, 'time_compile': t.duration('compile_complete'), 'time_performance': t.duration('performance'), 'time_run': t.duration('run_complete'), diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index abeb4e6b87..9b82f5c9d9 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -68,7 +68,7 @@ def _db_create(self): ) with sqlite3.connect(self.__db_file) as conn: conn.execute('CREATE TABLE IF NOT EXISTS sessions(' - 'id INTEGER PRIMARY KEY, ' + 'uuid TEXT PRIMARY KEY, ' 'session_start_unix REAL, ' 'session_end_unix REAL, ' 'json_blob TEXT, ' @@ -79,28 +79,28 @@ def _db_create(self): 'partition TEXT,' 'environ TEXT,' 'job_completion_time_unix REAL,' - 'session_id INTEGER,' + 'session_uuid TEXT,' 'run_index INTEGER,' 'test_index INTEGER,' - 'FOREIGN KEY(session_id) ' - 'REFERENCES sessions(session_id))') + 'FOREIGN KEY(session_uuid) ' + 'REFERENCES sessions(uuid))') def _db_store_report(self, conn, report, report_file_path): session_start_unix = report['session_info']['time_start_unix'] session_end_unix = report['session_info']['time_end_unix'] - cursor = conn.execute( + session_uuid = report['session_info']['uuid'] + conn.execute( 'INSERT INTO sessions VALUES(' - ':session_id, :session_start_unix, :session_end_unix, ' + ':uuid, :session_start_unix, :session_end_unix, ' ':json_blob, :report_file)', { - 'session_id': None, + 'uuid': session_uuid, 'session_start_unix': session_start_unix, 'session_end_unix': session_end_unix, 'json_blob': jsonext.dumps(report), 'report_file': report_file_path } ) - session_id = cursor.lastrowid for run_idx, run in enumerate(report['runs']): for test_idx, testcase in enumerate(run['testcases']): sys, part = testcase['system'], testcase['partition'] @@ -108,7 +108,7 @@ def _db_store_report(self, conn, report, report_file_path): 'INSERT INTO testcases VALUES(' ':name, :system, :partition, :environ, ' ':job_completion_time_unix, ' - ':session_id, :run_index, :test_index)', + ':session_uuid, :run_index, :test_index)', { 'name': testcase['name'], 'system': sys, @@ -117,29 +117,22 @@ def _db_store_report(self, conn, report, report_file_path): 'job_completion_time_unix': testcase[ 'job_completion_time_unix' ], - 'session_id': session_id, + 'session_uuid': session_uuid, 'run_index': run_idx, 'test_index': test_idx } ) - return session_id + return session_uuid def store(self, report, report_file): with sqlite3.connect(self._db_file()) as conn: return self._db_store_report(conn, report, report_file) - def _enrich_testcase(self, tc, session_uuid, runid): - '''Enrich testcase record with more information''' - - tc['runid'] = runid - tc['session_uuid'] = session_uuid - return tc - def _fetch_testcases_raw(self, condition): with sqlite3.connect(self._db_file()) as conn: - query = ('SELECT session_id, run_index, test_index, json_blob ' - 'FROM testcases JOIN sessions ON session_id==id ' + query = ('SELECT session_uuid, run_index, test_index, json_blob ' + 'FROM testcases JOIN sessions ON session_uuid == uuid ' f'WHERE {condition}') getlogger().debug(query) results = conn.execute(query).fetchall() @@ -147,19 +140,19 @@ def _fetch_testcases_raw(self, condition): # Retrieve files testcases = [] sessions = {} - for session_id, run_index, test_index, json_blob in results: - report = jsonext.loads(sessions.setdefault(session_id, json_blob)) - testcases.append(self._enrich_testcase( + for session_uuid, run_index, test_index, json_blob in results: + report = jsonext.loads(sessions.setdefault(session_uuid, + json_blob)) + testcases.append( report['runs'][run_index]['testcases'][test_index], - session_id, run_index - )) + ) return testcases def fetch_session_time_period(self, session_uuid): with sqlite3.connect(self._db_file()) as conn: query = ('SELECT session_start_unix, session_end_unix ' - f'FROM sessions WHERE id == "{session_uuid}" ' + f'FROM sessions WHERE uuid == "{session_uuid}" ' 'LIMIT 1') getlogger().debug(query) results = conn.execute(query).fetchall() @@ -177,7 +170,8 @@ def fetch_testcases_time_period(self, ts_start, ts_end): def fetch_testcases_from_session(self, session_uuid): with sqlite3.connect(self._db_file()) as conn: - query = f'SELECT json_blob from sessions WHERE id=={session_uuid}' + query = ('SELECT json_blob from sessions ' + f'WHERE uuid == "{session_uuid}"') getlogger().debug(query) results = conn.execute(query).fetchall() @@ -185,8 +179,4 @@ def fetch_testcases_from_session(self, session_uuid): return [] session_info = jsonext.loads(results[0][0]) - return [ - self._enrich_testcase(tc, session_uuid, runid) - for runid, run in enumerate(session_info['runs']) - for tc in run['testcases'] - ] + return [tc for run in session_info['runs'] for tc in run['testcases']] diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index 85a902e6ec..9a857a14d5 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -87,17 +87,20 @@ "num_skipped": {"type": "number"}, "prefix_output": {"type": "string"}, "prefix_stage": {"type": "string"}, + "runid": {"type": "number"}, + "session_uuid": {"type": "string"}, "time_elapsed": {"type": "number"}, "time_end": {"type": "string"}, "time_end_unix": {"type": "number"}, "time_start": {"type": "string"}, "time_start_unix": {"type": "number"}, "user": {"type": "string"}, + "uuid": {"type": "string"}, "version": {"type": "string"}, "workdir": {"type": "string"} }, "required": ["data_version", "hostname", - "time_elapsed", "time_end", "time_start"] + "time_elapsed", "time_end", "time_start", "uuid"] }, "restored_cases": { "type": "array", From d000d632ff82f63584656c4eb0a7294ff566ca71 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 12 Jul 2024 19:04:07 +0300 Subject: [PATCH 30/69] New command-line options for interacting with stored sessions --- reframe/frontend/cli.py | 98 +++++++++++++++++++++-- reframe/frontend/printer.py | 12 +-- reframe/frontend/reporting/__init__.py | 103 ++++++++++++++++++++++--- reframe/frontend/reporting/storage.py | 48 +++++++++--- reframe/frontend/reporting/utility.py | 4 +- reframe/schemas/runreport.json | 3 +- 6 files changed, 234 insertions(+), 34 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 342b167701..b6f2f6e73a 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -207,6 +207,29 @@ def calc_verbosity(site_config, quiesce): return curr_verbosity - quiesce +class exit_gracefuly_on_error: + def __init__(self, message, logger=None, exceptions=None, exitcode=1): + self.__message = message + self.__logger = logger or PrettyPrinter() + self.__exceptions = exceptions or (Exception,) + self.__exitcode = exitcode + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is SystemExit: + # Allow users to exit inside the context manager + return + + if isinstance(exc_val, self.__exceptions): + self.__logger.error(f'{self.__message}: {exc_val}') + self.__logger.verbose( + ''.join(traceback.format_exception(exc_type, exc_val, exc_tb)) + ) + sys.exit(self.__exitcode) + + @logging.time_function_noexit def main(): # Setup command line options @@ -366,16 +389,28 @@ def main(): help='Select checks that satisfy the expression EXPR' ) - # Action options action_options.add_argument( '--ci-generate', action='store', metavar='FILE', help=('Generate into FILE a Gitlab CI pipeline ' 'for the selected tests and exit'), ) + action_options.add_argument( + '--delete-stored-session', action='store', metavar='SESSION_UUID', + help='Delete stored session' + ) action_options.add_argument( '--describe', action='store_true', help='Give full details on the selected tests' ) + action_options.add_argument( + '--describe-stored-session', action='store', metavar='SESSION_UUID', + help='Get detailed session information in JSON' + ) + action_options.add_argument( + '--describe-stored-testcases', action='store', + metavar='SESSION_UUID|PERIOD', + help='Get detailed test case information in JSON' + ) action_options.add_argument( '--detect-host-topology', metavar='FILE', action='store', nargs='?', const='-', @@ -391,6 +426,15 @@ def main(): help=('List the selected tests (T) or the concretized test cases (C) ' 'providing more details') ) + action_options.add_argument( + '--list-stored-sessions', action='store_true', + help='List stored session' + ) + action_options.add_argument( + '--list-stored-testcases', action='store', + metavar='SESSION_UUID|PERIOD', + help='List stored testcases by session or time period' + ) action_options.add_argument( '-l', '--list', nargs='?', const='T', choices=['C', 'T'], help='List the selected tests (T) or the concretized test cases (C)' @@ -734,7 +778,10 @@ def restrict_logging(): ''' if (options.show_config or - options.detect_host_topology or options.describe): + options.detect_host_topology or + options.describe or + options.describe_stored_session or + options.describe_stored_testcases): logging.getlogger().setLevel(logging.ERROR) return True else: @@ -882,6 +929,47 @@ def restrict_logging(): printer.info(logfiles_message()) sys.exit(1) + if options.list_stored_sessions: + with exit_gracefuly_on_error('failed to retrieve session data', + printer): + printer.table(reporting.session_data()) + sys.exit(0) + + if options.list_stored_testcases: + with exit_gracefuly_on_error('failed to retrieve test case data', + printer): + printer.table(reporting.testcase_data( + options.list_stored_testcases + )) + sys.exit(0) + + if options.describe_stored_session: + # Restore logging level + printer.setLevel(logging.INFO) + with exit_gracefuly_on_error('failed to retrieve session data', + printer): + printer.info(jsonext.dumps(reporting.session_info( + options.describe_stored_session + ), indent=2)) + sys.exit(0) + + if options.describe_stored_testcases: + # Restore logging level + printer.setLevel(logging.INFO) + with exit_gracefuly_on_error('failed to retrieve test case data', + printer): + printer.info(jsonext.dumps(reporting.testcase_info( + options.describe_stored_testcases + ), indent=2)) + sys.exit(0) + + if options.delete_stored_session: + session_uuid = options.delete_stored_session + with exit_gracefuly_on_error('failed to delete session', printer): + reporting.delete_session(session_uuid) + printer.info(f'Session {session_uuid} deleted successfully.') + sys.exit(0) + # Show configuration after everything is set up if options.show_config: # Restore logging level @@ -1296,7 +1384,7 @@ def _sort_testcases(testcases): if options.performance_compare: try: printer.table( - *reporting.performance_compare(options.performance_compare) + reporting.performance_compare(options.performance_compare) ) except (errors.ReframeError, ValueError) as err: printer.error(f'failed to generate performance report: {err}') @@ -1458,7 +1546,7 @@ def module_unuse(*paths): if options.performance_report and not options.dry_run: try: - data, header = reporting.performance_compare( + data = reporting.performance_compare( rt.get_option('general/0/perf_report_spec'), report ) except (errors.ReframeError, ValueError) as err: @@ -1466,7 +1554,7 @@ def module_unuse(*paths): f'failed to generate performance report: {err}' ) else: - printer.performance_report(data, header) + printer.performance_report(data) # Generate the report for this session report_file = os.path.normpath( diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index 2409637a20..cf0badb6c6 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -247,15 +247,17 @@ def retry_report(self, report): self.info('\n'.join(lines)) - def performance_report(self, data, header): + def performance_report(self, data, **kwargs): width = min(80, shutil.get_terminal_size()[0]) self.info('') self.info(' PERFORMANCE REPORT '.center(width, '=')) self.info('') - self.table(data, header) + self.table(data, **kwargs) self.info('') - def table(self, data, header): - '''Print a table''' + def table(self, data, **kwargs): + '''Print tabular data''' - self.info(tabulate(data, headers=header, tablefmt='mixed_grid')) + kwargs.setdefault('headers', 'firstrow') + kwargs.setdefault('tablefmt', 'mixed_grid') + self.info(tabulate(data, **kwargs)) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 05a85c4f1b..01121b555a 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -23,10 +23,11 @@ import reframe.utility.osext as osext from reframe.core.exceptions import ReframeError, what, is_severe from reframe.core.logging import getlogger, _format_time_rfc3339 +from reframe.core.runtime import runtime from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev from .storage import StorageBackend -from .utility import Aggregator, parse_cmp_spec +from .utility import Aggregator, parse_cmp_spec, parse_time_period # The schema data version # Major version bumps are expected to break the validation of previous schemas @@ -247,7 +248,7 @@ def update_restored_cases(self, restored_cases, restored_session): for c in restored_cases] def update_timestamps(self, ts_start, ts_end): - fmt = r'%FT%T%z' + fmt = r'%Y%m%dT%H%M%S%z' self.__report['session_info'].update({ 'time_start': time.strftime(fmt, time.localtime(ts_start)), 'time_start_unix': ts_start, @@ -263,7 +264,7 @@ def update_run_stats(self, stats): num_failures = 0 num_aborted = 0 num_skipped = 0 - for t in tasks: + for tidx, t in enumerate(tasks): # We take partition and environment from the test case and not # from the check, since if the test fails before `setup()`, # these are not set inside the check. @@ -299,7 +300,8 @@ def update_run_stats(self, stats): 'time_run': t.duration('run_complete'), 'time_sanity': t.duration('sanity'), 'time_setup': t.duration('setup'), - 'time_total': t.duration('total') + 'time_total': t.duration('total'), + 'uuid': f'{session_uuid}:{runid}:{tidx}' } if check.job: entry['job_stderr'] = check.stderr.evaluate() @@ -412,6 +414,10 @@ def generate_xml_report(self): report = self.__report xml_testsuites = etree.Element('testsuites') + # Create a XSD-friendly timestamp + session_ts = time.strftime( + r'%FT%T', time.localtime(report['session_info']['time_start_unix']) + ) for run_id, rfm_run in enumerate(report['runs']): xml_testsuite = etree.SubElement( xml_testsuites, 'testsuite', @@ -424,9 +430,7 @@ def generate_xml_report(self): 'package': 'reframe', 'tests': str(rfm_run['num_cases']), 'time': str(report['session_info']['time_elapsed']), - # XSD schema does not like the timezone format, - # so we remove it - 'timestamp': report['session_info']['time_start'][:-5], + 'timestamp': session_ts } ) etree.SubElement(xml_testsuite, 'properties') @@ -534,7 +538,8 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, ptarget = _aggregate_perf(grouped_target, target_fn, []) # Build the final table data - data = [] + data = [['name', 'pvar', 'pval', + 'punit', 'pdiff'] + extra_group_by + extra_cols] for key, aggr_data in pbase.items(): pval = aggr_data['pval'] try: @@ -554,8 +559,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, line += [aggr_data[c] for c in extra_cols] data.append(line) - return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] + - extra_group_by + extra_cols)) + return data def performance_compare(cmp, report=None): @@ -590,3 +594,82 @@ def performance_compare(cmp, report=None): return compare_testcase_data(tcs_base, tcs_target, match.aggregator, match.aggregator, match.extra_groups, match.extra_cols) + + +def session_data(): + '''Retrieve all sessions''' + + data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']] + for sess_data in StorageBackend.default().fetch_all_sessions(): + session_info = sess_data['session_info'] + data.append( + [session_info['uuid'], + session_info['time_start'], + session_info['time_end'], + len(sess_data['runs']), + len(sess_data['runs'][0]['testcases'])] + ) + + return data + + +def testcase_data(spec): + storage = StorageBackend.default() + if spec.startswith('^'): + testcases = storage.fetch_testcases_from_session(spec[1:]) + else: + testcases = storage.fetch_testcases_time_period( + *parse_time_period(spec) + ) + + data = [['Name', 'System', 'Partition', 'Environment', + 'Nodelist', 'Result', 'UUID']] + for tc in testcases: + data.append([ + tc['name'], + tc['system'], tc['partition'], tc['environ'], + nodelist_abbrev(tc['job_nodelist']), tc['result'], + tc['uuid'] + ]) + + return data + + +def session_info(uuid): + '''Retrieve session details as JSON''' + + session = StorageBackend.default().fetch_session_json(uuid) + if not session: + raise ReframeError(f'no such session: {uuid}') + + return session + + +def testcase_info(spec): + '''Retrieve test case details as JSON''' + testcases = [] + if spec.startswith('^'): + session_uuid, *tc_index = spec[1:].split(':') + session = session_info(session_uuid) + if not tc_index: + for run in session['runs']: + testcases += run['testcases'] + else: + run_index, test_index = tc_index + testcases.append( + session['runs'][run_index]['testcases'][test_index] + ) + else: + testcases = StorageBackend.default().fetch_testcases_time_period( + *parse_time_period(spec) + ) + + return testcases + + +def delete_session(session_uuid): + prefix = os.path.dirname( + osext.expandvars(runtime().get_option('general/0/report_file')) + ) + with FileLock(os.path.join(prefix, '.db.lock')): + StorageBackend.default().remove_session(session_uuid) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 9b82f5c9d9..2a32dcc0b1 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -80,10 +80,9 @@ def _db_create(self): 'environ TEXT,' 'job_completion_time_unix REAL,' 'session_uuid TEXT,' - 'run_index INTEGER,' - 'test_index INTEGER,' + 'uuid TEXT,' 'FOREIGN KEY(session_uuid) ' - 'REFERENCES sessions(uuid))') + 'REFERENCES sessions(uuid) ON DELETE CASCADE)') def _db_store_report(self, conn, report, report_file_path): session_start_unix = report['session_info']['time_start_unix'] @@ -101,14 +100,14 @@ def _db_store_report(self, conn, report, report_file_path): 'report_file': report_file_path } ) - for run_idx, run in enumerate(report['runs']): - for test_idx, testcase in enumerate(run['testcases']): + for run in report['runs']: + for testcase in run['testcases']: sys, part = testcase['system'], testcase['partition'] conn.execute( 'INSERT INTO testcases VALUES(' ':name, :system, :partition, :environ, ' ':job_completion_time_unix, ' - ':session_uuid, :run_index, :test_index)', + ':session_uuid, :uuid)', { 'name': testcase['name'], 'system': sys, @@ -118,8 +117,7 @@ def _db_store_report(self, conn, report, report_file_path): 'job_completion_time_unix' ], 'session_uuid': session_uuid, - 'run_index': run_idx, - 'test_index': test_idx + 'uuid': testcase['uuid'] } ) @@ -131,8 +129,9 @@ def store(self, report, report_file): def _fetch_testcases_raw(self, condition): with sqlite3.connect(self._db_file()) as conn: - query = ('SELECT session_uuid, run_index, test_index, json_blob ' - 'FROM testcases JOIN sessions ON session_uuid == uuid ' + query = ('SELECT session_uuid, testcases.uuid as uuid, json_blob ' + 'FROM testcases ' + 'JOIN sessions ON session_uuid == sessions.uuid ' f'WHERE {condition}') getlogger().debug(query) results = conn.execute(query).fetchall() @@ -140,7 +139,8 @@ def _fetch_testcases_raw(self, condition): # Retrieve files testcases = [] sessions = {} - for session_uuid, run_index, test_index, json_blob in results: + for session_uuid, uuid, json_blob in results: + run_index, test_index = [int(x) for x in uuid.split(':')[1:]] report = jsonext.loads(sessions.setdefault(session_uuid, json_blob)) testcases.append( @@ -180,3 +180,29 @@ def fetch_testcases_from_session(self, session_uuid): session_info = jsonext.loads(results[0][0]) return [tc for run in session_info['runs'] for tc in run['testcases']] + + def fetch_all_sessions(self): + with sqlite3.connect(self._db_file()) as conn: + query = ('SELECT json_blob from sessions ' + 'ORDER BY session_start_unix') + getlogger().debug(query) + results = conn.execute(query).fetchall() + + if not results: + return [] + + return [jsonext.loads(json_blob) for json_blob, *_ in results] + + def fetch_session_json(self, uuid): + with sqlite3.connect(self._db_file()) as conn: + query = f'SELECT json_blob FROM sessions WHERE uuid == "{uuid}"' + getlogger().debug(query) + results = conn.execute(query).fetchall() + + return jsonext.loads(results[0][0]) if results else {} + + def remove_session(self, uuid): + with sqlite3.connect(self._db_file()) as conn: + query = f'DELETE FROM sessions WHERE uuid == "{uuid}"' + getlogger().debug(query) + conn.execute(query) diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 6f2ba477af..8d0b8e4092 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -128,7 +128,7 @@ def _do_parse(s): return ts.timestamp() -def _parse_time_period(s): +def parse_time_period(s): if s.startswith('^'): # Retrieve the period of a full session try: @@ -183,7 +183,7 @@ def _parse_period_spec(s): if s.startswith('^'): return s[1:], None - return None, _parse_time_period(s) + return None, parse_time_period(s) parts = spec.split('/') if len(parts) == 3: diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index 9a857a14d5..006f39bc28 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -57,7 +57,8 @@ "time_sanity": {"type": ["number", "null"]}, "time_setup": {"type": ["number", "null"]}, "time_total": {"type": ["number", "null"]}, - "unique_name": {"type": "string"} + "unique_name": {"type": "string"}, + "uuid": {"type": "string"} }, "required": ["environ", "fail_phase", "fail_reason", "filename", "job_completion_time_unix", "name", "perfvalues", From 50294e7d89b149fe1ac77ece71a463abac4cfdca Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 15 Jul 2024 17:17:29 +0300 Subject: [PATCH 31/69] Make report generation optional + add new configuration section for the results storage --- reframe/frontend/cli.py | 45 +++++++++++++++----------- reframe/frontend/reporting/__init__.py | 11 ++----- reframe/frontend/reporting/storage.py | 32 ++++++++++-------- reframe/schemas/config.json | 16 +++++++-- 4 files changed, 61 insertions(+), 43 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index b6f2f6e73a..869be7d66e 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -677,6 +677,13 @@ def main(): action='append', help='Directories where ReFrame will look for base configuration' ) + argparser.add_argument( + dest='generate_file_reports', + envvar='RFM_GENERATE_FILE_REPORTS', + configvar='general/generate_file_reports', + action='store_true', + help='Save session report in files' + ) argparser.add_argument( dest='git_timeout', envvar='RFM_GIT_TIMEOUT', @@ -1564,24 +1571,26 @@ def module_unuse(*paths): if basedir: os.makedirs(basedir, exist_ok=True) - # Save the report file - try: - default_loc = os.path.dirname( - osext.expandvars(rt.get_default('general/report_file')) - ) - report.save( - report_file, - compress=rt.get_option('general/0/compress_report'), - link_to_last=(default_loc == os.path.dirname(report_file)) - ) - except OSError as e: - printer.warning( - f'failed to generate report in {report_file!r}: {e}' - ) - except errors.ReframeError as e: - printer.warning( - f'failed to create symlink to latest report: {e}' - ) + if rt.get_option('general/0/generate_file_reports'): + # Save the report file + try: + default_loc = os.path.dirname( + osext.expandvars(rt.get_default('general/report_file')) + ) + report.save( + report_file, + compress=rt.get_option('general/0/compress_report'), + link_to_last=(default_loc == + os.path.dirname(report_file)) + ) + except OSError as e: + printer.warning( + f'failed to generate report in {report_file!r}: {e}' + ) + except errors.ReframeError as e: + printer.warning( + f'failed to create symlink to latest report: {e}' + ) # Store the generated report for analytics try: diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 01121b555a..f5938b5ef1 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -23,7 +23,6 @@ import reframe.utility.osext as osext from reframe.core.exceptions import ReframeError, what, is_severe from reframe.core.logging import getlogger, _format_time_rfc3339 -from reframe.core.runtime import runtime from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev from .storage import StorageBackend @@ -405,9 +404,7 @@ def save(self, filename, compress=False, link_to_last=True): def store(self): '''Store the report in the results storage.''' - prefix = os.path.dirname(self.filename) or '.' - with FileLock(os.path.join(prefix, '.db.lock')): - return StorageBackend.default().store(self, self.filename) + return StorageBackend.default().store(self, self.filename) def generate_xml_report(self): '''Generate a JUnit report from a standard ReFrame JSON report.''' @@ -668,8 +665,4 @@ def testcase_info(spec): def delete_session(session_uuid): - prefix = os.path.dirname( - osext.expandvars(runtime().get_option('general/0/report_file')) - ) - with FileLock(os.path.join(prefix, '.db.lock')): - StorageBackend.default().remove_session(session_uuid) + StorageBackend.default().remove_session(session_uuid) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 2a32dcc0b1..67869ad3dd 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -6,9 +6,11 @@ import abc import os import sqlite3 +from filelock import FileLock import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext +from reframe.core.exceptions import ReframeError from reframe.core.logging import getlogger from reframe.core.runtime import runtime @@ -22,12 +24,12 @@ def create(cls, backend, *args, **kwargs): if backend == 'sqlite': return _SqliteStorage(*args, **kwargs) else: - raise NotImplementedError + raise ReframeError(f'no such storage backend: {backend}') @classmethod def default(cls): '''Return default storage backend''' - return cls.create('sqlite') + return cls.create(runtime().get_option('storage/0/backend')) @abc.abstractmethod def store(self, report, report_file): @@ -44,11 +46,9 @@ def fetch_testcases_time_period(self, ts_start, ts_end): class _SqliteStorage(StorageBackend): def __init__(self): - site_config = runtime().site_config - prefix = os.path.dirname(osext.expandvars( - site_config.get('general/0/report_file') - )) - self.__db_file = os.path.join(prefix, 'results.db') + self.__db_file = os.path.join( + osext.expandvars(runtime().get_option('storage/0/sqlite_db_file')) + ) def _db_file(self): prefix = os.path.dirname(self.__db_file) @@ -123,9 +123,11 @@ def _db_store_report(self, conn, report, report_file_path): return session_uuid - def store(self, report, report_file): - with sqlite3.connect(self._db_file()) as conn: - return self._db_store_report(conn, report, report_file) + def store(self, report, report_file=None): + prefix = os.path.dirname(self.__db_file) + with FileLock(os.path.join(prefix, '.db.lock')): + with sqlite3.connect(self._db_file()) as conn: + return self._db_store_report(conn, report, report_file) def _fetch_testcases_raw(self, condition): with sqlite3.connect(self._db_file()) as conn: @@ -202,7 +204,9 @@ def fetch_session_json(self, uuid): return jsonext.loads(results[0][0]) if results else {} def remove_session(self, uuid): - with sqlite3.connect(self._db_file()) as conn: - query = f'DELETE FROM sessions WHERE uuid == "{uuid}"' - getlogger().debug(query) - conn.execute(query) + prefix = os.path.dirname(self.__db_file) + with FileLock(os.path.join(prefix, '.db.lock')): + with sqlite3.connect(self._db_file()) as conn: + query = f'DELETE FROM sessions WHERE uuid == "{uuid}"' + getlogger().debug(query) + conn.execute(query) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 51189b2811..8bb6a1c882 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -504,6 +504,7 @@ "clean_stagedir": {"type": "boolean"}, "colorize": {"type": "boolean"}, "compress_report": {"type": "boolean"}, + "generate_file_reports": {"type": "boolean"}, "git_timeout": {"type": "number"}, "keep_stage_files": {"type": "boolean"}, "module_map_file": {"type": "string"}, @@ -519,7 +520,6 @@ "purge_environment": {"type": "boolean"}, "remote_detect": {"type": "boolean"}, "remote_workdir": {"type": "string"}, - "report_database_file": {"type": "string"}, "report_file": {"type": "string"}, "report_junit": {"type": ["string", "null"]}, "resolve_module_conflicts": {"type": "boolean"}, @@ -534,6 +534,16 @@ }, "additionalProperties": false } + }, + "storage": { + "type": "array", + "items": { + "type": "object", + "properties": { + "backend": {"type": "string"}, + "sqlite_db_file": {"type": "string"} + } + } } }, "required": ["systems", "environments", "logging"], @@ -562,6 +572,7 @@ "general/clean_stagedir": true, "general/colorize": true, "general/compress_report": false, + "general/generate_file_reports": true, "general/git_timeout": 5, "general/keep_stage_files": false, "general/module_map_file": "", @@ -572,7 +583,6 @@ "general/purge_environment": false, "general/remote_detect": false, "general/remote_workdir": ".", - "general/report_database_file": "${HOME}/.reframe/reports_index.db", "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/report_junit": null, "general/resolve_module_conflicts": true, @@ -610,6 +620,8 @@ "logging/handlers_perflog/httpjson_debug": false, "modes/options": [], "modes/target_systems": ["*"], + "storage/backend": "sqlite", + "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db", "systems/descr": "", "systems/max_local_jobs": 8, "systems/modules_system": "nomod", From edf5d546eb6156572f5e137967d1645bc03dd62b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 16 Jul 2024 15:18:33 +0300 Subject: [PATCH 32/69] Add Sqlite DB schema version check --- reframe/frontend/reporting/storage.py | 36 ++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 67869ad3dd..0fc89d5743 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -45,6 +45,8 @@ def fetch_testcases_time_period(self, ts_start, ts_end): class _SqliteStorage(StorageBackend): + SCHEMA_VERSION = '1.0' + def __init__(self): self.__db_file = os.path.join( osext.expandvars(runtime().get_option('storage/0/sqlite_db_file')) @@ -59,6 +61,7 @@ def _db_file(self): self._db_create() + self._db_schema_check() return self.__db_file def _db_create(self): @@ -75,14 +78,35 @@ def _db_create(self): 'report_file TEXT)') conn.execute('CREATE TABLE IF NOT EXISTS testcases(' 'name TEXT,' - 'system TEXT,' - 'partition TEXT,' - 'environ TEXT,' - 'job_completion_time_unix REAL,' - 'session_uuid TEXT,' - 'uuid TEXT,' + 'system TEXT, ' + 'partition TEXT, ' + 'environ TEXT, ' + 'job_completion_time_unix REAL, ' + 'session_uuid TEXT, ' + 'uuid TEXT, ' 'FOREIGN KEY(session_uuid) ' 'REFERENCES sessions(uuid) ON DELETE CASCADE)') + conn.execute('CREATE TABLE IF NOT EXISTS metadata(' + 'schema_version TEXT)') + + def _db_schema_check(self): + with sqlite3.connect(self.__db_file) as conn: + results = conn.execute( + 'SELECT schema_version FROM metadata').fetchall() + + if not results: + # DB is new, insert the schema version + with sqlite3.connect(self.__db_file) as conn: + conn.execute('INSERT INTO metadata VALUES(:schema_version)', + {'schema_version': self.SCHEMA_VERSION}) + else: + found_ver = results[0][0] + if found_ver != self.SCHEMA_VERSION: + raise ReframeError( + f'results DB in {self.__db_file!r} is ' + 'of incompatible version: ' + f'found {found_ver}, required: {self.SCHEMA_VERSION}' + ) def _db_store_report(self, conn, report, report_file_path): session_start_unix = report['session_info']['time_start_unix'] From addcde08f218820f79c3595f8c96901cd102cdb8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 16 Jul 2024 15:27:58 +0300 Subject: [PATCH 33/69] Support `w` for denoting weeks in period specs --- reframe/frontend/reporting/utility.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 8d0b8e4092..6d4433815b 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -109,14 +109,18 @@ def _do_parse(s): ts = _do_parse(s) except ValueError as err: # Try the relative timestamps - match = re.match(r'(?P.*)(?P[\+|-]\d+)(?P[hdms])', s) + match = re.match( + r'(?P.*)(?P[\+|-]\d+)(?P[hdmsw])', s + ) if not match: raise err ts = _do_parse(match.group('ts')) amount = int(match.group('amount')) unit = match.group('unit') - if unit == 'd': + if unit == 'w': + ts += timedelta(weeks=amount) + elif unit == 'd': ts += timedelta(days=amount) elif unit == 'm': ts += timedelta(minutes=amount) From efb33194c1faf0e1fe66381f919332a6422e2027 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 16 Jul 2024 19:35:45 +0300 Subject: [PATCH 34/69] Improve error handling --- reframe/core/exceptions.py | 14 ++++++++++++++ reframe/frontend/cli.py | 24 ++++++++++++------------ reframe/frontend/reporting/__init__.py | 22 ++++++++++++++-------- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py index af9acc5751..d63403b587 100644 --- a/reframe/core/exceptions.py +++ b/reframe/core/exceptions.py @@ -388,3 +388,17 @@ def what(exc_type, exc_value, tb): reason += f': {exc_value}' return reason + + +class reraise_as: + def __init__(self, new_exc, exceptions=(Exception,), message=''): + self.__new_exc = new_exc + self.__exceptions = exceptions + self.__message = message + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if isinstance(exc_val, self.__exceptions): + raise self.__new_exc(self.__message) from exc_val diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 869be7d66e..20707c8c1c 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -207,7 +207,7 @@ def calc_verbosity(site_config, quiesce): return curr_verbosity - quiesce -class exit_gracefuly_on_error: +class exit_gracefully_on_error: def __init__(self, message, logger=None, exceptions=None, exitcode=1): self.__message = message self.__logger = logger or PrettyPrinter() @@ -937,14 +937,14 @@ def restrict_logging(): sys.exit(1) if options.list_stored_sessions: - with exit_gracefuly_on_error('failed to retrieve session data', - printer): + with exit_gracefully_on_error('failed to retrieve session data', + printer): printer.table(reporting.session_data()) sys.exit(0) if options.list_stored_testcases: - with exit_gracefuly_on_error('failed to retrieve test case data', - printer): + with exit_gracefully_on_error('failed to retrieve test case data', + printer): printer.table(reporting.testcase_data( options.list_stored_testcases )) @@ -953,8 +953,8 @@ def restrict_logging(): if options.describe_stored_session: # Restore logging level printer.setLevel(logging.INFO) - with exit_gracefuly_on_error('failed to retrieve session data', - printer): + with exit_gracefully_on_error('failed to retrieve session data', + printer): printer.info(jsonext.dumps(reporting.session_info( options.describe_stored_session ), indent=2)) @@ -963,8 +963,8 @@ def restrict_logging(): if options.describe_stored_testcases: # Restore logging level printer.setLevel(logging.INFO) - with exit_gracefuly_on_error('failed to retrieve test case data', - printer): + with exit_gracefully_on_error('failed to retrieve test case data', + printer): printer.info(jsonext.dumps(reporting.testcase_info( options.describe_stored_testcases ), indent=2)) @@ -972,7 +972,7 @@ def restrict_logging(): if options.delete_stored_session: session_uuid = options.delete_stored_session - with exit_gracefuly_on_error('failed to delete session', printer): + with exit_gracefully_on_error('failed to delete session', printer): reporting.delete_session(session_uuid) printer.info(f'Session {session_uuid} deleted successfully.') sys.exit(0) @@ -1393,7 +1393,7 @@ def _sort_testcases(testcases): printer.table( reporting.performance_compare(options.performance_compare) ) - except (errors.ReframeError, ValueError) as err: + except errors.ReframeError as err: printer.error(f'failed to generate performance report: {err}') sys.exit(1) else: @@ -1556,7 +1556,7 @@ def module_unuse(*paths): data = reporting.performance_compare( rt.get_option('general/0/perf_report_spec'), report ) - except (errors.ReframeError, ValueError) as err: + except errors.ReframeError as err: printer.warning( f'failed to generate performance report: {err}' ) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index f5938b5ef1..c542652dd7 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -21,7 +21,7 @@ import reframe as rfm import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext -from reframe.core.exceptions import ReframeError, what, is_severe +from reframe.core.exceptions import ReframeError, what, is_severe, reraise_as from reframe.core.logging import getlogger, _format_time_rfc3339 from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev @@ -473,7 +473,9 @@ def save_junit(self, filename): def _group_key(groups, testcase): key = [] for grp in groups: - val = testcase[grp] + with reraise_as(ReframeError, (KeyError,), 'no such group'): + val = testcase[grp] + if grp == 'job_nodelist': # Fold nodelist before adding as a key element key.append(nodelist_abbrev(val)) @@ -514,11 +516,12 @@ def _aggregate_perf(grouped_testcases, aggr_fn, cols): for key, seq in grouped_testcases.items(): aggr_data.setdefault(key, {}) aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq) - for c in cols: - aggr_data[key][c] = other_aggr( - nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c] - for tc in seq - ) + with reraise_as(ReframeError, (KeyError,), 'no such column'): + for c in cols: + aggr_data[key][c] = other_aggr( + nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c] + for tc in seq + ) return aggr_data @@ -560,7 +563,10 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, def performance_compare(cmp, report=None): - match = parse_cmp_spec(cmp) + with reraise_as(ReframeError, (ValueError,), + 'could not parse comparison spec'): + match = parse_cmp_spec(cmp) + if match.period_base is None and match.session_base is None: if report is None: raise ValueError('report cannot be `None` ' From 5b7412a916dfd8a55deda128277363bcb869c1b5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 16 Jul 2024 20:47:08 +0300 Subject: [PATCH 35/69] Do not generate reports and DB entries if report is empty --- reframe/frontend/cli.py | 28 +++++++++++++++----------- reframe/frontend/reporting/__init__.py | 17 ++++++++++++---- reframe/schemas/config.json | 2 +- unittests/test_cli.py | 2 +- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 20707c8c1c..14c3d435c6 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -317,7 +317,7 @@ def main(): '--timestamp', action='store', nargs='?', metavar='TIMEFMT', const=argparse.CONST_DEFAULT, help=('Append a timestamp to the output and stage directory prefixes ' - '(default: "%%y%%m%%dT%%H%%M%%S%%z")'), + '(default: "%%Y%%m%%dT%%H%%M%%S%%z")'), envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' ) @@ -1551,7 +1551,8 @@ def module_unuse(*paths): report, global_stats=options.duration or options.reruns ) - if options.performance_report and not options.dry_run: + if (options.performance_report and + not options.dry_run and not report.is_empty()): try: data = reporting.performance_compare( rt.get_option('general/0/perf_report_spec'), report @@ -1571,7 +1572,8 @@ def module_unuse(*paths): if basedir: os.makedirs(basedir, exist_ok=True) - if rt.get_option('general/0/generate_file_reports'): + if (rt.get_option('general/0/generate_file_reports') and + not report.is_empty()): # Save the report file try: default_loc = os.path.dirname( @@ -1593,18 +1595,20 @@ def module_unuse(*paths): ) # Store the generated report for analytics - try: - sess_uuid = report.store() - except Exception as e: - printer.warning( - f'failed to store results in the database: {e}' - ) - else: - printer.info(f'Current session stored with UUID: {sess_uuid}') + if not report.is_empty(): + try: + sess_uuid = report.store() + except Exception as e: + printer.warning( + f'failed to store results in the database: {e}' + ) + else: + printer.info('Current session stored with UUID: ' + f'{sess_uuid}') # Generate the junit xml report for this session junit_report_file = rt.get_option('general/0/report_junit') - if junit_report_file: + if junit_report_file and not report.is_empty(): # Expand variables in filename junit_report_file = osext.expandvars(junit_report_file) try: diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index c542652dd7..4194998ef3 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -396,6 +396,10 @@ def _save(self, filename, compress, link_to_last): else: raise ReframeError('path exists and is not a symlink') + def is_empty(self): + '''Return :obj:`True` is no test cases where run''' + return self.__report['session_info']['num_cases'] == 0 + def save(self, filename, compress=False, link_to_last=True): prefix = os.path.dirname(filename) or '.' with FileLock(os.path.join(prefix, '.report.lock')): @@ -625,13 +629,18 @@ def testcase_data(spec): *parse_time_period(spec) ) - data = [['Name', 'System', 'Partition', 'Environment', - 'Nodelist', 'Result', 'UUID']] + data = [['Name', 'SysEnv', + 'Nodelist', 'Completion Time', 'Result', 'UUID']] for tc in testcases: data.append([ tc['name'], - tc['system'], tc['partition'], tc['environ'], - nodelist_abbrev(tc['job_nodelist']), tc['result'], + f'{tc["system"]}:{tc["partition"]}@{tc["environ"]}', + nodelist_abbrev(tc['job_nodelist']), + # Always format the completion time as users can set their own + # formatting in the log record + time.strftime(r'%Y%m%dT%H%M%S%z', + time.localtime(tc['job_completion_time_unix'])), + tc['result'], tc['uuid'] ]) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 8bb6a1c882..900ab0de14 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -588,7 +588,7 @@ "general/resolve_module_conflicts": true, "general/save_log_files": false, "general/target_systems": ["*"], - "general/timestamp_dirs": "%y%m%dT%H%M%S%z", + "general/timestamp_dirs": "%Y%m%dT%H%M%S%z", "general/trap_job_errors": false, "general/unload_modules": [], "general/use_login_shell": false, diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 6585dae0f6..04051c7b36 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -571,7 +571,7 @@ def test_timestamp_option_default(run_reframe): assert returncode == 0 matches = re.findall( - r'(stage|output) directory: .*\/(\d{6}T\d{6}\+\d{4})', stdout + r'(stage|output) directory: .*\/(\d{8}T\d{6}\+\d{4})', stdout ) assert len(matches) == 2 From 240d72b58392e228b9cec38436070c356f9c95e0 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 17 Jul 2024 12:25:44 +0300 Subject: [PATCH 36/69] Address remaining MR comments --- reframe/frontend/reporting/__init__.py | 19 ++++++++++--------- reframe/schemas/runreport.json | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 4194998ef3..e4440236e2 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -33,6 +33,7 @@ DATA_VERSION = '4.0' _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') +_DATETIME_FMT = r'%Y%m%dT%H%M%S%z' def format_testcase(json, name='unique_name'): @@ -247,18 +248,18 @@ def update_restored_cases(self, restored_cases, restored_session): for c in restored_cases] def update_timestamps(self, ts_start, ts_end): - fmt = r'%Y%m%dT%H%M%S%z' self.__report['session_info'].update({ - 'time_start': time.strftime(fmt, time.localtime(ts_start)), + 'time_start': time.strftime(_DATETIME_FMT, + time.localtime(ts_start)), 'time_start_unix': ts_start, - 'time_end': time.strftime(fmt, time.localtime(ts_end)), + 'time_end': time.strftime(_DATETIME_FMT, time.localtime(ts_end)), 'time_end_unix': ts_end, 'time_elapsed': ts_end - ts_start }) def update_run_stats(self, stats): session_uuid = self.__report['session_info']['uuid'] - for runid, tasks in stats.runs(): + for runidx, tasks in stats.runs(): testcases = [] num_failures = 0 num_aborted = 0 @@ -291,7 +292,7 @@ def update_run_stats(self, stats): 'job_stdout': None, 'partition': partition.name, 'result': t.result, - 'runid': runid, + 'run_index': runidx, 'scheduler': partition.scheduler.registered_name, 'session_uuid': session_uuid, 'time_compile': t.duration('compile_complete'), @@ -300,7 +301,7 @@ def update_run_stats(self, stats): 'time_sanity': t.duration('sanity'), 'time_setup': t.duration('setup'), 'time_total': t.duration('total'), - 'uuid': f'{session_uuid}:{runid}:{tidx}' + 'uuid': f'{session_uuid}:{runidx}:{tidx}' } if check.job: entry['job_stderr'] = check.stderr.evaluate() @@ -357,7 +358,7 @@ def update_run_stats(self, stats): 'num_failures': num_failures, 'num_aborted': num_aborted, 'num_skipped': num_skipped, - 'runid': runid, + 'run_index': runidx, 'testcases': testcases }) @@ -515,7 +516,7 @@ def _group_testcases(testcases, group_by, extra_cols): def _aggregate_perf(grouped_testcases, aggr_fn, cols): - other_aggr = Aggregator.create('join_uniq', '|') + other_aggr = Aggregator.create('join_uniq', '\n') aggr_data = {} for key, seq in grouped_testcases.items(): aggr_data.setdefault(key, {}) @@ -638,7 +639,7 @@ def testcase_data(spec): nodelist_abbrev(tc['job_nodelist']), # Always format the completion time as users can set their own # formatting in the log record - time.strftime(r'%Y%m%dT%H%M%S%z', + time.strftime(_DATETIME_FMT, time.localtime(tc['job_completion_time_unix'])), tc['result'], tc['uuid'] diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index 006f39bc28..a216a5cd1b 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -88,7 +88,6 @@ "num_skipped": {"type": "number"}, "prefix_output": {"type": "string"}, "prefix_stage": {"type": "string"}, - "runid": {"type": "number"}, "session_uuid": {"type": "string"}, "time_elapsed": {"type": "number"}, "time_end": {"type": "string"}, @@ -101,7 +100,8 @@ "workdir": {"type": "string"} }, "required": ["data_version", "hostname", - "time_elapsed", "time_end", "time_start", "uuid"] + "time_elapsed", "time_end_unix", "time_start_unix", + "uuid"] }, "restored_cases": { "type": "array", @@ -116,7 +116,7 @@ "num_cases": {"type": "number"}, "num_failures": {"type": "number"}, "num_skipped": {"type": "number"}, - "runid": {"typeid": "number"}, + "run_index": {"type": "number"}, "testcases": { "type": "array", "items": {"$ref": "#/defs/testcase_type"} From b0548fc4b78d75ea911a0aff1ece32336a8966a8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 17 Jul 2024 14:57:26 +0300 Subject: [PATCH 37/69] Group by also system, partition and environment --- reframe/frontend/cli.py | 2 +- reframe/frontend/printer.py | 4 +-- reframe/frontend/reporting/__init__.py | 49 ++++++++++++++++---------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 14c3d435c6..8d27636ba3 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1230,7 +1230,7 @@ def print_infoline(param, value): sys.exit(1) def _case_failed(t): - rec = restored_session.case(*t) + rec = restored_session.case(t) if not rec: return False diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index cf0badb6c6..33450f971f 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -13,7 +13,7 @@ import reframe.core.runtime as rt import reframe.utility.color as color from reframe.core.exceptions import SanityError -from reframe.frontend.reporting import format_testcase +from reframe.frontend.reporting import format_testcase_from_json from reframe.utility import nodelist_abbrev @@ -236,7 +236,7 @@ def retry_report(self, report): for i, run in enumerate(report['runs'][1:], start=1): for tc in run['testcases']: # Overwrite entry from previous run if available - tc_info = format_testcase(tc) + tc_info = format_testcase_from_json(tc) messages[tc_info] = ( f" * Test {tc_info} was retried {i} time(s) and" f" {'failed' if tc['result'] == 'fail' else 'passed'}." diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index e4440236e2..314bcd83fe 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -36,13 +36,24 @@ _DATETIME_FMT = r'%Y%m%dT%H%M%S%z' -def format_testcase(json, name='unique_name'): +def _format_sysenv(system, partition, environ): + return f'{system}:{partition}+{environ}' + + +def format_testcase_from_json(tc): '''Format test case from its json representation''' - name = json[name] - system = json['system'] - partition = json['partition'] - environ = json['environ'] - return f'{name}@{system}:{partition}+{environ}' + name = tc['name'] + system = tc['system'] + partition = tc['partition'] + environ = tc['environ'] + return f'{name} @{_format_sysenv(system, partition, environ)}' + + +def format_testcase(tc): + return format_testcase_from_json({'name': tc.check.name, + 'system': tc.check.current_system.name, + 'partition': tc.partition.name, + 'environ': tc.environ.name}) class _RestoredSessionInfo: @@ -57,11 +68,11 @@ def __init__(self, report): self._cases_index = {} for run in self._report['runs']: for tc in run['testcases']: - self._cases_index[format_testcase(tc)] = tc + self._cases_index[format_testcase_from_json(tc)] = tc # Index also the restored cases for tc in self._report['restored_cases']: - self._cases_index[format_testcase(tc)] = tc + self._cases_index[format_testcase_from_json(tc)] = tc def __getitem__(self, key): return self._report[key] @@ -95,8 +106,8 @@ def slice(self, prop, when=None, unique=False): yield val - def case(self, check, part, env): - key = f'{check.unique_name}@{part.fullname}+{env.name}' + def case(self, tc): + key = format_testcase(tc) ret = self._cases_index.get(key) if ret is None: # Look up the case in the fallback reports @@ -123,7 +134,7 @@ def restore_dangling(self, graph): return graph, restored def _do_restore(self, testcase): - tc = self.case(*testcase) + tc = self.case(testcase) if tc is None: raise ReframeError( f'could not restore testcase {testcase!r}: ' @@ -244,7 +255,7 @@ def update_session_info(self, session_info): self.__report['session_info'][key] = val def update_restored_cases(self, restored_cases, restored_session): - self.__report['restored_cases'] = [restored_session.case(*c) + self.__report['restored_cases'] = [restored_session.case(c) for c in restored_cases] def update_timestamps(self, ts_start, ts_end): @@ -437,7 +448,7 @@ def generate_xml_report(self): ) etree.SubElement(xml_testsuite, 'properties') for tc in rfm_run['testcases']: - casename = f'{format_testcase(tc, name="name")}' + casename = f'{format_testcase_from_json(tc)}' testcase = etree.SubElement( xml_testsuite, 'testcase', attrib={ @@ -535,7 +546,8 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, extra_group_by=None, extra_cols=None): extra_group_by = extra_group_by or [] extra_cols = extra_cols or [] - group_by = ['name', 'pvar', 'punit'] + extra_group_by + group_by = (['name', 'system', 'partition', 'environ', 'pvar', 'punit'] + + extra_group_by) grouped_base = _group_testcases(base_testcases, group_by, extra_cols) grouped_target = _group_testcases(target_testcases, group_by, extra_cols) @@ -543,7 +555,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, ptarget = _aggregate_perf(grouped_target, target_fn, []) # Build the final table data - data = [['name', 'pvar', 'pval', + data = [['name', 'sysenv', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols] for key, aggr_data in pbase.items(): pval = aggr_data['pval'] @@ -558,8 +570,9 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, pdiff = (pval - target_pval) / target_pval pdiff = '{:+7.2%}'.format(pdiff) - name, pvar, punit, *extras = key - line = [name, pvar, pval, punit, pdiff, *extras] + name, system, partition, environ, pvar, punit, *extras = key + line = [name, _format_sysenv(system, partition, environ), + pvar, pval, punit, pdiff, *extras] # Add the extra columns line += [aggr_data[c] for c in extra_cols] data.append(line) @@ -635,7 +648,7 @@ def testcase_data(spec): for tc in testcases: data.append([ tc['name'], - f'{tc["system"]}:{tc["partition"]}@{tc["environ"]}', + _format_sysenv(tc['system'], tc['partition'], tc['environ']), nodelist_abbrev(tc['job_nodelist']), # Always format the completion time as users can set their own # formatting in the log record From a85d1bf33eafac9202170089e8d2d80bc0cd92e7 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 17 Jul 2024 17:10:52 +0300 Subject: [PATCH 38/69] Use the last successful retry as the reference test case for the performance report --- reframe/frontend/printer.py | 22 +++++++++++----------- reframe/frontend/reporting/__init__.py | 10 ++++++++-- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index 33450f971f..0767ac4dfe 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -228,22 +228,22 @@ def retry_report(self, report): # Do nothing if no retries return - line_width = shutil.get_terminal_size()[0] - lines = [line_width * '='] + line_width = min(80, shutil.get_terminal_size()[0]) + lines = ['', line_width * '='] lines.append('SUMMARY OF RETRIES') lines.append(line_width * '-') - messages = {} - for i, run in enumerate(report['runs'][1:], start=1): + retried_tc = set() + for run in reversed(report['runs'][1:]): + runidx = run['run_index'] for tc in run['testcases']: # Overwrite entry from previous run if available tc_info = format_testcase_from_json(tc) - messages[tc_info] = ( - f" * Test {tc_info} was retried {i} time(s) and" - f" {'failed' if tc['result'] == 'fail' else 'passed'}." - ) - - for msg in sorted(messages): - lines.append(msg) + if tc_info not in retried_tc: + lines.append( + f" * Test {tc_info} was retried {runidx} time(s) and " + f" {'failed' if tc['result'] == 'fail' else 'passed'}." + ) + retried_tc.add(tc_info) self.info('\n'.join(lines)) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 314bcd83fe..6139802cf2 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -590,10 +590,16 @@ def performance_compare(cmp, report=None): raise ValueError('report cannot be `None` ' 'for current run comparisons') try: - tcs_base = report['runs'][0]['testcases'] + # Get the last retry from every test case + num_runs = len(report['runs']) + tcs_base = [] + for run in report['runs']: + run_idx = run['run_index'] + for tc in run['testcases']: + if tc['result'] != 'fail' or run_idx == num_runs - 1: + tcs_base.append(tc) except IndexError: tcs_base = [] - elif match.period_base is not None: tcs_base = StorageBackend.default().fetch_testcases_time_period( *match.period_base From e2b4c51cb9dc7ce74d1c1c2d438fe3f8479d7165 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 18 Jul 2024 11:39:34 +0300 Subject: [PATCH 39/69] Move perflog testing to a different file --- unittests/conftest.py | 38 +++ unittests/test_perflogging.py | 458 ++++++++++++++++++++++++++++++++ unittests/test_policies.py | 476 ---------------------------------- 3 files changed, 496 insertions(+), 476 deletions(-) create mode 100644 unittests/test_perflogging.py diff --git a/unittests/conftest.py b/unittests/conftest.py index 1a2034f560..62d6ca126d 100644 --- a/unittests/conftest.py +++ b/unittests/conftest.py @@ -14,7 +14,10 @@ import reframe.core.settings as settings import reframe.core.runtime as rt +import reframe.frontend.executors as executors +import reframe.frontend.executors.policies as policies import reframe.utility as util +from reframe.frontend.loader import RegressionCheckLoader from .utility import TEST_CONFIG_FILE @@ -80,6 +83,41 @@ def _make_exec_ctx(*args, **kwargs): yield _make_exec_ctx +@pytest.fixture +def common_exec_ctx(make_exec_ctx_g): + '''Execution context for the default generic system.''' + yield from make_exec_ctx_g(system='generic') + + +@pytest.fixture +def testsys_exec_ctx(make_exec_ctx_g): + '''Execution context for the `testsys:gpu` system.''' + yield from make_exec_ctx_g(system='testsys:gpu') + + +@pytest.fixture +def make_loader(): + '''Test loader''' + def _make_loader(check_search_path, *args, **kwargs): + return RegressionCheckLoader(check_search_path, *args, **kwargs) + + return _make_loader + + +@pytest.fixture(params=[policies.SerialExecutionPolicy, + policies.AsynchronousExecutionPolicy]) +def make_runner(request): + '''Test runner with all the execution policies''' + + def _make_runner(*args, **kwargs): + # Use a much higher poll rate for the unit tests + policy = request.param() + policy._pollctl.SLEEP_MIN = 0.001 + return executors.Runner(policy, *args, **kwargs) + + return _make_runner + + @pytest.fixture def make_config_file(tmp_path): '''Create a temporary configuration file from the given configuration. diff --git a/unittests/test_perflogging.py b/unittests/test_perflogging.py new file mode 100644 index 0000000000..afc5e26f5e --- /dev/null +++ b/unittests/test_perflogging.py @@ -0,0 +1,458 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import contextlib +import io +import os +import pytest + +import reframe as rfm +import reframe.core.logging as logging +import reframe.core.runtime as rt +import reframe.frontend.executors as executors +import reframe.utility.osext as osext +import reframe.utility.sanity as sn + + +class _MyPerfTest(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo perf0=100 && echo perf1=50' + + @sanity_function + def validate(self): + return sn.assert_found(r'perf0', self.stdout) + + @performance_function('unit0') + def perf0(self): + return sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float) + + @performance_function('unit1') + def perf1(self): + return sn.extractsingle(r'perf1=(\S+)', self.stdout, 1, float) + + +class _MyPerfParamTest(_MyPerfTest): + p = parameter([1, 2]) + + +class _MyFailingTest(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo perf0=100' + + @sanity_function + def validate(self): + return False + + @performance_function('unit0') + def perf0(self): + return sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float) + + +class _LazyPerfTest(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo perf0=100' + + @sanity_function + def validate(self): + return True + + @run_before('performance') + def set_perf_vars(self): + self.perf_variables = { + 'perf0': sn.make_performance_function( + sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float), + 'unit0' + ) + } + + +@pytest.fixture +def perf_test(): + return _MyPerfTest() + + +@pytest.fixture +def perf_param_tests(): + return [_MyPerfParamTest(variant_num=v) + for v in range(_MyPerfParamTest.num_variants)] + + +@pytest.fixture +def failing_perf_test(): + return _MyFailingTest() + + +@pytest.fixture +def lazy_perf_test(): + return _LazyPerfTest() + + +@pytest.fixture +def simple_test(): + class _MySimpleTest(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo hello' + + @sanity_function + def validate(self): + return sn.assert_found(r'hello', self.stdout) + + return _MySimpleTest() + + +@pytest.fixture +def config_perflog(make_config_file): + def _config_perflog(fmt, perffmt=None, logging_opts=None): + logging_config = { + 'level': 'debug2', + 'handlers': [{ + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' + }], + 'handlers_perflog': [{ + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': fmt + }] + } + if logging_opts: + logging_config.update(logging_opts) + + if perffmt is not None: + logging_config['handlers_perflog'][0]['format_perfvars'] = perffmt + + return make_config_file({'logging': [logging_config]}) + + return _config_perflog + + +def _count_lines(filepath): + count = 0 + with open(filepath) as fp: + for line in fp: + count += 1 + + return count + + +def _assert_header(filepath, header): + with open(filepath) as fp: + assert fp.readline().strip() == header + + +def _assert_no_logging_error(fn, *args, **kwargs): + captured_stderr = io.StringIO() + with contextlib.redirect_stderr(captured_stderr): + fn(*args, **kwargs) + + assert 'Logging error' not in captured_stderr.getvalue() + + +def test_perf_logging(make_runner, make_exec_ctx, perf_test, + config_perflog, tmp_path): + make_exec_ctx( + config_perflog( + fmt=( + '%(check_job_completion_time)s,%(version)s,' + '%(check_display_name)s,%(check_system)s,' + '%(check_partition)s,%(check_environ)s,' + '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s' + ), + perffmt=( + '%(check_perf_value)s,%(check_perf_unit)s,' + '%(check_perf_ref)s,%(check_perf_lower_thres)s,' + '%(check_perf_upper_thres)s,' + ) + ) + ) + logging.configure_logging(rt.runtime().site_config) + runner = make_runner() + testcases = executors.generate_testcases([perf_test]) + runner.runall(testcases) + + logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log' + assert os.path.exists(logfile) + assert _count_lines(logfile) == 2 + + # Rerun with the same configuration and check that new entry is appended + testcases = executors.generate_testcases([perf_test]) + runner = make_runner() + _assert_no_logging_error(runner.runall, testcases) + assert _count_lines(logfile) == 3 + + # Change the configuration and rerun + make_exec_ctx( + config_perflog( + fmt=( + '%(check_job_completion_time)s,%(version)s,' + '%(check_display_name)s,%(check_system)s,' + '%(check_partition)s,%(check_environ)s,' + '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s' + ), + perffmt='%(check_perf_value)s,%(check_perf_unit)s,' + ) + ) + logging.configure_logging(rt.runtime().site_config) + testcases = executors.generate_testcases([perf_test]) + runner = make_runner() + _assert_no_logging_error(runner.runall, testcases) + assert _count_lines(logfile) == 2 + _assert_header(logfile, + 'job_completion_time,version,display_name,system,partition,' + 'environ,jobid,result,perf0_value,perf0_unit,' + 'perf1_value,perf1_unit') + + logfile_prev = [(str(logfile) + '.h0', 3)] + for f, num_lines in logfile_prev: + assert os.path.exists(f) + _count_lines(f) == num_lines + + # Change the test and rerun + perf_test.perf_variables['perfN'] = perf_test.perf_variables['perf1'] + + # We reconfigure the logging in order for the filelog handler to start + # from a clean state + logging.configure_logging(rt.runtime().site_config) + testcases = executors.generate_testcases([perf_test]) + runner = make_runner() + _assert_no_logging_error(runner.runall, testcases) + assert _count_lines(logfile) == 2 + _assert_header(logfile, + 'job_completion_time,version,display_name,system,partition,' + 'environ,jobid,result,perf0_value,perf0_unit,' + 'perf1_value,perf1_unit,perfN_value,perfN_unit') + + logfile_prev = [(str(logfile) + '.h0', 3), (str(logfile) + '.h1', 2)] + for f, num_lines in logfile_prev: + assert os.path.exists(f) + _count_lines(f) == num_lines + + +def test_perf_logging_no_end_delim(make_runner, make_exec_ctx, perf_test, + config_perflog, tmp_path): + make_exec_ctx( + config_perflog( + fmt=( + '%(check_job_completion_time)s,%(version)s,' + '%(check_display_name)s,%(check_system)s,' + '%(check_partition)s,%(check_environ)s,' + '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s' + ), + perffmt='%(check_perf_value)s,%(check_perf_unit)s' + ) + ) + logging.configure_logging(rt.runtime().site_config) + runner = make_runner() + testcases = executors.generate_testcases([perf_test]) + _assert_no_logging_error(runner.runall, testcases) + + logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log' + assert os.path.exists(logfile) + assert _count_lines(logfile) == 2 + + with open(logfile) as fp: + lines = fp.readlines() + + assert len(lines) == 2 + assert lines[0] == ( + 'job_completion_time,version,display_name,system,partition,' + 'environ,jobid,result,perf0_value,perf0_unitperf1_value,perf1_unit\n' + ) + assert ' Date: Thu, 18 Jul 2024 13:01:39 +0300 Subject: [PATCH 40/69] More report tests to a `test_reporting.py` --- unittests/conftest.py | 28 +++++ unittests/test_policies.py | 214 +----------------------------------- unittests/test_reporting.py | 174 +++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 208 deletions(-) create mode 100644 unittests/test_reporting.py diff --git a/unittests/conftest.py b/unittests/conftest.py index 62d6ca126d..69294e4100 100644 --- a/unittests/conftest.py +++ b/unittests/conftest.py @@ -14,6 +14,7 @@ import reframe.core.settings as settings import reframe.core.runtime as rt +import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors import reframe.frontend.executors.policies as policies import reframe.utility as util @@ -118,6 +119,33 @@ def _make_runner(*args, **kwargs): return _make_runner +@pytest.fixture +def make_cases(make_loader): + def _make_cases(checks=None, sort=False, *args, **kwargs): + if checks is None: + checks = make_loader( + ['unittests/resources/checks'], *args, **kwargs + ).load_all(force=True) + + cases = executors.generate_testcases(checks) + if sort: + depgraph, _ = dependencies.build_deps(cases) + dependencies.validate_deps(depgraph) + cases = dependencies.toposort(depgraph) + + return cases + + return _make_cases + + +@pytest.fixture +def cases_with_deps(make_loader, make_cases): + checks = make_loader( + ['unittests/resources/checks_unlisted/deps_complex.py'] + ).load_all() + return make_cases(checks, sort=True) + + @pytest.fixture def make_config_file(tmp_path): '''Create a temporary configuration file from the given configuration. diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 09c00bd338..37e40df5e9 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -4,32 +4,22 @@ # SPDX-License-Identifier: BSD-3-Clause import contextlib -import json -import jsonschema import os import pytest -import socket -import sys -import time import reframe as rfm import reframe.core.runtime as rt import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors import reframe.frontend.executors.policies as policies -import reframe.frontend.reporting as reporting -import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext import unittests.utility as test_util -from lxml import etree from reframe.core.exceptions import (AbortTaskError, FailureLimitError, ForceExitError, - ReframeError, RunSessionTimeout, TaskDependencyError) -from reframe.frontend.reporting import RunReport from unittests.resources.checks.hellocheck import HelloTest from unittests.resources.checks.frontend_checks import ( BadSetupCheck, @@ -43,25 +33,6 @@ ) -# NOTE: We could move this to utility -class timer: - '''Context manager for timing''' - - def __init__(self): - self._time_start = None - self._time_end = None - - def __enter__(self): - self._time_start = time.time() - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._time_end = time.time() - - def timestamps(self): - return self._time_start, self._time_end - - def make_kbd_check(phase='wait'): return test_util.make_check(KeyboardInterruptCheck, phase=phase) @@ -82,25 +53,6 @@ def _do_make_check(sleep_time, poll_fail=None): return _do_make_check -@pytest.fixture -def make_cases(make_loader): - def _make_cases(checks=None, sort=False, *args, **kwargs): - if checks is None: - checks = make_loader( - ['unittests/resources/checks'], *args, **kwargs - ).load_all(force=True) - - cases = executors.generate_testcases(checks) - if sort: - depgraph, _ = dependencies.build_deps(cases) - dependencies.validate_deps(depgraph) - cases = dependencies.toposort(depgraph) - - return cases - - return _make_cases - - @pytest.fixture(params=['pre_setup', 'post_setup', 'pre_compile', 'post_compile', 'pre_run', 'post_run', @@ -163,50 +115,9 @@ def num_failures_stage(runner, stage): return len([t for t in stats.failed() if t.failed_stage == stage]) -def _validate_runreport(report): - schema_filename = 'reframe/schemas/runreport.json' - with open(schema_filename) as fp: - schema = json.loads(fp.read()) - - jsonschema.validate(json.loads(report), schema) - - -def _validate_junit_report(report): - # Cloned from - # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd - schema_file = 'reframe/schemas/junit.xsd' - with open(schema_file, encoding='utf-8') as fp: - schema = etree.XMLSchema(etree.parse(fp)) - - schema.assert_(report) - - -def _generate_runreport(run_stats, time_start=None, time_end=None): - report = RunReport() - report.update_session_info({ - 'cmdline': ' '.join(sys.argv), - 'config_files': rt.runtime().site_config.sources, - 'data_version': reporting.DATA_VERSION, - 'hostname': socket.gethostname(), - 'prefix_output': rt.runtime().output_prefix, - 'prefix_stage': rt.runtime().stage_prefix, - 'user': osext.osuser(), - 'version': osext.reframe_version(), - 'workdir': os.getcwd() - }) - if time_start and time_end: - report.update_timestamps(time_start, time_end) - - if run_stats: - report.update_run_stats(run_stats) - - return report - - def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): runner = make_runner() - with timer() as tm: - runner.runall(make_cases()) + runner.runall(make_cases()) assert 9 == runner.stats.num_cases() assert_runall(runner) @@ -216,39 +127,6 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): assert 1 == num_failures_stage(runner, 'performance') assert 1 == num_failures_stage(runner, 'cleanup') - # We dump the report first, in order to get any object conversions right - report = _generate_runreport(runner.stats, *tm.timestamps()) - report.save(tmp_path / 'report.json') - - # We explicitly set `time_total` to `None` in the last test case, in order - # to test the proper handling of `None`. - report['runs'][0]['testcases'][-1]['time_total'] = None - - # Validate the junit report - _validate_junit_report(report.generate_xml_report()) - - # Read and validate the report using the `reporting` module - reporting.restore_session(tmp_path / 'report.json') - - # Try to load a non-existent report - with pytest.raises(ReframeError, match='failed to load report file'): - reporting.restore_session(tmp_path / 'does_not_exist.json') - - # Generate an invalid JSON - with open(tmp_path / 'invalid.json', 'w') as fp: - jsonext.dump(report, fp) - fp.write('invalid') - - with pytest.raises(ReframeError, match=r'is not a valid JSON file'): - reporting.restore_session(tmp_path / 'invalid.json') - - # Generate a report that does not comply to the schema - del report['session_info']['data_version'] - report.save(tmp_path / 'invalid-version.json') - with pytest.raises(ReframeError, - match=r'failed to validate report'): - reporting.restore_session(tmp_path / 'invalid-version.json') - def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx): runner = make_runner() @@ -339,18 +217,6 @@ def test_runall_skip_tests(make_runner, make_cases, more_cases, stage = make_cases_for_skipping() cases = make_cases() + more_cases runner.runall(cases) - - def assert_reported_skipped(num_skipped): - run_report = _generate_runreport(runner.stats)['runs'] - assert run_report[0]['num_skipped'] == num_skipped - - num_reported = 0 - for tc in run_report[0]['testcases']: - if tc['result'] == 'skip': - num_reported += 1 - - assert num_reported == num_skipped - assert_runall(runner) assert 11 == runner.stats.num_cases(0) assert 5 == runner.stats.num_cases(1) @@ -359,11 +225,9 @@ def assert_reported_skipped(num_skipped): if stage.endswith('cleanup'): assert 0 == len(runner.stats.skipped(0)) assert 0 == len(runner.stats.skipped(1)) - assert_reported_skipped(0) else: assert 2 == len(runner.stats.skipped(0)) assert 0 == len(runner.stats.skipped(1)) - assert_reported_skipped(2) # We explicitly ask for a system with a non-local scheduler here, to make sure @@ -478,18 +342,6 @@ def test_duration_limit(make_runner, make_cases, common_exec_ctx): assert num_aborted == 1 -@pytest.fixture -def dep_checks(make_loader): - return make_loader( - ['unittests/resources/checks_unlisted/deps_complex.py'] - ).load_all() - - -@pytest.fixture -def dep_cases(dep_checks, make_cases): - return make_cases(dep_checks, sort=True) - - def assert_dependency_run(runner): assert_runall(runner) stats = runner.stats @@ -511,15 +363,16 @@ def assert_dependency_run(runner): assert os.path.exists(os.path.join(check.outputdir, 'out.txt')) -def test_dependencies(make_runner, dep_cases, common_exec_ctx): +def test_dependencies(make_runner, cases_with_deps, common_exec_ctx): runner = make_runner() - runner.runall(dep_cases) + runner.runall(cases_with_deps) assert_dependency_run(runner) -def test_dependencies_with_retries(make_runner, dep_cases, common_exec_ctx): +def test_dependencies_with_retries(make_runner, cases_with_deps, + common_exec_ctx): runner = make_runner(max_retries=2) - runner.runall(dep_cases) + runner.runall(cases_with_deps) assert_dependency_run(runner) @@ -856,61 +709,6 @@ def test_compile_fail_reschedule_busy_loop(make_async_runner, make_cases, assert num_checks == len(stats.failed()) -@pytest.fixture -def report_file(make_runner, dep_cases, common_exec_ctx, tmp_path): - runner = make_runner() - runner.policy.keep_stage_files = True - with timer() as tm: - runner.runall(dep_cases) - - filename = tmp_path / 'report.json' - report = _generate_runreport(runner.stats, *tm.timestamps()) - report.save(filename) - return filename - - -def test_restore_session(report_file, make_runner, - dep_cases, common_exec_ctx, tmp_path): - # Select a single test to run and create the pruned graph - selected = [tc for tc in dep_cases if tc.check.name == 'T1'] - testgraph = dependencies.prune_deps( - dependencies.build_deps(dep_cases)[0], selected, max_depth=1 - ) - - # Restore the required test cases - report = reporting.restore_session(report_file) - testgraph, restored_cases = report.restore_dangling(testgraph) - - assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} - - # Run the selected test cases - runner = make_runner() - with timer() as tm: - runner.runall(selected, restored_cases) - - new_report = _generate_runreport(runner.stats, *tm.timestamps()) - assert new_report['runs'][0]['num_cases'] == 1 - assert new_report['runs'][0]['testcases'][0]['name'] == 'T1' - - # Generate an empty report and load it as primary with the original report - # as a fallback, in order to test if the dependencies are still resolved - # correctly - empty_report = _generate_runreport(None, *tm.timestamps()) - empty_report_file = tmp_path / 'empty.json' - empty_report.save(empty_report_file) - - report2 = reporting.restore_session(empty_report_file, report_file) - restored_cases = report2.restore_dangling(testgraph)[1] - assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} - - # Remove the test case dump file and retry - os.remove(tmp_path / 'stage' / 'generic' / 'default' / - 'builtin' / 'T4' / '.rfm_testcase.json') - - with pytest.raises(ReframeError, match=r'could not restore testcase'): - report.restore_dangling(testgraph) - - def test_config_params(make_runner, make_exec_ctx): '''Test that configuration parameters are properly retrieved with the various execution policies. diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py new file mode 100644 index 0000000000..2f809d0c64 --- /dev/null +++ b/unittests/test_reporting.py @@ -0,0 +1,174 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +import jsonschema +import os +import pytest +import socket +import sys +import time +from lxml import etree + +import reframe.core.runtime as rt +import reframe.utility.osext as osext +import reframe.utility.jsonext as jsonext +import reframe.frontend.dependencies as dependencies +import reframe.frontend.reporting as reporting +from reframe.core.exceptions import ReframeError +from reframe.frontend.reporting import RunReport + + +# NOTE: We could move this to utility +class _timer: + '''Context manager for timing''' + + def __init__(self): + self._time_start = None + self._time_end = None + + def __enter__(self): + self._time_start = time.time() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._time_end = time.time() + + def timestamps(self): + return self._time_start, self._time_end + + +def _validate_runreport(report): + schema_filename = 'reframe/schemas/runreport.json' + with open(schema_filename) as fp: + schema = json.loads(fp.read()) + + jsonschema.validate(json.loads(report), schema) + + +def _validate_junit_report(report): + # Cloned from + # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd + schema_file = 'reframe/schemas/junit.xsd' + with open(schema_file, encoding='utf-8') as fp: + schema = etree.XMLSchema(etree.parse(fp)) + + schema.assert_(report) + + +def _generate_runreport(run_stats, time_start=None, time_end=None): + report = RunReport() + report.update_session_info({ + 'cmdline': ' '.join(sys.argv), + 'config_files': rt.runtime().site_config.sources, + 'data_version': reporting.DATA_VERSION, + 'hostname': socket.gethostname(), + 'prefix_output': rt.runtime().output_prefix, + 'prefix_stage': rt.runtime().stage_prefix, + 'user': osext.osuser(), + 'version': osext.reframe_version(), + 'workdir': os.getcwd() + }) + if time_start and time_end: + report.update_timestamps(time_start, time_end) + + if run_stats: + report.update_run_stats(run_stats) + + return report + + +def test_run_report(make_runner, make_cases, common_exec_ctx, tmp_path): + runner = make_runner() + with _timer() as tm: + runner.runall(make_cases()) + + # We dump the report first, in order to get any object conversions right + report = _generate_runreport(runner.stats, *tm.timestamps()) + report.save(tmp_path / 'report.json') + + # We explicitly set `time_total` to `None` in the last test case, in order + # to test the proper handling of `None`. + report['runs'][0]['testcases'][-1]['time_total'] = None + + # Validate the junit report + _validate_junit_report(report.generate_xml_report()) + + # Read and validate the report using the `reporting` module + reporting.restore_session(tmp_path / 'report.json') + + # Try to load a non-existent report + with pytest.raises(ReframeError, match='failed to load report file'): + reporting.restore_session(tmp_path / 'does_not_exist.json') + + # Generate an invalid JSON + with open(tmp_path / 'invalid.json', 'w') as fp: + jsonext.dump(report, fp) + fp.write('invalid') + + with pytest.raises(ReframeError, match=r'is not a valid JSON file'): + reporting.restore_session(tmp_path / 'invalid.json') + + # Generate a report that does not comply to the schema + del report['session_info']['data_version'] + report.save(tmp_path / 'invalid-version.json') + with pytest.raises(ReframeError, + match=r'failed to validate report'): + reporting.restore_session(tmp_path / 'invalid-version.json') + + +@pytest.fixture +def report_file(make_runner, cases_with_deps, common_exec_ctx, tmp_path): + runner = make_runner() + runner.policy.keep_stage_files = True + with _timer() as tm: + runner.runall(cases_with_deps) + + filename = tmp_path / 'report.json' + report = _generate_runreport(runner.stats, *tm.timestamps()) + report.save(filename) + return filename + + +def test_restore_session(report_file, make_runner, cases_with_deps, + common_exec_ctx, tmp_path): + # Select a single test to run and create the pruned graph + selected = [tc for tc in cases_with_deps if tc.check.name == 'T1'] + testgraph = dependencies.prune_deps( + dependencies.build_deps(cases_with_deps)[0], selected, max_depth=1 + ) + + # Restore the required test cases + report = reporting.restore_session(report_file) + testgraph, restored_cases = report.restore_dangling(testgraph) + + assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} + + # Run the selected test cases + runner = make_runner() + with _timer() as tm: + runner.runall(selected, restored_cases) + + new_report = _generate_runreport(runner.stats, *tm.timestamps()) + assert new_report['runs'][0]['num_cases'] == 1 + assert new_report['runs'][0]['testcases'][0]['name'] == 'T1' + + # Generate an empty report and load it as primary with the original report + # as a fallback, in order to test if the dependencies are still resolved + # correctly + empty_report = _generate_runreport(None, *tm.timestamps()) + empty_report_file = tmp_path / 'empty.json' + empty_report.save(empty_report_file) + + report2 = reporting.restore_session(empty_report_file, report_file) + restored_cases = report2.restore_dangling(testgraph)[1] + assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} + + # Remove the test case dump file and retry + os.remove(tmp_path / 'stage' / 'generic' / 'default' / + 'builtin' / 'T4' / '.rfm_testcase.json') + + with pytest.raises(ReframeError, match=r'could not restore testcase'): + report.restore_dangling(testgraph) From b8d8e767e6f271bbf25e644283469cb00e70957e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 18 Jul 2024 17:54:15 +0300 Subject: [PATCH 41/69] Add unit tests for reporting backend --- reframe/frontend/executors/__init__.py | 3 +- reframe/frontend/reporting/__init__.py | 3 + reframe/frontend/reporting/storage.py | 11 +- reframe/frontend/reporting/utility.py | 17 +- unittests/conftest.py | 10 + unittests/test_reporting.py | 278 ++++++++++++++++++++++++- 6 files changed, 303 insertions(+), 19 deletions(-) diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 8bf3f58117..d96cbaeb72 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -314,7 +314,8 @@ def failed_stage(self): @property def succeeded(self): - return self._current_stage in {'finalize', 'cleanup'} + return (self._current_stage in {'finalize', 'cleanup'} and + not self._failed_stage == 'cleanup') @property def completed(self): diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 6139802cf2..3cca83792b 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -235,6 +235,9 @@ def __init__(self): now = time.time() self.update_timestamps(now, now) + def data(self): + return self.__report + @property def filename(self): return self.__filename diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 0fc89d5743..e4548de3fb 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -149,8 +149,8 @@ def _db_store_report(self, conn, report, report_file_path): def store(self, report, report_file=None): prefix = os.path.dirname(self.__db_file) - with FileLock(os.path.join(prefix, '.db.lock')): - with sqlite3.connect(self._db_file()) as conn: + with sqlite3.connect(self._db_file()) as conn: + with FileLock(os.path.join(prefix, '.db.lock')): return self._db_store_report(conn, report, report_file) def _fetch_testcases_raw(self, condition): @@ -231,6 +231,9 @@ def remove_session(self, uuid): prefix = os.path.dirname(self.__db_file) with FileLock(os.path.join(prefix, '.db.lock')): with sqlite3.connect(self._db_file()) as conn: - query = f'DELETE FROM sessions WHERE uuid == "{uuid}"' + query = (f'DELETE FROM sessions WHERE uuid == "{uuid}" ' + 'RETURNING *') getlogger().debug(query) - conn.execute(query) + deleted = conn.execute(query).fetchall() + if not deleted: + raise ReframeError(f'no such session: {uuid}') diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 6d4433815b..2da8030f53 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -110,7 +110,7 @@ def _do_parse(s): except ValueError as err: # Try the relative timestamps match = re.match( - r'(?P.*)(?P[\+|-]\d+)(?P[hdmsw])', s + r'(?P.*)(?P[\+|-]\d+)(?P[mhdw])', s ) if not match: raise err @@ -122,12 +122,10 @@ def _do_parse(s): ts += timedelta(weeks=amount) elif unit == 'd': ts += timedelta(days=amount) - elif unit == 'm': - ts += timedelta(minutes=amount) elif unit == 'h': ts += timedelta(hours=amount) - elif unit == 's': - ts += timedelta(seconds=amount) + elif unit == 'm': + ts += timedelta(minutes=amount) return ts.timestamp() @@ -156,12 +154,11 @@ def parse_time_period(s): def _parse_extra_cols(s): - try: - extra_cols = s.split('+')[1:] - except (ValueError, IndexError): - raise ValueError(f'invalid extra groups spec: {s}') from None + if s and not s.startswith('+'): + raise ValueError(f'invalid column spec: {s}') - return extra_cols + # Remove any empty columns + return [x for x in s.split('+')[1:] if x] def _parse_aggregation(s): diff --git a/unittests/conftest.py b/unittests/conftest.py index 69294e4100..711716d174 100644 --- a/unittests/conftest.py +++ b/unittests/conftest.py @@ -119,6 +119,16 @@ def _make_runner(*args, **kwargs): return _make_runner +@pytest.fixture +def make_async_runner(): + def _make_runner(*args, **kwargs): + policy = policies.AsynchronousExecutionPolicy() + policy._pollctl.SLEEP_MIN = 0.001 + return executors.Runner(policy, *args, **kwargs) + + return _make_runner + + @pytest.fixture def make_cases(make_loader): def _make_cases(checks=None, sort=False, *args, **kwargs): diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py index 2f809d0c64..a25d6410b4 100644 --- a/unittests/test_reporting.py +++ b/unittests/test_reporting.py @@ -7,7 +7,6 @@ import jsonschema import os import pytest -import socket import sys import time from lxml import etree @@ -17,6 +16,8 @@ import reframe.utility.jsonext as jsonext import reframe.frontend.dependencies as dependencies import reframe.frontend.reporting as reporting +import reframe.frontend.reporting.storage as report_storage +import reframe.frontend.reporting.utility as report_util from reframe.core.exceptions import ReframeError from reframe.frontend.reporting import RunReport @@ -45,7 +46,7 @@ def _validate_runreport(report): with open(schema_filename) as fp: schema = json.loads(fp.read()) - jsonschema.validate(json.loads(report), schema) + jsonschema.validate(jsonext.loads(jsonext.dumps(report)), schema) def _validate_junit_report(report): @@ -63,8 +64,6 @@ def _generate_runreport(run_stats, time_start=None, time_end=None): report.update_session_info({ 'cmdline': ' '.join(sys.argv), 'config_files': rt.runtime().site_config.sources, - 'data_version': reporting.DATA_VERSION, - 'hostname': socket.gethostname(), 'prefix_output': rt.runtime().output_prefix, 'prefix_stage': rt.runtime().stage_prefix, 'user': osext.osuser(), @@ -88,6 +87,7 @@ def test_run_report(make_runner, make_cases, common_exec_ctx, tmp_path): # We dump the report first, in order to get any object conversions right report = _generate_runreport(runner.stats, *tm.timestamps()) report.save(tmp_path / 'report.json') + _validate_runreport(report) # We explicitly set `time_total` to `None` in the last test case, in order # to test the proper handling of `None`. @@ -172,3 +172,273 @@ def test_restore_session(report_file, make_runner, cases_with_deps, with pytest.raises(ReframeError, match=r'could not restore testcase'): report.restore_dangling(testgraph) + + +@pytest.fixture(params=[ + ('20240701:20240701T0100', 3600), + ('20240701T0000:20240701T0010', 600), + ('20240701T000000:20240701T001010', 610), + ('20240701T000000+0000:20240701T000000+0100', -3600), + ('20240701T000000+0000:20240701T000000-0100', 3600), + ('20240701T0000:20240701T0000+1m', 60), + ('20240701T0000:20240701T0000-1m', -60), + ('20240701T0000:20240701T0000+1h', 3600), + ('20240701T0000:20240701T0000-1h', -3600), + ('20240701T0000:20240701T0000+1d', 86400), + ('20240701T0000:20240701T0000-1d', -86400), + ('20240701T0000:20240701T0000+1w', 604800), + ('20240701T0000:20240701T0000-1w', -604800), + ('now:now+1m', 60), + ('now:now-1m', -60), + ('now:now+1h', 3600), + ('now:now-1h', -3600), + ('now:now+1d', 86400), + ('now:now-1d', -86400), + ('now:now+1w', 604800), + ('now:now-1w', -604800) +]) +def time_period(request): + return request.param + + +def test_parse_cmp_spec_period(time_period): + spec, duration = time_period + duration = int(duration) + match = report_util.parse_cmp_spec(f'{spec}/{spec}/mean:/') + for period in ('period_base', 'period_target'): + ts_start, ts_end = getattr(match, period) + if 'now' in spec: + # Truncate splits of seconds if using `now` timestamps + ts_start = int(ts_start) + ts_end = int(ts_end) + + assert ts_end - ts_start == duration + + # Check variant without base period + match = report_util.parse_cmp_spec(f'{spec}/mean:/') + assert match.period_base is None + + +@pytest.fixture(params=['first', 'last', 'mean', 'median', + 'min', 'max']) +def aggregator(request): + return request.param + + +def test_parse_cmp_spec_aggregations(aggregator): + match = report_util.parse_cmp_spec(f'now-1m:now/now-1d:now/{aggregator}:/') + data = [1, 2, 3, 4, 5] + if aggregator == 'first': + match.aggregator(data) == data[0] + elif aggregator == 'last': + match.aggregator(data) == data[-1] + elif aggregator == 'min': + match.aggregator(data) == 1 + elif aggregator == 'max': + match.aggregator(data) == 5 + elif aggregator == 'median': + match.aggregator(data) == 3 + elif aggregator == 'mean': + match.aggregator(data) == sum(data) / len(data) + + # Check variant without base period + match = report_util.parse_cmp_spec(f'now-1d:now/{aggregator}:/') + assert match.period_base is None + + +@pytest.fixture(params=[('', []), ('+', []), + ('+col1', ['col1']), ('+col1+', ['col1']), + ('+col1+col2', ['col1', 'col2'])]) +def extra_cols(request): + return request.param + + +def test_parse_cmp_spec_group_by(extra_cols): + spec, expected = extra_cols + match = report_util.parse_cmp_spec( + f'now-1m:now/now-1d:now/min:{spec}/' + ) + assert match.extra_groups == expected + + # Check variant without base period + match = report_util.parse_cmp_spec(f'now-1d:now/min:{spec}/') + assert match.period_base is None + + +def test_parse_cmp_spec_extra_cols(extra_cols): + spec, expected = extra_cols + match = report_util.parse_cmp_spec( + f'now-1m:now/now-1d:now/min:/{spec}' + ) + assert match.extra_cols == expected + + # Check variant without base period + match = report_util.parse_cmp_spec(f'now-1d:now/min:/{spec}') + assert match.period_base is None + + +@pytest.fixture(params=['^uuid/now-1d:now/min:/', + 'now-1d:now/^uuid/min:/', + '^uuid0/^uuid1/min:/', + 'now-1m:now/now-1d:now/min:/']) +def uuid_spec(request): + return request.param + + +def test_parse_cmp_spec_with_uuid(uuid_spec): + def _uuids(s): + parts = s.split('/') + base, target = None, None + if len(parts) == 3: + base = None + target = parts[0][1:] if parts[0].startswith('^') else None + else: + base = parts[0][1:] if parts[0].startswith('^') else None + target = parts[1][1:] if parts[1].startswith('^') else None + + return base, target + + match = report_util.parse_cmp_spec(uuid_spec) + base_uuid, target_uuid = _uuids(uuid_spec) + assert match.session_base == base_uuid + assert match.session_target == target_uuid + + +@pytest.fixture(params=['2024:07:01T12:34:56', '20240701', '20240701:', + '20240701T:now', 'now-1v:now', 'now:then', + '20240701:now:']) +def invalid_time_period(request): + return request.param + + +def test_parse_cmp_spec_invalid_period(invalid_time_period): + with pytest.raises(ValueError): + report_util.parse_cmp_spec(f'{invalid_time_period}/now-1d:now/min:/') + + with pytest.raises(ValueError): + report_util.parse_cmp_spec(f'now-1d:now/{invalid_time_period}/min:/') + + +@pytest.fixture(params=['mean', 'foo:', 'mean:col1+col2', 'mean:col1,col2']) +def invalid_aggr_spec(request): + return request.param + + +def test_parse_cmp_spec_invalid_aggregation(invalid_aggr_spec): + with pytest.raises(ValueError): + report_util.parse_cmp_spec( + f'now-1m:now/now-1d:now/{invalid_aggr_spec}/' + ) + + +@pytest.fixture(params=['col1+col2', 'col1,col2']) +def invalid_col_spec(request): + return request.param + + +def test_parse_cmp_spec_invalid_extra_cols(invalid_col_spec): + with pytest.raises(ValueError): + report_util.parse_cmp_spec( + f'now-1m:now/now-1d:now/mean:/{invalid_col_spec}' + ) + + +@pytest.fixture(params=['now-1d:now', + 'now-1m:now/now-1d:now', + 'now-1m:now/now-1d:now/mean', + 'now-1m:now/now-1d:now/mean:', + 'now-1m:now/now-1d:now/mean:', + '/now-1d:now/mean:/', + 'now-1m:now//mean:']) +def various_invalid_specs(request): + return request.param + + +def test_parse_cmp_spec_various_invalid(various_invalid_specs): + with pytest.raises(ValueError): + report_util.parse_cmp_spec(various_invalid_specs) + + +def test_storage_api(make_async_runner, make_cases, common_exec_ctx, + monkeypatch, tmp_path): + def _count_failed(testcases): + count = 0 + for tc in testcases: + if tc['result'] == 'fail': + count += 1 + + return count + + monkeypatch.setenv('HOME', str(tmp_path)) + uuids = [] + timestamps = [] + for _ in range(2): + runner = make_async_runner() + with _timer() as tm: + runner.runall(make_cases()) + + timestamps.append(tm.timestamps()) + report = _generate_runreport(runner.stats, *tm.timestamps()) + uuids.append(report.store()) + + backend = report_storage.StorageBackend.default() + + # Test `fetch_all_sessions` + stored_sessions = backend.fetch_all_sessions() + assert len(stored_sessions) == 2 + for i, sess in enumerate(stored_sessions): + assert sess['session_info']['uuid'] == uuids[i] + + # Test `fetch_session_json` + for uuid in uuids: + stored_session = backend.fetch_session_json(uuid) + assert stored_session['session_info']['uuid'] == uuid + + # Test an invalid uuid + assert backend.fetch_session_json(0) == {} + + # Test `fetch_session_time_period` + for i, uuid in enumerate(uuids): + ts_session = backend.fetch_session_time_period(uuid) + assert ts_session == timestamps[i] + + # Test an invalid uuid + assert backend.fetch_session_time_period(0) == (None, None) + + # Test `fetch_testcases_time_period` + testcases = backend.fetch_testcases_time_period(timestamps[0][0], + timestamps[1][1]) + + # NOTE: test cases without an associated (run) job are not fetched by + # `fetch_testcases_time_period`; in + # this case 3 test cases per session are ignored: `BadSetupCheckEarly`, + # `BadSetupCheck`, `CompileOnlyHelloTest`, which requires us to adapt the + # expected counts below + assert len(testcases) == 12 + assert _count_failed(testcases) == 6 + + # Test the inverted period + assert backend.fetch_testcases_time_period(timestamps[1][1], + timestamps[0][0]) == [] + + # Test `fetch_testcases_from_session` + for i, uuid in enumerate(uuids): + testcases = backend.fetch_testcases_from_session(uuid) + assert len(testcases) == 9 + assert _count_failed(testcases) == 5 + + # Test an invalid uuid + assert backend.fetch_testcases_from_session(0) == [] + + # Test session removal + backend.remove_session(uuids[-1]) + assert len(backend.fetch_all_sessions()) == 1 + + testcases = backend.fetch_testcases_time_period(timestamps[0][0], + timestamps[1][1]) + assert len(testcases) == 6 + assert _count_failed(testcases) == 3 + + # Try an invalid uuid + with pytest.raises(ReframeError): + backend.remove_session(0) From 88f68635d05b7f2b7bbae72a875af3dd4c710c47 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 7 Aug 2024 13:32:36 +0300 Subject: [PATCH 42/69] Add CLI unit tests for storage options --- reframe/frontend/cli.py | 4 +- unittests/test_cli.py | 132 +++++++++++++++++++++++++++++++--------- 2 files changed, 104 insertions(+), 32 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 8d27636ba3..9acf2ae73e 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -408,7 +408,7 @@ def main(): ) action_options.add_argument( '--describe-stored-testcases', action='store', - metavar='SESSION_UUID|PERIOD', + metavar='^SESSION_UUID|PERIOD', help='Get detailed test case information in JSON' ) action_options.add_argument( @@ -428,7 +428,7 @@ def main(): ) action_options.add_argument( '--list-stored-sessions', action='store_true', - help='List stored session' + help='List stored sessions' ) action_options.add_argument( '--list-stored-testcases', action='store', diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 04051c7b36..a5d02d6db4 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -107,6 +107,8 @@ def _run_reframe(system='generic:default', argv += ['-h'] elif action == 'describe': argv += ['--describe'] + else: + argv += [action] if perflogdir: argv += ['--perflogdir', perflogdir] @@ -164,10 +166,7 @@ def test_check_restore_session_failed(run_reframe, tmp_path): run_reframe( checkpath=['unittests/resources/checks_unlisted/deps_complex.py'] ) - returncode, stdout, _ = run_reframe( - checkpath=[], - more_options=['--restore-session', '--failed'] - ) + run_reframe(checkpath=[], more_options=['--restore-session', '--failed']) report = reporting.restore_session( f'{tmp_path}/.reframe/reports/latest.json' ) @@ -186,10 +185,7 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path): checkpath=['unittests/resources/checks_unlisted/deps_complex.py'], more_options=['--keep-stage-files'] ) - returncode, stdout, _ = run_reframe( - checkpath=[], - more_options=['--restore-session', '-n', 'T1'] - ) + run_reframe(checkpath=[], more_options=['--restore-session', '-n', 'T1']) report = reporting.restore_session( f'{tmp_path}/.reframe/reports/latest.json' ) @@ -747,9 +743,8 @@ def test_filtering_by_expr(run_reframe): def test_show_config_all(run_reframe): # Just make sure that this option does not make the frontend crash returncode, stdout, stderr = run_reframe( - more_options=['--show-config'], - system='testsys', - action=None + action='--show-config', + system='testsys' ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -759,9 +754,8 @@ def test_show_config_all(run_reframe): def test_show_config_param(run_reframe): # Just make sure that this option does not make the frontend crash returncode, stdout, stderr = run_reframe( - more_options=['--show-config=systems'], - system='testsys', - action=None + action='--show-config=systems', + system='testsys' ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -771,9 +765,8 @@ def test_show_config_param(run_reframe): def test_show_config_unknown_param(run_reframe): # Just make sure that this option does not make the frontend crash returncode, stdout, stderr = run_reframe( - more_options=['--show-config=foo'], - system='testsys', - action=None + action='--show-config=foo', + system='testsys' ) assert 'no such configuration parameter found' in stdout assert 'Traceback' not in stdout @@ -783,9 +776,8 @@ def test_show_config_unknown_param(run_reframe): def test_show_config_null_param(run_reframe): returncode, stdout, stderr = run_reframe( - more_options=['--show-config=general/report_junit'], - system='testsys', - action=None + action='--show-config=general/report_junit', + system='testsys' ) assert 'null' in stdout assert 'Traceback' not in stdout @@ -1115,10 +1107,7 @@ def test_exec_order(run_reframe, exec_order): def test_detect_host_topology(run_reframe): from reframe.utility.cpuinfo import cpuinfo - returncode, stdout, stderr = run_reframe( - more_options=['--detect-host-topology'], - action=None - ) + returncode, stdout, stderr = run_reframe(action='--detect-host-topology') assert 'Traceback' not in stdout assert 'Traceback' not in stderr assert returncode == 0 @@ -1130,8 +1119,7 @@ def test_detect_host_topology_file(run_reframe, tmp_path): topo_file = tmp_path / 'topo.json' returncode, stdout, stderr = run_reframe( - more_options=[f'--detect-host-topology={topo_file}'], - action=None + action=f'--detect-host-topology={topo_file}', ) assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -1198,7 +1186,7 @@ def test_fixture_registry_env_sys(run_reframe): assert returncode == 0 assert 'e1' in stdout assert 'sys1:p0' in stdout - returncode, stdout, stderr = run_reframe( + returncode, stdout, _ = run_reframe( system='sys1:p1', environs=['e1'], checkpath=['unittests/resources/checks_unlisted/fixtures_simple.py'], @@ -1208,7 +1196,7 @@ def test_fixture_registry_env_sys(run_reframe): assert returncode == 0 assert 'e1' in stdout assert 'sys1:p1' in stdout - returncode, stdout, stderr = run_reframe( + returncode, stdout, _ = run_reframe( system='sys1:p1', environs=['e2'], checkpath=['unittests/resources/checks_unlisted/fixtures_simple.py'], @@ -1230,7 +1218,7 @@ def test_fixture_resolution(run_reframe, run_action): assert returncode == 0 -def test_dynamic_tests(run_reframe, tmp_path, run_action): +def test_dynamic_tests(run_reframe, run_action): returncode, stdout, _ = run_reframe( system='sys0', environs=[], @@ -1243,7 +1231,7 @@ def test_dynamic_tests(run_reframe, tmp_path, run_action): assert 'FAILED' not in stdout -def test_dynamic_tests_filtering(run_reframe, tmp_path, run_action): +def test_dynamic_tests_filtering(run_reframe, run_action): returncode, stdout, _ = run_reframe( system='sys1', environs=[], @@ -1267,3 +1255,87 @@ def test_testlib_inherit_fixture_in_different_files(run_reframe): assert returncode == 0 assert 'Ran 3/3 test case(s)' in stdout assert 'FAILED' not in stdout + + +def test_storage_options(run_reframe, tmp_path): + def assert_no_crash(returncode, stdout, stderr, exitcode=0): + assert returncode == exitcode + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + return returncode, stdout, stderr + + run_reframe2 = functools.partial( + run_reframe, + checkpath=['unittests/resources/checks/frontend_checks.py'] + ) + + # Run first a normal run with a performance test to initialize the DB + run_reframe2(action='run') + assert os.path.exists(tmp_path / '.reframe' / 'reports' / 'results.db') + + stdout = assert_no_crash( + *run_reframe(action='--list-stored-sessions') + )[1] + + # Get the session uuid for later queries + uuid = re.search(r'(\S+-\S+)', stdout).group(1) + + # Get details from the last session + stdout = assert_no_crash( + *run_reframe2(action=f'--describe-stored-session={uuid}') + )[1] + session_json = json.loads(stdout) + + # List test cases by session + assert_no_crash(*run_reframe2(action=f'--list-stored-testcases=^{uuid}')) + assert_no_crash( + *run_reframe2(action=f'--describe-stored-testcases=^{uuid}') + ) + + # List test cases by time period + ts_start = session_json['session_info']['time_start'] + assert_no_crash( + *run_reframe2(action=f'--list-stored-testcases={ts_start}:now') + ) + assert_no_crash( + *run_reframe2(action=f'--describe-stored-testcases={ts_start}:now') + ) + + # Check that invalid argument do not crash CLI + assert_no_crash(*run_reframe2(action='--describe-stored-session=0'), + exitcode=1) + assert_no_crash(*run_reframe2(action='--describe-stored-testcases=0'), + exitcode=1) + assert_no_crash(*run_reframe2(action='--list-stored-testcases=0'), + exitcode=1) + + # Remove session + assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}')) + + +def test_performance_compare(run_reframe): + def assert_no_crash(returncode, stdout, stderr, exitcode=0): + assert returncode == exitcode + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + return returncode, stdout, stderr + + run_reframe2 = functools.partial( + run_reframe, + checkpath=['unittests/resources/checks/frontend_checks.py'] + ) + run_reframe2(action='run') + + # Rerun with various arguments + assert_no_crash( + *run_reframe2( + action='--performance-compare=now-1m:now/now-1d:now/mean:/+result' + ) + ) + + # Check that invalid arguments do not crash the CLI + assert_no_crash( + *run_reframe2( + action='--performance-compare=now-1m:now/now-1d:now/mean:+foo/+bar' + ), exitcode=1 + ) From d05e0f4096c2efa9fe7e1818f8e4a60eec5e4ecc Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 7 Aug 2024 16:03:01 +0300 Subject: [PATCH 43/69] Use alternative method for session deletion with Sqlite < 3.35 --- reframe/frontend/reporting/storage.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index e4548de3fb..940ed24840 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -227,7 +227,22 @@ def fetch_session_json(self, uuid): return jsonext.loads(results[0][0]) if results else {} - def remove_session(self, uuid): + def _do_remove(self, uuid): + prefix = os.path.dirname(self.__db_file) + with FileLock(os.path.join(prefix, '.db.lock')): + with sqlite3.connect(self._db_file()) as conn: + # Check first if the uuid exists + query = f'SELECT * FROM sessions WHERE uuid == "{uuid}"' + getlogger().debug(query) + if not conn.execute(query).fetchall(): + raise ReframeError(f'no such session: {uuid}') + + query = f'DELETE FROM sessions WHERE uuid == "{uuid}"' + getlogger().debug(query) + conn.execute(query) + + def _do_remove2(self, uuid): + '''Remove a session using the RETURNING keyword''' prefix = os.path.dirname(self.__db_file) with FileLock(os.path.join(prefix, '.db.lock')): with sqlite3.connect(self._db_file()) as conn: @@ -237,3 +252,9 @@ def remove_session(self, uuid): deleted = conn.execute(query).fetchall() if not deleted: raise ReframeError(f'no such session: {uuid}') + + def remove_session(self, uuid): + if sqlite3.sqlite_version_info >= (3, 35, 0): + self._do_remove2(uuid) + else: + self._do_remove(uuid) From 60b37919b1106655f167cb88440ec91f07789ac8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 7 Aug 2024 16:21:06 +0300 Subject: [PATCH 44/69] Fix unit tests for Python 3.6 --- unittests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index a5d02d6db4..b63590afbc 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1278,7 +1278,7 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0): )[1] # Get the session uuid for later queries - uuid = re.search(r'(\S+-\S+)', stdout).group(1) + uuid = re.search(r'(\w{8}-\w{4}-\w{4}-\w{4}-\w{12})', stdout).group(1) # Get details from the last session stdout = assert_no_crash( From 4fd81cad0815a9d6cc23b907741f05fae9429960 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 8 Aug 2024 17:56:04 +0300 Subject: [PATCH 45/69] Update manpage --- docs/manpage.rst | 376 ++++++++++++++++++++++++++++++++-------- reframe/frontend/cli.py | 6 +- 2 files changed, 305 insertions(+), 77 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 60b0f9d2d8..053abae1e0 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -17,8 +17,16 @@ The ``reframe`` command is part of ReFrame's frontend. This frontend is responsible for loading and running regression tests written in ReFrame. ReFrame executes tests by sending them down to a well defined pipeline. The implementation of the different stages of this pipeline is part of ReFrame's core architecture, but the frontend is responsible for driving this pipeline and executing tests through it. -There are three basic phases that the frontend goes through, which are described briefly in the following. +Usually, ReFrame processes tests in three phases: +1. It :ref:`discovers and loads tests ` from the filesystem. +2. It :ref:`filters ` the loaded tests based on the current system and any other criteria specified by the user. +3. It :ref:`acts ` upon the selected tests. + +There are also ReFrame commands that do not operate on a set of tests. + + +.. _test-discovery: ------------------------------- Test discovery and test loading @@ -234,18 +242,24 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep The ``NAME`` pattern is matched anywhere in the test name and not at its beginning. If you want to match at the beginning of a test name, you should prepend ``^``. ------------- -Test actions ------------- -ReFrame will finally act upon the selected tests. -There are currently two actions that can be performed on tests: (a) list the tests and (b) execute the tests. -An action must always be specified. +.. _commands: + +-------- +Commands +-------- + +ReFrame commands are mutually exclusive and one of them must always be specified. +There are commands that act upon the selected tests and others that have a helper function, such as querying the configuration, querying the results database etc. + +.. versionchanged:: 4.7 + + ReFrame commands are now mutually exclusive and only one can be specified every time. .. option:: --ci-generate=FILE - Do not run the tests, but generate a Gitlab `child pipeline `__ specification in ``FILE``. + Generate a Gitlab `child pipeline `__ specification in ``FILE`` that will run the selected tests. You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies. For more information, have a look in :ref:`generate-ci-pipeline`. @@ -256,6 +270,12 @@ An action must always be specified. .. versionadded:: 3.4.1 +.. option:: --delete-stored-session=UUID + + Delete the stored session with the specified UUID from the results database. + + .. versionadded:: 4.7 + .. option:: --describe Print a detailed description of the `selected tests <#test-filtering>`__ in JSON format and exit. @@ -267,6 +287,34 @@ An action must always be specified. .. versionadded:: 3.10.0 +.. option:: --describe-stored-session=UUID + + Get detailed information of the session with the specified UUID. + The output is in JSON format. + + .. versionadded:: 4.7 + +.. option:: --describe-stored-testcases=^SESSION_UUID|TIME_PERIOD + + Get detailed test case information of the session with the specified UUID or from the specified time period. + + If a session UUID is provided only information about the test cases of this session will be provided. + Note that the session UUID must be prefixed with ``^``. + + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. + + .. versionadded:: 4.7 + +.. _--detect-host-topology: + +.. option:: --detect-host-topology[=FILE] + + Detect the local host processor topology, store it to ``FILE`` and exit. + + If no ``FILE`` is specified, the standard output will be used. + + .. versionadded:: 3.7.0 + .. option:: --dry-run Dry run the selected tests. @@ -280,7 +328,6 @@ An action must always be specified. .. versionadded:: 4.1 - .. option:: -L, --list-detailed[=T|C] List selected tests providing more details for each test. @@ -297,6 +344,23 @@ An action must always be specified. The variable names to which fixtures are bound are also listed. See :ref:`test_naming_scheme` for more information. +.. option:: --list-stored-sessions + + List all sessions stored in the results database. + + .. versionadded:: 4.7 + +.. option:: --list-stored-testcases=^SESSION_UUID|TIME_PERIOD + + List all test cases from the session with the specified UUID or from the specified time period. + + If a session UUID is provided only the test cases of this session will be listed. + Note that the session UUID must be prefixed with ``^``. + + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. + + .. versionadded:: 4.7 + .. option:: -l, --list[=T|C] List selected tests and their dependencies. @@ -323,15 +387,50 @@ An action must always be specified. .. versionadded:: 3.6.0 +.. option:: --performance-compare=CMPSPEC + + Compare the performance of test cases that have run in the past. + + Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``. + + .. versionadded:: 4.7 + .. option:: -r, --run - Execute the selected tests. + Run the selected tests. + +.. option:: --show-config [PARAM] + + Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit. + + The parameter value is printed in JSON format. + If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown. + Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter. + The ``/`` character acts as a selector of configuration object properties or an index in array objects. + The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property. + Here are some example queries: + + - Retrieve all the partitions of the current system: + + .. code:: bash + + reframe --show-config=systems/0/partitions + + - Retrieve the job scheduler of the partition named ``default``: + + .. code:: bash -If more than one action options are specified, the precedence order is the following: + reframe --show-config=systems/0/partitions/@default/scheduler - .. code-block:: console + - Retrieve the check search path for system ``foo``: - --describe > --list-detailed > --list > --list-tags > --ci-generate + .. code:: bash + + reframe --system=foo --show-config=general/0/check_search_path + +.. option:: -V, --version + + Print version and exit. ---------------------------------- @@ -971,16 +1070,6 @@ Miscellaneous options .. versionchanged:: 4.0.0 -.. _--detect-host-topology: - -.. option:: --detect-host-topology[=FILE] - - Detect the local host processor topology, store it to ``FILE`` and exit. - - If no ``FILE`` is specified, the standard output will be used. - - .. versionadded:: 3.7.0 - .. option:: --failure-stats Print failure statistics at the end of the run. @@ -995,11 +1084,21 @@ Miscellaneous options This option can also be set using the :envvar:`RFM_COLORIZE` environment variable or the :attr:`~config.general.colorize` general configuration parameter. -.. option:: --performance-report +.. _--performance-report: + +.. option:: --performance-report[=CMPSPEC] - Print a performance report for all the performance tests that have been run. + Print a report summarizing the performance of all performance tests that have run in the current session. - The report shows the performance values retrieved for the different performance variables defined in the tests. + For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified. + + If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run oif the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``). + + For the exact syntax of ``CMPSPEC``, refer to :ref:`performance-comparisons`. + + .. versionchanged:: 4.7 + + The format of the performance report has changed and the optional ``CMPSPEC`` argument is now added. .. option:: -q, --quiet @@ -1014,35 +1113,6 @@ Miscellaneous options .. versionadded:: 3.9.3 -.. option:: --show-config [PARAM] - - Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit. - - The parameter value is printed in JSON format. - If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown. - Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter. - The ``/`` character acts as a selector of configuration object properties or an index in array objects. - The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property. - Here are some example queries: - - - Retrieve all the partitions of the current system: - - .. code:: bash - - reframe --show-config=systems/0/partitions - - - Retrieve the job scheduler of the partition named ``default``: - - .. code:: bash - - reframe --show-config=systems/0/partitions/@default/scheduler - - - Retrieve the check search path for system ``foo``: - - .. code:: bash - - reframe --system=foo --show-config=general/0/check_search_path - .. option:: --system=NAME Load the configuration for system ``NAME``. @@ -1062,10 +1132,6 @@ Miscellaneous options If a new file is not given, a file in the system temporary directory will be created. -.. option:: -V, --version - - Print version and exit. - .. option:: -v, --verbose Increase verbosity level of output. @@ -1210,6 +1276,139 @@ Very large test names meant also very large path names which could also lead to Fixtures followed a similar naming pattern making them hard to debug. +Result storage +-------------- + +.. versionadded:: 4.7 + +ReFrame stores the results of every session that has exectuted at least one test into a database. +There is only one storage backend supported at the moment and this is Sqlite. +The full session information as recorded in a run report file (see :option:`--report-file`) is stored in the database. +The test cases of the session are indexed by their run job completion time for quick retrieval of all the test cases that have run in a certain period of time. + +The database file is controlled by the :attr:`~config.storage.sqlite_db_file` configuration parameter and multiple ReFrame processes can access it safely simultaneously. + +There are several command-line options that allow users to query the results database, such as the :option:`--list-stored-sessions`, :option:`--list-stored-testcases`, :option:`--describe-stored-session` etc. +Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` options which compare the performance results of the same test cases in different periods of time or from different sessions. +Check the :ref:`commands` section for the complete list and details of each option related to the results database. + +Since the report file information is now kept in the results database, there is no need to keep the report files separately, although this remains the default behavior for backward compatibility. +You can disable the report generation by turning off the :attr:`~config.general.generate_file_reports` configuration parameter. +The file report of any session can be retrieved from the database with the :option:`--describe-stored-session` option. + + +.. _performance-comparisons: + +Performance comparisons +----------------------- + +.. versionadded:: 4.7 + +The :option:`--performance-compare` and :option:`--performance-report` options accept a ``CMPSPEC`` argument that specifies how to select and compare test cases. +The full syntax of ``CMPSPEC`` is the following: + +.. code-block:: console + + /// + +The ```` and ```` subspecs specify how the base and target test cases will be retrieved. +The base test cases will be compared against those from the target period. + +.. note:: + + The ```` subspec is ommitted from the ``CMPSPEC`` of the :option:`--performance-report` option as the base test cases are always the test cases from the current session. + +The test cases for comparison can either be retrieved from an existing past session or a past time period. +A past session is denoted with the ``^`` syntax and only the test cases of that particular session will be selected. +To view the UUIDs of all stored sessions, use the :option:`--list-stored-sessions` option. + +To retrieve results from a time period, check the :ref:`time period syntax ` below. + +The ```` subspec specifies how the performance of both the base and target cases should be grouped and aggregated. +The syntax is the following: + +.. code-block:: console + + :[+]* + +The ```` is a symbolic name for a function to aggregate the grouped test cases. +It can take one of the following values: + +- ``first``: retrieve the performance data of the first test case only +- ``last``: retrieve the performance data of the last test case only +- ``max``: retrieve the maximum of all test cases +- ``mean``: calculate the mean over all test cases +- ``median``: retrieve the median of all test cases +- ``min``: retrieve the minimum of all test cases + +The test cases are always grouped by the following attributes: + +- The test :attr:`~reframe.core.pipeline.RegressionTest.name` +- The system name +- The partition name +- The environment name +- The performance variable name (see :func:`@performance_function ` and :attr:`~reframe.core.pipeline.RegressionTest.perf_variables`) +- The performance variable unit + +The ``+`` subspec specifies additional attributes to group the test cases by. +Any loggable test attribute can be selected. + +.. note:: + + The loggable attributes of a test are the same as the ones list in the logging :attr:`~config.logging.handlers_perflog.format` option but without the ``check_`` prefix. + +Finally, the ```` subspec specifies additional test attributes to list as columns in the resulting comparison table. +The syntax is the following: + +.. code-block:: console + + [+]* + +``col`` refers to any loggable attribute of the test. +If these attributes have different values across the aggregated test cases, +the unique values will be joined using the ``|`` separator. + +Here are some examples of performance comparison specs: + +- Compare the test cases of the session ``7a70b2da-1544-4ac4-baf4-0fcddd30b672`` with the mean performance of the last 10 days: + + .. code-block:: console + + ^7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/ + +- Compare the best performance of the test cases run on two specific days, group by the node list and report also the test result: + + .. code-block:: console + + 20240701:20240701+1d/20240705:20240705+1d/mean:+job_nodelist/+result + +.. _time-period-syntax: + +Time periods +------------ + +A time period needs to be specified as part of the ``CMPSPEC`` of the :option:`--performance-compare` and :option:`--performance-report` options or as an argument to options that request past results from results database. + +The general syntax of time period subspec is the following: + +.. code-block:: console + + : + +```` and ```` are timestamp denoting the start and end of the requested period. +More specifically, the syntax of each timestamp is the following: + +.. code-block:: console + + [+|-w|d|h|m] + +The ```` is an absolute timestamp in one of the following ``strptime``-compatible formats or the special value ``now``: ``%Y%m%d``, ``%Y%m%dT%H%M``, ``%Y%m%dT%H%M%S``, ``%Y%m%dT%H%M%S%z``. + +Optionally, a shift argument can be appended with ``+`` or ``-`` signs, followed by an amount of weeks (``w``), days (``d``), hours (``h``) or minutes (``m``). + +For example, the period of the last 10 days can be specified as ``now:now-10d``. +Similarly, the period of the week starting on August 5, 2024 will be specified as ``20240805:20240805+1w``. + Environment ----------- @@ -1226,21 +1425,6 @@ Here is an alphabetical list of the environment variables recognized by ReFrame. Whenever an environment variable is associated with a configuration option, its default value is omitted as it is the same. -.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT - - Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm). - - .. table:: - :align: left - - ================================== ================== - Associated command line option N/A - Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit` - ================================== ================== - -.. versionadded:: 4.7 - - .. envvar:: RFM_AUTODETECT_FQDN Use the fully qualified domain name as the hostname. @@ -1452,6 +1636,20 @@ Whenever an environment variable is associated with a configuration option, its .. versionadded:: 4.7 +.. envvar:: RFM_GENERATE_FILE_REPORTS + + Store session reports also in files. + + .. table:: + :align: left + + ================================== ================== + Associated command line option n/a + Associated configuration parameter :attr:`~config.general.generate_file_reports` + ================================== ================== + + .. versionadded:: 4.7 + .. envvar:: RFM_GIT_TIMEOUT Timeout value in seconds used when checking if a git repository exists. @@ -1600,6 +1798,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_PERF_REPORT_SPEC + + The default ``CMPSPEC`` of the :option:`--performance-report` option. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--performance-report` + Associated configuration parameter :attr:`~config.general.perf_report_spec` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_PERFLOG_DIR Directory prefix for logging performance data. @@ -1744,6 +1957,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT + + Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm). + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit` + ================================== ================== + +.. versionadded:: 4.7 + + .. envvar:: RFM_STAGE_DIR Directory prefix for staging test resources. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 9acf2ae73e..6835ea797c 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -395,7 +395,7 @@ def main(): 'for the selected tests and exit'), ) action_options.add_argument( - '--delete-stored-session', action='store', metavar='SESSION_UUID', + '--delete-stored-session', action='store', metavar='UUID', help='Delete stored session' ) action_options.add_argument( @@ -403,7 +403,7 @@ def main(): help='Give full details on the selected tests' ) action_options.add_argument( - '--describe-stored-session', action='store', metavar='SESSION_UUID', + '--describe-stored-session', action='store', metavar='UUID', help='Get detailed session information in JSON' ) action_options.add_argument( @@ -432,7 +432,7 @@ def main(): ) action_options.add_argument( '--list-stored-testcases', action='store', - metavar='SESSION_UUID|PERIOD', + metavar='^SESSION_UUID|PERIOD', help='List stored testcases by session or time period' ) action_options.add_argument( From 61c29fc113c0e5813118ef22e810af8efeed2017 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 9 Aug 2024 19:15:59 +0300 Subject: [PATCH 46/69] Adapt tutorial --- docs/config_reference.rst | 32 ++++++++ docs/tutorial.rst | 159 +++++++++++++++++++++++++++++++++++--- 2 files changed, 182 insertions(+), 9 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index c8715dbd36..9f245dd89e 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -60,6 +60,13 @@ It consists of the following properties, which we also call conventionally *conf A list of `general configuration objects <#general-configuration>`__. +.. py:data:: storage + + :required: No + + A list of :ref:`storage configuration objects ` + + .. versionadded:: 4.7 .. py:data:: autodetect_methods @@ -1602,6 +1609,31 @@ The options of an execution mode will be passed to ReFrame as if they were speci For a detailed description of this property, have a look at the :attr:`~environments.target_systems` definition for environments. +.. _storage-configuration: + +Result storage configuration +============================ + +.. versionadded:: 4.7 + +.. py:attribute:: storage.backend + + :required: No + :default: ``"sqlite"`` + + The backend to use for storing the test results. + + Currently, only Sqlite can be used as a storage backend. + + +.. py:attribute:: storage.sqlite_db_file + + :required: No + :default: ``"${HOME}/.reframe/reports/results.db"`` + + The Sqlite database file to use. + + General Configuration ===================== diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 78b1d3a672..5a27e9799d 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -168,24 +168,29 @@ This can be suppressed by increasing the level at which this information is logg Run reports and performance logging ----------------------------------- -Once a test session finishes, ReFrame generates a detailed JSON report under ``$HOME/.reframe/reports``. -Every time ReFrame is run a new report will be generated automatically. -The latest one is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given. +Once a test session finishes, ReFrame stores the detailed session information in database file located under ``$HOME/.reframe/reports``. +Past performance data can be retrieved from this database and compared with the current or another run. +We detail handling of the results database in section :ref:`inspecting-past-results`. + +By default, the session information is also saved in a JSON report file under ``$HOME/.reframe/reports``. +The latest report is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given. For performance tests, in particular, an additional CSV file is generated with all the relevant information. These files are located by default under ``perflogs///.log``. In our example, this translates to ``perflogs/generic/default/stream_test.log``. The information that is being logged is fully configurable and we will cover this in the :ref:`logging` section. -Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session. +Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and compare them (by default) with their last obtained performance. .. code-block:: console - [stream_test /2e15a047 @generic:default:builtin] - num_tasks: 1 - performance: - - copy_bw: 22704.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - triad_bw: 16040.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) + ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑ + │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ job_nodelist │ result │ + ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥ + │ stream_test │ generic:default+builtin │ copy_bw │ 40304.2 │ MB/s │ -0.08% │ myhost │ pass │ + ├─────────────┼─────────────────────────┼──────────┼─────────┼─────────┼─────────┼────────────────┼──────────┤ + │ stream_test │ generic:default+builtin │ triad_bw │ 30550.3 │ MB/s │ +0.04% │ myhost │ pass │ + ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙ Inspecting the test artifacts @@ -1969,3 +1974,139 @@ The format function takes the raw log record, the extras and the keys to ignore Since we can't know the exact log record attributes, we iterate over its :attr:`__dict__` items and format the record keys as we go. Also note that we ignore all private field of the record starting with ``_``. Rerunning the previous example with ``CUSTOM_JSON=1`` will generated the modified JSON record. + + +.. _inspecting-past-results: + +Inspecting past results +======================= + +.. versionadded:: 4.7 + +For every session that has run at least one test case, ReFrame stores all its details, including the test cases, in a database. +Essentially, the stored information is the same as the one found in the :ref:`report file `. + +To list all the stored sessions use the :option:`--list-stored-sessions` option: + +.. code-block:: bash + + reframe --list-stored-sessions + +This produces a table where the most important information about a session is listed: +its unique identifier, its start and end time and how many test cases have run: + +.. code-block:: console + + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑ + │ UUID │ Start time │ End time │ Num runs │ Num cases │ + ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥ + │ fddb6678-6de2-427c-96b5-d1c6b3215b0e │ 20240809T135331+0000 │ 20240809T135335+0000 │ 1 │ 1 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ d96a133c-e5a8-4ceb-88de-f8adfb393f28 │ 20240809T135342+0000 │ 20240809T135345+0000 │ 1 │ 1 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ c7508042-64be-406a-89f1-c5d31b90f838 │ 20240809T143710+0000 │ 20240809T143713+0000 │ 1 │ 1 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ 3bc5c067-42fa-4496-a5e4-50b92b3cc38e │ 20240809T144025+0000 │ 20240809T144026+0000 │ 1 │ 2 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ 53481b75-b98a-4668-b6ab-82b199cc2efe │ 20240809T144056+0000 │ 20240809T144057+0000 │ 1 │ 4 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ + ... + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ bed145ca-0013-4b68-bfd4-620054121f91 │ 20240809T144459+0000 │ 20240809T144500+0000 │ 1 │ 10 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ a72c7536-274a-4a21-92c3-4116f38febd0 │ 20240809T144500+0000 │ 20240809T144500+0000 │ 1 │ 1 │ + ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ + │ 8cb26ff4-7897-42fc-a993-fbdef57c8983 │ 20240809T144510+0000 │ 20240809T144511+0000 │ 1 │ 5 │ + ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━┙ + +You can use the :option:`--list-stored-testcases` to list the test cases of a specific session or those that have run within a certain period of time: + +.. code-block:: bash + + reframe --list-stored-testcases=^53481b75-b98a-4668-b6ab-82b199cc2efe + +.. code-block:: console + + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ + │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ + ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ + │ build_stream ~tutorialsys:default+gnu │ tutorialsys:default+gnu │ │ 20240809T145439+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:0 │ + ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │ │ 20240809T145439+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:1 │ + ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240809T144057+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:2 │ + ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240809T144057+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:3 │ + ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ + +.. note:: + + Note that you have to precede the session UUID with a ``^``. + +The test case UUID comprises the UUID of the session where this test case belongs to, its run index (which run inside the session) and its test case index inside the run. +A session may have multiple runs if it has retried some failed test cases (see :option:`--max-retries`) or if it has run its tests repeatedly (see :option:`--reruns` and :option:`--duration`). + +You can also list the test cases that have run in a certain period of time use the :ref:`time period ` of :option:`--list-stored-testcases`: + +.. code-block:: bash + + reframe --list-stored-testcases=20240809T144500+0000:now + +.. code-block:: console + + ┍━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ + │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ + ┝━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ + │ T2 │ generic:default+builtin │ myhost │ 20240809T144500+0000 │ fail │ bed145ca-0013-4b68-bfd4-620054121f91:0:7 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T3 │ generic:default+builtin │ myhost │ 20240809T144500+0000 │ pass │ bed145ca-0013-4b68-bfd4-620054121f91:0:9 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T6 │ generic:default+builtin │ myhost │ 20240809T144500+0000 │ pass │ a72c7536-274a-4a21-92c3-4116f38febd0:0:0 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T0 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:0 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T4 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:1 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T5 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:2 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T1 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:3 │ + ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ + │ T6 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:4 │ + ┕━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ + +To get all the details of a session or a set of test cases you can use the :option:`--describe-stored-session` and :option:`--describe-stored-testcases` options which will return a JSON record with all the details. + +Comparing performance of test cases +----------------------------------- + +ReFrame can be used to compare the performance of the same test cases run in different time periods using the :option:`--performance-compare` option. +The following will compare the performance of the test cases of the session ``a120b895-8fe9-4209-a742-997442e37c47`` with any other same test case that has run the last 24h: + +.. code-block:: bash + + reframe --performance-compare=^a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/ + +.. code-block:: console + + ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┑ + │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ + ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┥ + │ stream_test │ tutorialsys:default+gnu │ copy_bw │ 31274.5 │ MB/s │ -22.47% │ + ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤ + │ stream_test │ tutorialsys:default+gnu │ triad_bw │ 18993.4 │ MB/s │ -42.02% │ + ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤ + │ stream_test │ tutorialsys:default+clang │ copy_bw │ 38546.2 │ MB/s │ -9.24% │ + ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤ + │ stream_test │ tutorialsys:default+clang │ triad_bw │ 36866.3 │ MB/s │ -4.72% │ + ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┙ + +Similarly to the :option:`--performance-compare` option, the :option:`--performance-report` option can compare the performance of the current run with any arbitrary past session or past time period. + +Finally, you can delete complete a stored session using the :option:`--delete-stored-session` option: + +.. code-block:: bash + + reframe --delete-stored-session=a120b895-8fe9-4209-a742-997442e37c47 + +Deleting a session will also delete all its test cases from the database. From 8e2cb61c78a0b2953e915bf681dd35b679f78197 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 9 Aug 2024 19:18:18 +0300 Subject: [PATCH 47/69] Fix comment typos --- reframe/frontend/argparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py index 93a4070f36..ceec757b1a 100644 --- a/reframe/frontend/argparse.py +++ b/reframe/frontend/argparse.py @@ -46,7 +46,7 @@ class _Undefined: # We use a special value for denoting const values that are to be set from the # configuration default. This placeholder must be used as the `const` argument -# for options with `nargs='?'`. The underlying `ArugmentParser` will use the +# for options with `nargs='?'`. The underlying `ArgumentParser` will use the # `const` value as if it were supplied from the command-line thus fooling our # machinery of environment variables and configuration options overriding any # defaults. For this reason, we use a unique placeholder so that we can @@ -244,7 +244,7 @@ class ArgumentParser(_ArgumentHolder): '''Reframe's extended argument parser. This argument parser behaves almost identical to the original - `argparse.ArgumenParser`. In fact, it uses such a parser internally, + `argparse.ArgumentParser`. In fact, it uses such a parser internally, delegating all the calls to it. The key difference is how newly parsed options are combined with existing namespaces in `parse_args()`.''' From 71f8e5a2fdffa066caee7f7f9c52734b83c28c86 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 9 Aug 2024 19:57:22 +0300 Subject: [PATCH 48/69] Treat correctly invalid completion times in `--list-stored-testcases` --- .../tutorial/dockerfiles/singlenode.Dockerfile | 8 ++++++++ reframe/frontend/reporting/__init__.py | 17 ++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/examples/tutorial/dockerfiles/singlenode.Dockerfile b/examples/tutorial/dockerfiles/singlenode.Dockerfile index 647f26d231..86750b20d8 100644 --- a/examples/tutorial/dockerfiles/singlenode.Dockerfile +++ b/examples/tutorial/dockerfiles/singlenode.Dockerfile @@ -10,8 +10,16 @@ RUN apt-get -y update && \ ARG REFRAME_TAG=develop ARG REFRAME_REPO=reframe-hpc WORKDIR /usr/local/share + +# Clone reframe RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ cd reframe/ && ./bootstrap.sh + +# Comment the above line and uncomment the following two for development + +# COPY . /usr/local/share/reframe +# RUN cd reframe && ./bootstrap.sh + ENV PATH=/usr/local/share/reframe/bin:$PATH # Install stream diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 3cca83792b..b1ae4549d7 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -235,9 +235,6 @@ def __init__(self): now = time.time() self.update_timestamps(now, now) - def data(self): - return self.__report - @property def filename(self): return self.__filename @@ -655,14 +652,20 @@ def testcase_data(spec): data = [['Name', 'SysEnv', 'Nodelist', 'Completion Time', 'Result', 'UUID']] for tc in testcases: + ts_completed = tc['job_completion_time_unix'] + if not ts_completed: + completion_time = 'n/a' + else: + # Always format the completion time as users can set their own + # formatting in the log record + completion_time = time.strftime(_DATETIME_FMT, + time.localtime(ts_completed)) + data.append([ tc['name'], _format_sysenv(tc['system'], tc['partition'], tc['environ']), nodelist_abbrev(tc['job_nodelist']), - # Always format the completion time as users can set their own - # formatting in the log record - time.strftime(_DATETIME_FMT, - time.localtime(tc['job_completion_time_unix'])), + completion_time, tc['result'], tc['uuid'] ]) From 7fa684bfe1e363e0b1d75a20614fe46dedb1167f Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 9 Aug 2024 20:28:43 +0300 Subject: [PATCH 49/69] Do not prefix UUIDs with `^` --- docs/manpage.rst | 10 ++++---- docs/tutorial.rst | 7 ++---- reframe/frontend/cli.py | 4 ++-- reframe/frontend/reporting/__init__.py | 10 ++++---- reframe/frontend/reporting/utility.py | 16 +++++++++---- unittests/test_cli.py | 4 ++-- unittests/test_reporting.py | 32 ++++++++++++++++++++------ 7 files changed, 52 insertions(+), 31 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 053abae1e0..7ed2c45192 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -294,12 +294,11 @@ There are commands that act upon the selected tests and others that have a helpe .. versionadded:: 4.7 -.. option:: --describe-stored-testcases=^SESSION_UUID|TIME_PERIOD +.. option:: --describe-stored-testcases=SESSION_UUID|TIME_PERIOD Get detailed test case information of the session with the specified UUID or from the specified time period. If a session UUID is provided only information about the test cases of this session will be provided. - Note that the session UUID must be prefixed with ``^``. For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. @@ -350,12 +349,11 @@ There are commands that act upon the selected tests and others that have a helpe .. versionadded:: 4.7 -.. option:: --list-stored-testcases=^SESSION_UUID|TIME_PERIOD +.. option:: --list-stored-testcases=SESSION_UUID|TIME_PERIOD List all test cases from the session with the specified UUID or from the specified time period. If a session UUID is provided only the test cases of this session will be listed. - Note that the session UUID must be prefixed with ``^``. For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. @@ -1319,7 +1317,7 @@ The base test cases will be compared against those from the target period. The ```` subspec is ommitted from the ``CMPSPEC`` of the :option:`--performance-report` option as the base test cases are always the test cases from the current session. The test cases for comparison can either be retrieved from an existing past session or a past time period. -A past session is denoted with the ``^`` syntax and only the test cases of that particular session will be selected. +A past session is denoted with the ```` syntax and only the test cases of that particular session will be selected. To view the UUIDs of all stored sessions, use the :option:`--list-stored-sessions` option. To retrieve results from a time period, check the :ref:`time period syntax ` below. @@ -1374,7 +1372,7 @@ Here are some examples of performance comparison specs: .. code-block:: console - ^7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/ + 7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/ - Compare the best performance of the test cases run on two specific days, group by the node list and report also the test result: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 5a27e9799d..419a6e0dc9 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -2024,7 +2024,7 @@ You can use the :option:`--list-stored-testcases` to list the test cases of a sp .. code-block:: bash - reframe --list-stored-testcases=^53481b75-b98a-4668-b6ab-82b199cc2efe + reframe --list-stored-testcases=53481b75-b98a-4668-b6ab-82b199cc2efe .. code-block:: console @@ -2040,9 +2040,6 @@ You can use the :option:`--list-stored-testcases` to list the test cases of a sp │ stream_test │ tutorialsys:default+clang │ myhost │ 20240809T144057+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:3 │ ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ -.. note:: - - Note that you have to precede the session UUID with a ``^``. The test case UUID comprises the UUID of the session where this test case belongs to, its run index (which run inside the session) and its test case index inside the run. A session may have multiple runs if it has retried some failed test cases (see :option:`--max-retries`) or if it has run its tests repeatedly (see :option:`--reruns` and :option:`--duration`). @@ -2085,7 +2082,7 @@ The following will compare the performance of the test cases of the session ``a1 .. code-block:: bash - reframe --performance-compare=^a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/ + reframe --performance-compare=a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/ .. code-block:: console diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 6835ea797c..516b085d38 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -408,7 +408,7 @@ def main(): ) action_options.add_argument( '--describe-stored-testcases', action='store', - metavar='^SESSION_UUID|PERIOD', + metavar='SESSION_UUID|PERIOD', help='Get detailed test case information in JSON' ) action_options.add_argument( @@ -432,7 +432,7 @@ def main(): ) action_options.add_argument( '--list-stored-testcases', action='store', - metavar='^SESSION_UUID|PERIOD', + metavar='SESSION_UUID|PERIOD', help='List stored testcases by session or time period' ) action_options.add_argument( diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index b1ae4549d7..a4c860a28f 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -26,7 +26,7 @@ from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev from .storage import StorageBackend -from .utility import Aggregator, parse_cmp_spec, parse_time_period +from .utility import Aggregator, parse_cmp_spec, parse_time_period, is_uuid # The schema data version # Major version bumps are expected to break the validation of previous schemas @@ -642,8 +642,8 @@ def session_data(): def testcase_data(spec): storage = StorageBackend.default() - if spec.startswith('^'): - testcases = storage.fetch_testcases_from_session(spec[1:]) + if is_uuid(spec): + testcases = storage.fetch_testcases_from_session(spec) else: testcases = storage.fetch_testcases_time_period( *parse_time_period(spec) @@ -686,8 +686,8 @@ def session_info(uuid): def testcase_info(spec): '''Retrieve test case details as JSON''' testcases = [] - if spec.startswith('^'): - session_uuid, *tc_index = spec[1:].split(':') + if is_uuid(spec): + session_uuid, *tc_index = spec.split(':') session = session_info(session_uuid) if not tc_index: for run in session['runs']: diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py index 2da8030f53..fa98f39c20 100644 --- a/reframe/frontend/reporting/utility.py +++ b/reframe/frontend/reporting/utility.py @@ -130,11 +130,19 @@ def _do_parse(s): return ts.timestamp() +_UUID_PATTERN = re.compile(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}(:\d+)?(:\d+)?$') + + +def is_uuid(s): + '''Return true if `s` is a valid session, run or test case UUID''' + return _UUID_PATTERN.match(s) is not None + + def parse_time_period(s): - if s.startswith('^'): + if is_uuid(s): # Retrieve the period of a full session try: - session_uuid = s[1:] + session_uuid = s except IndexError: raise ValueError(f'invalid session uuid: {s}') from None else: @@ -181,8 +189,8 @@ def _parse_period_spec(s): if s is None: return None, None - if s.startswith('^'): - return s[1:], None + if is_uuid(s): + return s, None return None, parse_time_period(s) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index b63590afbc..21f71d355f 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1287,9 +1287,9 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0): session_json = json.loads(stdout) # List test cases by session - assert_no_crash(*run_reframe2(action=f'--list-stored-testcases=^{uuid}')) + assert_no_crash(*run_reframe2(action=f'--list-stored-testcases={uuid}')) assert_no_crash( - *run_reframe2(action=f'--describe-stored-testcases=^{uuid}') + *run_reframe2(action=f'--describe-stored-testcases={uuid}') ) # List test cases by time period diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py index a25d6410b4..cff94f3cf8 100644 --- a/unittests/test_reporting.py +++ b/unittests/test_reporting.py @@ -277,10 +277,28 @@ def test_parse_cmp_spec_extra_cols(extra_cols): assert match.period_base is None -@pytest.fixture(params=['^uuid/now-1d:now/min:/', - 'now-1d:now/^uuid/min:/', - '^uuid0/^uuid1/min:/', - 'now-1m:now/now-1d:now/min:/']) +def test_is_uuid(): + # Test a standard UUID + assert report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2') + + # Test a run UUID + assert report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2:0') + + # Test a test case UUID + assert report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2:0:1') + + # Test invalid UUIDs + assert not report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c') + assert not report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2:') + assert not report_util.is_uuid('foo') + + +@pytest.fixture(params=[ + '7daf4a71-997b-4417-9bda-225c9cab96c2/now-1d:now/min:/', + 'now-1d:now/7daf4a71-997b-4417-9bda-225c9cab96c2/min:/', + '7daf4a71-997b-4417-9bda-225c9cab96c2/7daf4a71-997b-4417-9bda-225c9cab96c2/min:/', # noqa: E501 + 'now-1m:now/now-1d:now/min:/'] +) def uuid_spec(request): return request.param @@ -291,10 +309,10 @@ def _uuids(s): base, target = None, None if len(parts) == 3: base = None - target = parts[0][1:] if parts[0].startswith('^') else None + target = parts[0] if report_util.is_uuid(parts[0]) else None else: - base = parts[0][1:] if parts[0].startswith('^') else None - target = parts[1][1:] if parts[1].startswith('^') else None + base = parts[0] if report_util.is_uuid(parts[0]) else None + target = parts[1] if report_util.is_uuid(parts[1]) else None return base, target From d2e0844b5ec807150c2cd45ece68051d3fd46836 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 12 Aug 2024 12:30:15 +0300 Subject: [PATCH 50/69] Accept a time period argument in `--list-stored-sessions` --- docs/manpage.rst | 9 +++++++-- docs/tutorial.rst | 2 +- reframe/frontend/cli.py | 10 +++++++--- reframe/frontend/reporting/__init__.py | 6 ++++-- reframe/frontend/reporting/storage.py | 16 +++++++++++++--- unittests/test_reporting.py | 13 ++++++++++--- 6 files changed, 42 insertions(+), 14 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 7ed2c45192..d62e427136 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -343,9 +343,14 @@ There are commands that act upon the selected tests and others that have a helpe The variable names to which fixtures are bound are also listed. See :ref:`test_naming_scheme` for more information. -.. option:: --list-stored-sessions +.. _--list-stored-sessions: - List all sessions stored in the results database. +.. option:: --list-stored-sessions[=TIME_PERIOD] + + List sessions stored in the results database. + + If ``TIME_PERIOD`` is not specified or if ``all`` is passed, all stored sessions will be listed. + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. .. versionadded:: 4.7 diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 419a6e0dc9..a6b46c1deb 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1986,7 +1986,7 @@ Inspecting past results For every session that has run at least one test case, ReFrame stores all its details, including the test cases, in a database. Essentially, the stored information is the same as the one found in the :ref:`report file `. -To list all the stored sessions use the :option:`--list-stored-sessions` option: +To list the stored sessions use the :option:`--list-stored-sessions` option: .. code-block:: bash diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 516b085d38..8965467546 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -427,8 +427,8 @@ def main(): 'providing more details') ) action_options.add_argument( - '--list-stored-sessions', action='store_true', - help='List stored sessions' + '--list-stored-sessions', nargs='?', action='store', const='all', + metavar='PERIOD', help='List stored sessions' ) action_options.add_argument( '--list-stored-testcases', action='store', @@ -939,7 +939,11 @@ def restrict_logging(): if options.list_stored_sessions: with exit_gracefully_on_error('failed to retrieve session data', printer): - printer.table(reporting.session_data()) + time_period = options.list_stored_sessions + if time_period == 'all': + time_period = None + + printer.table(reporting.session_data(time_period)) sys.exit(0) if options.list_stored_testcases: diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index a4c860a28f..07a98e885e 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -623,11 +623,13 @@ def performance_compare(cmp, report=None): match.extra_cols) -def session_data(): +def session_data(time_period): '''Retrieve all sessions''' data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']] - for sess_data in StorageBackend.default().fetch_all_sessions(): + for sess_data in StorageBackend.default().fetch_sessions_time_period( + *parse_time_period(time_period) if time_period else (None, None) + ): session_info = sess_data['session_info'] data.append( [session_info['uuid'], diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 940ed24840..434887ce7c 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -207,10 +207,20 @@ def fetch_testcases_from_session(self, session_uuid): session_info = jsonext.loads(results[0][0]) return [tc for run in session_info['runs'] for tc in run['testcases']] - def fetch_all_sessions(self): + def fetch_sessions_time_period(self, ts_start=None, ts_end=None): with sqlite3.connect(self._db_file()) as conn: - query = ('SELECT json_blob from sessions ' - 'ORDER BY session_start_unix') + query = 'SELECT json_blob from sessions' + if ts_start or ts_end: + query += ' WHERE (' + if ts_start: + query += f'session_start_unix >= {ts_start}' + + if ts_end: + query += f' AND session_start_unix <= {ts_end}' + + query += ')' + + query += ' ORDER BY session_start_unix' getlogger().debug(query) results = conn.execute(query).fetchall() diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py index cff94f3cf8..509e8d4d7a 100644 --- a/unittests/test_reporting.py +++ b/unittests/test_reporting.py @@ -401,8 +401,15 @@ def _count_failed(testcases): backend = report_storage.StorageBackend.default() - # Test `fetch_all_sessions` - stored_sessions = backend.fetch_all_sessions() + # Test `fetch_sessions_time_period` + stored_sessions = backend.fetch_sessions_time_period() + assert len(stored_sessions) == 2 + for i, sess in enumerate(stored_sessions): + assert sess['session_info']['uuid'] == uuids[i] + + # Test the time period version + now = time.time() + stored_sessions = backend.fetch_sessions_time_period(now - 60, now) assert len(stored_sessions) == 2 for i, sess in enumerate(stored_sessions): assert sess['session_info']['uuid'] == uuids[i] @@ -450,7 +457,7 @@ def _count_failed(testcases): # Test session removal backend.remove_session(uuids[-1]) - assert len(backend.fetch_all_sessions()) == 1 + assert len(backend.fetch_sessions_time_period()) == 1 testcases = backend.fetch_testcases_time_period(timestamps[0][0], timestamps[1][1]) From d08528f3028ee31a49faedd965c2c7ef899273ee Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 13 Aug 2024 20:19:12 +0300 Subject: [PATCH 51/69] Fix broken `nodelist` check for PBS-based backends --- reframe/core/schedulers/pbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index a15bea5df9..86dbb6063d 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -185,7 +185,7 @@ def finished(self, job): return job.completed def _update_nodelist(self, job, nodespec): - if job.nodelist is not None: + if job.nodelist: return job._nodelist = [x.split('/')[0] for x in nodespec.split('+')] From ddb8ad728d313cfc3f3234812f0d43b6fa17744d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 13 Aug 2024 21:35:32 +0300 Subject: [PATCH 52/69] Allow setting the database file from an environment variable --- docs/manpage.rst | 15 +++++++++++++++ reframe/frontend/cli.py | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/docs/manpage.rst b/docs/manpage.rst index d62e427136..cad06c510e 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1988,6 +1988,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_SQLITE_DB_FILE + + The SQlite database file for storing test results. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr:`~config.storage.sqlite_db_file` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_SYSLOG_ADDRESS The address of the Syslog server to send performance logs. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 8965467546..41b8a9c249 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -751,6 +751,12 @@ def main(): action='store_true', help='Resolve module conflicts automatically' ) + argparser.add_argument( + dest='sqlite_db_file', + envvar='RFM_SQLITE_DB_FILE', + configvar='storage/sqlite_db_file', + help='DB file where the results database resides' + ) argparser.add_argument( dest='syslog_address', envvar='RFM_SYSLOG_ADDRESS', From b1206ad6ae5b7234d017c4a5d90d6815c7ed005b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 14 Aug 2024 11:43:25 +0300 Subject: [PATCH 53/69] Add CLI and config option to control the table format --- docs/config_reference.rst | 14 +++++++++++++ docs/manpage.rst | 27 ++++++++++++++++++++++++++ reframe/frontend/cli.py | 5 +++++ reframe/frontend/printer.py | 18 ++++++++++++++++- reframe/frontend/reporting/__init__.py | 9 ++++++++- reframe/schemas/config.json | 2 ++ unittests/test_cli.py | 15 ++++++++++---- 7 files changed, 84 insertions(+), 6 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 9f245dd89e..beeb9b457b 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1891,6 +1891,20 @@ General Configuration For a detailed description of this property, have a look at the :attr:`~environments.target_systems` definition for environments. +.. py:attribute:: general.table_format + + :required: No + :default: ``"pretty"`` + + Set the formatting of tabular output. + + The acceptable values are the following: + + - ``csv``: Generate CSV output + - ``plain``: Generate a plain table without any lines + - ``pretty``: (default) Generate a pretty table + + .. py:attribute:: general.timestamp_dirs :required: No diff --git a/docs/manpage.rst b/docs/manpage.rst index cad06c510e..0ab2707771 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1129,6 +1129,18 @@ Miscellaneous options This option can also be set using the :envvar:`RFM_SYSTEM` environment variable. +.. option:: --table-format=csv|plain|pretty + + Set the formatting of tabular output printed by options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions. + + The acceptable values are the following: + + - ``csv``: Generate CSV output + - ``plain``: Generate a plain table without any lines + - ``pretty``: (default) Generate a pretty table + + .. versionadded:: 4.7 + .. option:: --upgrade-config-file=OLD[:NEW] Convert the old-style configuration file ``OLD``, place it into the new file ``NEW`` and exit. @@ -2033,6 +2045,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_TABLE_FORMAT + + Set the format of the tables printed by various options accessing the results storage. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--table-format` + Associated configuration parameter :attr:`~config.general.table_format` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_TIMESTAMP_DIRS Append a timestamp to the output and stage directory prefixes. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 41b8a9c249..88f2df5567 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -625,6 +625,11 @@ def main(): '--system', action='store', help='Load configuration for SYSTEM', envvar='RFM_SYSTEM' ) + misc_options.add_argument( + '--table-format', choices=['csv', 'plain', 'pretty'], + help='Table formatting', + envvar='RFM_TABLE_FORMAT', configvar='general/table_format' + ) misc_options.add_argument( '-v', '--verbose', action='count', help='Increase verbosity level of output', diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index 0767ac4dfe..6613107df4 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -255,9 +255,25 @@ def performance_report(self, data, **kwargs): self.table(data, **kwargs) self.info('') + def _table_as_csv(self, data): + for line in data: + self.info(','.join(str(x) for x in line)) + def table(self, data, **kwargs): '''Print tabular data''' + table_format = rt.runtime().get_option('general/0/table_format') + if table_format == 'csv': + return self._table_as_csv(data) + + # Map our options to tabulate + if table_format == 'plain': + tablefmt = 'plain' + elif table_format == 'pretty': + tablefmt = 'mixed_grid' + else: + raise ValueError(f'invalid table format: {table_format}') + kwargs.setdefault('headers', 'firstrow') - kwargs.setdefault('tablefmt', 'mixed_grid') + kwargs.setdefault('tablefmt', tablefmt) self.info(tabulate(data, **kwargs)) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 07a98e885e..79831141ae 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -23,6 +23,7 @@ import reframe.utility.osext as osext from reframe.core.exceptions import ReframeError, what, is_severe, reraise_as from reframe.core.logging import getlogger, _format_time_rfc3339 +from reframe.core.runtime import runtime from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev from .storage import StorageBackend @@ -527,7 +528,13 @@ def _group_testcases(testcases, group_by, extra_cols): def _aggregate_perf(grouped_testcases, aggr_fn, cols): - other_aggr = Aggregator.create('join_uniq', '\n') + if runtime().get_option('general/0/table_format') == 'csv': + # Use a csv friendly delimiter + delim = '|' + else: + delim = '\n' + + other_aggr = Aggregator.create('join_uniq', delim) aggr_data = {} for key, seq in grouped_testcases.items(): aggr_data.setdefault(key, {}) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 900ab0de14..06765f604a 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -525,6 +525,7 @@ "resolve_module_conflicts": {"type": "boolean"}, "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, + "table_format": {"enum": ["csv", "plain", "pretty"]}, "timestamp_dirs": {"type": "string"}, "trap_job_errors": {"type": "boolean"}, "unload_modules": {"$ref": "#/defs/modules_list"}, @@ -587,6 +588,7 @@ "general/report_junit": null, "general/resolve_module_conflicts": true, "general/save_log_files": false, + "general/table_format": "pretty", "general/target_systems": ["*"], "general/timestamp_dirs": "%Y%m%dT%H%M%S%z", "general/trap_job_errors": false, diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 21f71d355f..c5f517819e 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -1257,7 +1257,12 @@ def test_testlib_inherit_fixture_in_different_files(run_reframe): assert 'FAILED' not in stdout -def test_storage_options(run_reframe, tmp_path): +@pytest.fixture(params=['csv', 'plain', 'pretty']) +def table_format(request): + return request.param + + +def test_storage_options(run_reframe, tmp_path, table_format): def assert_no_crash(returncode, stdout, stderr, exitcode=0): assert returncode == exitcode assert 'Traceback' not in stdout @@ -1266,7 +1271,8 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0): run_reframe2 = functools.partial( run_reframe, - checkpath=['unittests/resources/checks/frontend_checks.py'] + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=[f'--table-format={table_format}'] ) # Run first a normal run with a performance test to initialize the DB @@ -1313,7 +1319,7 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0): assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}')) -def test_performance_compare(run_reframe): +def test_performance_compare(run_reframe, table_format): def assert_no_crash(returncode, stdout, stderr, exitcode=0): assert returncode == exitcode assert 'Traceback' not in stdout @@ -1322,7 +1328,8 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0): run_reframe2 = functools.partial( run_reframe, - checkpath=['unittests/resources/checks/frontend_checks.py'] + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=[f'--table-format={table_format}'] ) run_reframe2(action='run') From db5b4bec772bf2f3d4143822423b0ef06789bdcf Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 14 Aug 2024 15:46:40 +0300 Subject: [PATCH 54/69] More compact pretty table output --- reframe/frontend/printer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index 6613107df4..a8a88b91cd 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -270,7 +270,7 @@ def table(self, data, **kwargs): if table_format == 'plain': tablefmt = 'plain' elif table_format == 'pretty': - tablefmt = 'mixed_grid' + tablefmt = 'mixed_outline' else: raise ValueError(f'invalid table format: {table_format}') From 91c87c2be27de8141d33a599895cb28a096c78f3 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 14 Aug 2024 23:36:03 +0300 Subject: [PATCH 55/69] Add profiler calls to SQlite storage backend --- reframe/frontend/cli.py | 1 + reframe/frontend/reporting/__init__.py | 11 ++++++++++- reframe/frontend/reporting/storage.py | 14 +++++++++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 88f2df5567..72b8531934 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -218,6 +218,7 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): + logging.getprofiler().print_report(self.__logger.debug) if exc_type is SystemExit: # Allow users to exit inside the context manager return diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 79831141ae..eb936ece38 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -22,7 +22,7 @@ import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext from reframe.core.exceptions import ReframeError, what, is_severe, reraise_as -from reframe.core.logging import getlogger, _format_time_rfc3339 +from reframe.core.logging import getlogger, _format_time_rfc3339, time_function from reframe.core.runtime import runtime from reframe.core.warnings import suppress_deprecations from reframe.utility import nodelist_abbrev @@ -504,6 +504,7 @@ def _group_key(groups, testcase): return tuple(key) +@time_function def _group_testcases(testcases, group_by, extra_cols): grouped = {} for tc in testcases: @@ -527,6 +528,7 @@ def _group_testcases(testcases, group_by, extra_cols): return grouped +@time_function def _aggregate_perf(grouped_testcases, aggr_fn, cols): if runtime().get_option('general/0/table_format') == 'csv': # Use a csv friendly delimiter @@ -549,6 +551,7 @@ def _aggregate_perf(grouped_testcases, aggr_fn, cols): return aggr_data +@time_function def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, extra_group_by=None, extra_cols=None): extra_group_by = extra_group_by or [] @@ -587,6 +590,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, return data +@time_function def performance_compare(cmp, report=None): with reraise_as(ReframeError, (ValueError,), 'could not parse comparison spec'): @@ -630,6 +634,7 @@ def performance_compare(cmp, report=None): match.extra_cols) +@time_function def session_data(time_period): '''Retrieve all sessions''' @@ -649,6 +654,7 @@ def session_data(time_period): return data +@time_function def testcase_data(spec): storage = StorageBackend.default() if is_uuid(spec): @@ -682,6 +688,7 @@ def testcase_data(spec): return data +@time_function def session_info(uuid): '''Retrieve session details as JSON''' @@ -692,6 +699,7 @@ def session_info(uuid): return session +@time_function def testcase_info(spec): '''Retrieve test case details as JSON''' testcases = [] @@ -714,5 +722,6 @@ def testcase_info(spec): return testcases +@time_function def delete_session(session_uuid): StorageBackend.default().remove_session(session_uuid) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 434887ce7c..b32833212e 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -11,7 +11,7 @@ import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext from reframe.core.exceptions import ReframeError -from reframe.core.logging import getlogger +from reframe.core.logging import getlogger, time_function, getprofiler from reframe.core.runtime import runtime @@ -153,7 +153,9 @@ def store(self, report, report_file=None): with FileLock(os.path.join(prefix, '.db.lock')): return self._db_store_report(conn, report, report_file) + @time_function def _fetch_testcases_raw(self, condition): + getprofiler().enter_region('sqlite query') with sqlite3.connect(self._db_file()) as conn: query = ('SELECT session_uuid, testcases.uuid as uuid, json_blob ' 'FROM testcases ' @@ -162,19 +164,24 @@ def _fetch_testcases_raw(self, condition): getlogger().debug(query) results = conn.execute(query).fetchall() + getprofiler().exit_region() + # Retrieve files testcases = [] sessions = {} for session_uuid, uuid, json_blob in results: run_index, test_index = [int(x) for x in uuid.split(':')[1:]] + getprofiler().enter_region('json decode') report = jsonext.loads(sessions.setdefault(session_uuid, json_blob)) + getprofiler().exit_region() testcases.append( report['runs'][run_index]['testcases'][test_index], ) return testcases + @time_function def fetch_session_time_period(self, session_uuid): with sqlite3.connect(self._db_file()) as conn: query = ('SELECT session_start_unix, session_end_unix ' @@ -187,6 +194,7 @@ def fetch_session_time_period(self, session_uuid): return None, None + @time_function def fetch_testcases_time_period(self, ts_start, ts_end): return self._fetch_testcases_raw( f'(job_completion_time_unix >= {ts_start} AND ' @@ -194,6 +202,7 @@ def fetch_testcases_time_period(self, ts_start, ts_end): 'ORDER BY job_completion_time_unix' ) + @time_function def fetch_testcases_from_session(self, session_uuid): with sqlite3.connect(self._db_file()) as conn: query = ('SELECT json_blob from sessions ' @@ -207,6 +216,7 @@ def fetch_testcases_from_session(self, session_uuid): session_info = jsonext.loads(results[0][0]) return [tc for run in session_info['runs'] for tc in run['testcases']] + @time_function def fetch_sessions_time_period(self, ts_start=None, ts_end=None): with sqlite3.connect(self._db_file()) as conn: query = 'SELECT json_blob from sessions' @@ -229,6 +239,7 @@ def fetch_sessions_time_period(self, ts_start=None, ts_end=None): return [jsonext.loads(json_blob) for json_blob, *_ in results] + @time_function def fetch_session_json(self, uuid): with sqlite3.connect(self._db_file()) as conn: query = f'SELECT json_blob FROM sessions WHERE uuid == "{uuid}"' @@ -263,6 +274,7 @@ def _do_remove2(self, uuid): if not deleted: raise ReframeError(f'no such session: {uuid}') + @time_function def remove_session(self, uuid): if sqlite3.sqlite_version_info >= (3, 35, 0): self._do_remove2(uuid) From 34969e50b938c5e397d6cf4fd5cf45c0456744d8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 15 Aug 2024 00:01:20 +0300 Subject: [PATCH 56/69] Optimize decoding of JSON blobs --- reframe/frontend/reporting/storage.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index b32833212e..9395dd8fe2 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -166,15 +166,26 @@ def _fetch_testcases_raw(self, condition): getprofiler().exit_region() - # Retrieve files - testcases = [] + # Retrieve session info sessions = {} + for session_uuid, uuid, json_blob in results: + sessions.setdefault(session_uuid, json_blob) + + # Join all sessions and decode them at once + reports_blob = '[' + ','.join(sessions.values()) + ']' + getprofiler().enter_region('json decode') + reports = jsonext.loads(reports_blob) + getprofiler().exit_region() + + # Reindex sessions with their decoded data + for rpt in reports: + sessions[rpt['session_info']['uuid']] = rpt + + # Extract the test case data + testcases = [] for session_uuid, uuid, json_blob in results: run_index, test_index = [int(x) for x in uuid.split(':')[1:]] - getprofiler().enter_region('json decode') - report = jsonext.loads(sessions.setdefault(session_uuid, - json_blob)) - getprofiler().exit_region() + report = sessions[session_uuid] testcases.append( report['runs'][run_index]['testcases'][test_index], ) From 5aa3da5a1cb0f854baddc19c1ffd816e3685f04c Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 22 Aug 2024 17:44:38 +0300 Subject: [PATCH 57/69] Create DB index for testcases table --- reframe/frontend/reporting/storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index 9395dd8fe2..c3627dce33 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -86,6 +86,8 @@ def _db_create(self): 'uuid TEXT, ' 'FOREIGN KEY(session_uuid) ' 'REFERENCES sessions(uuid) ON DELETE CASCADE)') + conn.execute('CREATE INDEX IF NOT EXISTS index_testcases_time ' + 'on testcases(job_completion_time_unix)') conn.execute('CREATE TABLE IF NOT EXISTS metadata(' 'schema_version TEXT)') From dd37f86c7e15779fb8f3fadade5ccd47aef380e8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 22 Aug 2024 20:36:31 +0300 Subject: [PATCH 58/69] Support filtering by testcase query options --- reframe/frontend/cli.py | 10 +++++++--- reframe/frontend/reporting/__init__.py | 20 +++++++++---------- reframe/frontend/reporting/storage.py | 27 ++++++++++++++++++++------ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 72b8531934..8cc57a0d63 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -959,10 +959,11 @@ def restrict_logging(): sys.exit(0) if options.list_stored_testcases: + namepatt = '|'.join(options.names) with exit_gracefully_on_error('failed to retrieve test case data', printer): printer.table(reporting.testcase_data( - options.list_stored_testcases + options.list_stored_testcases, namepatt )) sys.exit(0) @@ -979,10 +980,11 @@ def restrict_logging(): if options.describe_stored_testcases: # Restore logging level printer.setLevel(logging.INFO) + namepatt = '|'.join(options.names) with exit_gracefully_on_error('failed to retrieve test case data', printer): printer.info(jsonext.dumps(reporting.testcase_info( - options.describe_stored_testcases + options.describe_stored_testcases, namepatt ), indent=2)) sys.exit(0) @@ -1405,9 +1407,11 @@ def _sort_testcases(testcases): sys.exit(0) if options.performance_compare: + namepatt = '|'.join(options.names) try: printer.table( - reporting.performance_compare(options.performance_compare) + reporting.performance_compare(options.performance_compare, + namepatt=namepatt) ) except errors.ReframeError as err: printer.error(f'failed to generate performance report: {err}') diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index eb936ece38..9500c4ec13 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -591,7 +591,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn, @time_function -def performance_compare(cmp, report=None): +def performance_compare(cmp, report=None, namepatt=None): with reraise_as(ReframeError, (ValueError,), 'could not parse comparison spec'): match = parse_cmp_spec(cmp) @@ -613,20 +613,20 @@ def performance_compare(cmp, report=None): tcs_base = [] elif match.period_base is not None: tcs_base = StorageBackend.default().fetch_testcases_time_period( - *match.period_base + *match.period_base, namepatt ) else: tcs_base = StorageBackend.default().fetch_testcases_from_session( - match.session_base + match.session_base, namepatt ) if match.period_target: tcs_target = StorageBackend.default().fetch_testcases_time_period( - *match.period_target + *match.period_target, namepatt ) else: tcs_target = StorageBackend.default().fetch_testcases_from_session( - match.session_target + match.session_target, namepatt ) return compare_testcase_data(tcs_base, tcs_target, match.aggregator, @@ -655,13 +655,13 @@ def session_data(time_period): @time_function -def testcase_data(spec): +def testcase_data(spec, namepatt=None): storage = StorageBackend.default() if is_uuid(spec): - testcases = storage.fetch_testcases_from_session(spec) + testcases = storage.fetch_testcases_from_session(spec, namepatt) else: testcases = storage.fetch_testcases_time_period( - *parse_time_period(spec) + *parse_time_period(spec), namepatt ) data = [['Name', 'SysEnv', @@ -700,7 +700,7 @@ def session_info(uuid): @time_function -def testcase_info(spec): +def testcase_info(spec, namepatt=None): '''Retrieve test case details as JSON''' testcases = [] if is_uuid(spec): @@ -716,7 +716,7 @@ def testcase_info(spec): ) else: testcases = StorageBackend.default().fetch_testcases_time_period( - *parse_time_period(spec) + *parse_time_period(spec), namepatt ) return testcases diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py index c3627dce33..87fed08720 100644 --- a/reframe/frontend/reporting/storage.py +++ b/reframe/frontend/reporting/storage.py @@ -5,6 +5,7 @@ import abc import os +import re import sqlite3 from filelock import FileLock @@ -64,6 +65,13 @@ def _db_file(self): self._db_schema_check() return self.__db_file + def _db_matches(self, patt, item): + if patt is None: + return True + + regex = re.compile(patt) + return regex.match(item) is not None + def _db_create(self): clsname = type(self).__name__ getlogger().debug( @@ -164,6 +172,9 @@ def _fetch_testcases_raw(self, condition): 'JOIN sessions ON session_uuid == sessions.uuid ' f'WHERE {condition}') getlogger().debug(query) + + # Create SQLite function for filtering using name patterns + conn.create_function('REGEXP', 2, self._db_matches) results = conn.execute(query).fetchall() getprofiler().exit_region() @@ -208,15 +219,18 @@ def fetch_session_time_period(self, session_uuid): return None, None @time_function - def fetch_testcases_time_period(self, ts_start, ts_end): + def fetch_testcases_time_period(self, ts_start, ts_end, name_pattern=None): + expr = (f'job_completion_time_unix >= {ts_start} AND ' + f'job_completion_time_unix <= {ts_end}') + if name_pattern: + expr += f' AND name REGEXP "{name_pattern}"' + return self._fetch_testcases_raw( - f'(job_completion_time_unix >= {ts_start} AND ' - f'job_completion_time_unix <= {ts_end}) ' - 'ORDER BY job_completion_time_unix' + f'({expr}) ORDER BY job_completion_time_unix' ) @time_function - def fetch_testcases_from_session(self, session_uuid): + def fetch_testcases_from_session(self, session_uuid, name_pattern=None): with sqlite3.connect(self._db_file()) as conn: query = ('SELECT json_blob from sessions ' f'WHERE uuid == "{session_uuid}"') @@ -227,7 +241,8 @@ def fetch_testcases_from_session(self, session_uuid): return [] session_info = jsonext.loads(results[0][0]) - return [tc for run in session_info['runs'] for tc in run['testcases']] + return [tc for run in session_info['runs'] for tc in run['testcases'] + if self._db_matches(name_pattern, tc['name'])] @time_function def fetch_sessions_time_period(self, ts_start=None, ts_end=None): From e0f432008f296d1b6635a8c92508382a0f77e39a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 12:12:29 +0300 Subject: [PATCH 59/69] Add unit tests for test filtering in testcase query options --- unittests/test_reporting.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py index 509e8d4d7a..3b602b0166 100644 --- a/unittests/test_reporting.py +++ b/unittests/test_reporting.py @@ -442,6 +442,13 @@ def _count_failed(testcases): assert len(testcases) == 12 assert _count_failed(testcases) == 6 + # Test name filtering + testcases = backend.fetch_testcases_time_period(timestamps[0][0], + timestamps[1][1], + '^HelloTest') + assert len(testcases) == 2 + assert _count_failed(testcases) == 0 + # Test the inverted period assert backend.fetch_testcases_time_period(timestamps[1][1], timestamps[0][0]) == [] @@ -452,6 +459,11 @@ def _count_failed(testcases): assert len(testcases) == 9 assert _count_failed(testcases) == 5 + # Test name filtering + testcases = backend.fetch_testcases_from_session(uuid, '^HelloTest') + assert len(testcases) == 1 + assert _count_failed(testcases) == 0 + # Test an invalid uuid assert backend.fetch_testcases_from_session(0) == [] From 159894dc377d68bf0d576aba81bc1e4d641663cf Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 14:05:18 +0300 Subject: [PATCH 60/69] Do not print "ReFrame setup" header with `--performance-compare` option --- reframe/frontend/cli.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 8cc57a0d63..ec66aacf93 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -995,6 +995,19 @@ def restrict_logging(): printer.info(f'Session {session_uuid} deleted successfully.') sys.exit(0) + if options.performance_compare: + namepatt = '|'.join(options.names) + try: + printer.table( + reporting.performance_compare(options.performance_compare, + namepatt=namepatt) + ) + except errors.ReframeError as err: + printer.error(f'failed to generate performance report: {err}') + sys.exit(1) + else: + sys.exit(0) + # Show configuration after everything is set up if options.show_config: # Restore logging level @@ -1406,19 +1419,6 @@ def _sort_testcases(testcases): ) sys.exit(0) - if options.performance_compare: - namepatt = '|'.join(options.names) - try: - printer.table( - reporting.performance_compare(options.performance_compare, - namepatt=namepatt) - ) - except errors.ReframeError as err: - printer.error(f'failed to generate performance report: {err}') - sys.exit(1) - else: - sys.exit(0) - # Manipulate ReFrame's environment if site_config.get('general/0/purge_environment'): rt.modules_system.unload_all() From c4f6fc1baa448376ed3f7095d9effe627184b8ac Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 13:35:16 +0300 Subject: [PATCH 61/69] Do not store session if dry run --- docs/tutorial.rst | 5 ++--- reframe/frontend/cli.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index a6b46c1deb..830da136d1 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -187,9 +187,8 @@ Finally, you can use also the :option:`--performance-report` option, which will ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑ │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ job_nodelist │ result │ ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥ - │ stream_test │ generic:default+builtin │ copy_bw │ 40304.2 │ MB/s │ -0.08% │ myhost │ pass │ - ├─────────────┼─────────────────────────┼──────────┼─────────┼─────────┼─────────┼────────────────┼──────────┤ - │ stream_test │ generic:default+builtin │ triad_bw │ 30550.3 │ MB/s │ +0.04% │ myhost │ pass │ + │ stream_test │ generic:default+builtin │ copy_bw │ 40310.2 │ MB/s │ +0.05% │ myhost │ pass │ + │ stream_test │ generic:default+builtin │ triad_bw │ 30533.2 │ MB/s │ -0.06% │ myhost │ pass │ ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙ diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index ec66aacf93..6fa3c51b34 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1615,7 +1615,7 @@ def module_unuse(*paths): ) # Store the generated report for analytics - if not report.is_empty(): + if not report.is_empty() and not options.dry_run: try: sess_uuid = report.store() except Exception as e: From f2d3223abf51253fddaa9be1441b2b46f9d7d518 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 13:34:04 +0300 Subject: [PATCH 62/69] Update docs and tutorial --- docs/manpage.rst | 9 +- docs/tutorial.rst | 120 +++++++++--------- .../dockerfiles/singlenode.Dockerfile | 8 +- 3 files changed, 70 insertions(+), 67 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 0ab2707771..50387abdbb 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -299,8 +299,8 @@ There are commands that act upon the selected tests and others that have a helpe Get detailed test case information of the session with the specified UUID or from the specified time period. If a session UUID is provided only information about the test cases of this session will be provided. - - For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. + This option can be combined with :option:`--name` to restrict the listing to specific tests. + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section. .. versionadded:: 4.7 @@ -359,8 +359,8 @@ There are commands that act upon the selected tests and others that have a helpe List all test cases from the session with the specified UUID or from the specified time period. If a session UUID is provided only the test cases of this session will be listed. - - For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. + This option can be combined with :option:`--name` to restrict the listing to specific tests. + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section. .. versionadded:: 4.7 @@ -394,6 +394,7 @@ There are commands that act upon the selected tests and others that have a helpe Compare the performance of test cases that have run in the past. + This option can be combined with :option:`--name` to restrict the comparison to specific tests. Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``. .. versionadded:: 4.7 diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 830da136d1..888671c660 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -187,8 +187,8 @@ Finally, you can use also the :option:`--performance-report` option, which will ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑ │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ job_nodelist │ result │ ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥ - │ stream_test │ generic:default+builtin │ copy_bw │ 40310.2 │ MB/s │ +0.05% │ myhost │ pass │ - │ stream_test │ generic:default+builtin │ triad_bw │ 30533.2 │ MB/s │ -0.06% │ myhost │ pass │ + │ stream_test │ generic:default+builtin │ copy_bw │ 40292.1 │ MB/s │ -0.04% │ myhost │ pass │ + │ stream_test │ generic:default+builtin │ triad_bw │ 30564.7 │ MB/s │ +0.12% │ myhost │ pass │ ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙ @@ -1988,6 +1988,7 @@ Essentially, the stored information is the same as the one found in the :ref:`re To list the stored sessions use the :option:`--list-stored-sessions` option: .. code-block:: bash + :caption: Run in the single-node container. reframe --list-stored-sessions @@ -1996,47 +1997,42 @@ its unique identifier, its start and end time and how many test cases have run: .. code-block:: console - ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑ + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑ │ UUID │ Start time │ End time │ Num runs │ Num cases │ ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥ - │ fddb6678-6de2-427c-96b5-d1c6b3215b0e │ 20240809T135331+0000 │ 20240809T135335+0000 │ 1 │ 1 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ d96a133c-e5a8-4ceb-88de-f8adfb393f28 │ 20240809T135342+0000 │ 20240809T135345+0000 │ 1 │ 1 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ c7508042-64be-406a-89f1-c5d31b90f838 │ 20240809T143710+0000 │ 20240809T143713+0000 │ 1 │ 1 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ 3bc5c067-42fa-4496-a5e4-50b92b3cc38e │ 20240809T144025+0000 │ 20240809T144026+0000 │ 1 │ 2 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ 53481b75-b98a-4668-b6ab-82b199cc2efe │ 20240809T144056+0000 │ 20240809T144057+0000 │ 1 │ 4 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ - ... - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ bed145ca-0013-4b68-bfd4-620054121f91 │ 20240809T144459+0000 │ 20240809T144500+0000 │ 1 │ 10 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ a72c7536-274a-4a21-92c3-4116f38febd0 │ 20240809T144500+0000 │ 20240809T144500+0000 │ 1 │ 1 │ - ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤ - │ 8cb26ff4-7897-42fc-a993-fbdef57c8983 │ 20240809T144510+0000 │ 20240809T144511+0000 │ 1 │ 5 │ + │ fedb2cf8-6efa-43d8-a6dc-e72c868deba6 │ 20240823T104554+0000 │ 20240823T104557+0000 │ 1 │ 1 │ + │ 4253d6b3-3926-4c4c-a7e8-3f7dffe9bf23 │ 20240823T104608+0000 │ 20240823T104612+0000 │ 1 │ 1 │ + │ 453e64a2-f941-49e2-b628-bf50883a6387 │ 20240823T104721+0000 │ 20240823T104725+0000 │ 1 │ 1 │ + │ d923cca2-a72b-43ca-aca1-de741b65088b │ 20240823T104753+0000 │ 20240823T104757+0000 │ 1 │ 1 │ + │ 300b973b-84a6-4932-89eb-577a832fe357 │ 20240823T104814+0000 │ 20240823T104815+0000 │ 1 │ 2 │ + │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e │ 20240823T104834+0000 │ 20240823T104835+0000 │ 1 │ 4 │ + │ 2a00c55d-4492-498c-89f0-7cf821f308c1 │ 20240823T104843+0000 │ 20240823T104845+0000 │ 1 │ 4 │ + │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143 │ 20240823T104902+0000 │ 20240823T104903+0000 │ 1 │ 4 │ + │ 4bbc27bc-be50-4cca-9d1b-c5fb4988a5c0 │ 20240823T104922+0000 │ 20240823T104933+0000 │ 1 │ 26 │ + │ 200ea28f-6c3a-4973-a2b7-aa08408dbeec │ 20240823T104939+0000 │ 20240823T104943+0000 │ 1 │ 10 │ + │ b756755b-3181-4bb4-9eaa-cc8c3a9d7a43 │ 20240823T104955+0000 │ 20240823T104956+0000 │ 1 │ 10 │ + │ a8a99808-c22d-4b9c-83bc-164289fe6aa7 │ 20240823T105007+0000 │ 20240823T105007+0000 │ 1 │ 4 │ + │ f9b63cdc-7dda-44c5-ab85-1e9752047834 │ 20240823T105019+0000 │ 20240823T105020+0000 │ 1 │ 10 │ + │ 271fc2e7-b550-4325-b8bb-57bdf95f1d0d │ 20240823T105020+0000 │ 20240823T105020+0000 │ 1 │ 1 │ + │ 50cdb774-f231-4f61-8472-7daaa5199d57 │ 20240823T105031+0000 │ 20240823T105032+0000 │ 1 │ 5 │ ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━┙ You can use the :option:`--list-stored-testcases` to list the test cases of a specific session or those that have run within a certain period of time: .. code-block:: bash + :caption: Run in the single-node container. - reframe --list-stored-testcases=53481b75-b98a-4668-b6ab-82b199cc2efe + reframe --list-stored-testcases=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e .. code-block:: console ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ - │ build_stream ~tutorialsys:default+gnu │ tutorialsys:default+gnu │ │ 20240809T145439+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:0 │ - ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │ │ 20240809T145439+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:1 │ - ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240809T144057+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:2 │ - ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ stream_test │ tutorialsys:default+clang │ myhost │ 20240809T144057+0000 │ pass │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:3 │ + │ build_stream ~tutorialsys:default+gnu │ tutorialsys:default+gnu │ │ n/a │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:0 │ + │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │ │ n/a │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:1 │ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:3 │ ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ @@ -2046,63 +2042,69 @@ A session may have multiple runs if it has retried some failed test cases (see : You can also list the test cases that have run in a certain period of time use the :ref:`time period ` of :option:`--list-stored-testcases`: .. code-block:: bash + :caption: Run in the single-node container. - reframe --list-stored-testcases=20240809T144500+0000:now + reframe --list-stored-testcases=20240823T104835+0000:now .. code-block:: console - ┍━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ - │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ - ┝━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ - │ T2 │ generic:default+builtin │ myhost │ 20240809T144500+0000 │ fail │ bed145ca-0013-4b68-bfd4-620054121f91:0:7 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T3 │ generic:default+builtin │ myhost │ 20240809T144500+0000 │ pass │ bed145ca-0013-4b68-bfd4-620054121f91:0:9 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T6 │ generic:default+builtin │ myhost │ 20240809T144500+0000 │ pass │ a72c7536-274a-4a21-92c3-4116f38febd0:0:0 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T0 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:0 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T4 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:1 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T5 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:2 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T1 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:3 │ - ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤ - │ T6 │ generic:default+builtin │ myhost │ 20240809T144510+0000 │ pass │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:4 │ - ┕━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ + │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ + ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:3 │ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104844+0000 │ pass │ 2a00c55d-4492-498c-89f0-7cf821f308c1:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104845+0000 │ pass │ 2a00c55d-4492-498c-89f0-7cf821f308c1:0:3 │ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104903+0000 │ pass │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104903+0000 │ pass │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143:0:3 │ + ... + │ T6 │ generic:default+builtin │ myhost │ 20240823T105020+0000 │ pass │ 271fc2e7-b550-4325-b8bb-57bdf95f1d0d:0:0 │ + │ T0 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:0 │ + │ T4 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:1 │ + │ T5 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:2 │ + │ T1 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:3 │ + │ T6 │ generic:default+builtin │ myhost │ 20240823T105032+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:4 │ + ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ To get all the details of a session or a set of test cases you can use the :option:`--describe-stored-session` and :option:`--describe-stored-testcases` options which will return a JSON record with all the details. +You can also combine the :option:`-n` option with the :option:`--list-stored-testcases` and :option:`--describe-stored-testcases` options in order to restrict the listing to specific tests only: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --list-stored-testcases=20240823T104835+0000:now -n stream_test + + Comparing performance of test cases ----------------------------------- ReFrame can be used to compare the performance of the same test cases run in different time periods using the :option:`--performance-compare` option. -The following will compare the performance of the test cases of the session ``a120b895-8fe9-4209-a742-997442e37c47`` with any other same test case that has run the last 24h: +The following will compare the performance of the test cases of the session ``1fb8488e-c361-4355-b7df-c0dcf3cdcc1e`` with any other same test case that has run the last 24h: .. code-block:: bash + :caption: Run in the single-node container. - reframe --performance-compare=a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/ + reframe --performance-compare=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e/now-1d:now/mean:/ .. code-block:: console ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┑ │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┥ - │ stream_test │ tutorialsys:default+gnu │ copy_bw │ 31274.5 │ MB/s │ -22.47% │ - ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤ - │ stream_test │ tutorialsys:default+gnu │ triad_bw │ 18993.4 │ MB/s │ -42.02% │ - ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤ - │ stream_test │ tutorialsys:default+clang │ copy_bw │ 38546.2 │ MB/s │ -9.24% │ - ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤ - │ stream_test │ tutorialsys:default+clang │ triad_bw │ 36866.3 │ MB/s │ -4.72% │ + │ stream_test │ tutorialsys:default+gnu │ copy_bw │ 44139 │ MB/s │ +11.14% │ + │ stream_test │ tutorialsys:default+gnu │ triad_bw │ 39344.7 │ MB/s │ +20.77% │ + │ stream_test │ tutorialsys:default+clang │ copy_bw │ 44979.1 │ MB/s │ +10.81% │ + │ stream_test │ tutorialsys:default+clang │ triad_bw │ 39330.8 │ MB/s │ +8.28% │ ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┙ +The :option:`-n` option can also be combined with :option:`--performance-compare` to restrict the test cases listed. Similarly to the :option:`--performance-compare` option, the :option:`--performance-report` option can compare the performance of the current run with any arbitrary past session or past time period. Finally, you can delete complete a stored session using the :option:`--delete-stored-session` option: .. code-block:: bash - reframe --delete-stored-session=a120b895-8fe9-4209-a742-997442e37c47 + reframe --delete-stored-session=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e Deleting a session will also delete all its test cases from the database. diff --git a/examples/tutorial/dockerfiles/singlenode.Dockerfile b/examples/tutorial/dockerfiles/singlenode.Dockerfile index 86750b20d8..2c8fed9c78 100644 --- a/examples/tutorial/dockerfiles/singlenode.Dockerfile +++ b/examples/tutorial/dockerfiles/singlenode.Dockerfile @@ -12,13 +12,13 @@ ARG REFRAME_REPO=reframe-hpc WORKDIR /usr/local/share # Clone reframe -RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ - cd reframe/ && ./bootstrap.sh +# RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ +# cd reframe/ && ./bootstrap.sh # Comment the above line and uncomment the following two for development -# COPY . /usr/local/share/reframe -# RUN cd reframe && ./bootstrap.sh +COPY . /usr/local/share/reframe +RUN cd reframe && ./bootstrap.sh ENV PATH=/usr/local/share/reframe/bin:$PATH From e2c7cb5e4f8705643a45fe355c1a1112b453a652 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 15:09:14 +0300 Subject: [PATCH 63/69] Fix listing formatting in tutorial --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 888671c660..f842125069 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1997,7 +1997,7 @@ its unique identifier, its start and end time and how many test cases have run: .. code-block:: console - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑ + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑ │ UUID │ Start time │ End time │ Num runs │ Num cases │ ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥ │ fedb2cf8-6efa-43d8-a6dc-e72c868deba6 │ 20240823T104554+0000 │ 20240823T104557+0000 │ 1 │ 1 │ From d85690ca205aaf1f94f6a78e3bf2f62fefe57abb Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 15:59:22 +0300 Subject: [PATCH 64/69] Support for using always unqualified hostnames in local scheduler --- docs/config_reference.rst | 10 ++++++++++ reframe/core/schedulers/local.py | 7 ++++++- reframe/schemas/config.json | 2 ++ unittests/test_schedulers.py | 15 +++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index beeb9b457b..57fbcd4daa 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -439,6 +439,16 @@ System Partition Configuration No other test would be able to proceed. +.. py:attribute:: systems.partitions.sched_options.unqualified_hostnames + + :required: No + :default: ``false`` + + Use unqualified hostnames in the ``local`` scheduler backend. + + .. versionadded:: 4.7 + + .. py:attribute:: systems.partitions.sched_options.use_nodes_option :required: No diff --git a/reframe/core/schedulers/local.py b/reframe/core/schedulers/local.py index 5e41491f0f..87eead7530 100644 --- a/reframe/core/schedulers/local.py +++ b/reframe/core/schedulers/local.py @@ -70,7 +70,12 @@ def submit(self, job): # Update job info job._jobid = proc.pid - job._nodelist = [socket.gethostname()] + hostname = socket.gethostname() + if self.get_option('unqualified_hostnames'): + job._nodelist = [hostname.split('.')[0]] + else: + job._nodelist = [hostname] + job._proc = proc job._f_stdout = f_stdout job._f_stderr = f_stderr diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 06765f604a..7183f787d0 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -120,6 +120,7 @@ "type": "array", "items": {"type": "string"} }, + "unqualified_hostnames": {"type": "boolean"}, "use_nodes_option": {"type": "boolean"} } }, @@ -659,6 +660,7 @@ "systems*/sched_options/ignore_reqnodenotavail": false, "systems*/sched_options/job_submit_timeout": 60, "systems*/sched_options/resubmit_on_errors": [], + "systems*/sched_options/unqualified_hostnames": false, "systems*/sched_options/use_nodes_option": false } } diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index b2da80bcfb..a11223121f 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -522,6 +522,21 @@ def test_submit_timelimit(minimal_job, local_only): assert minimal_job.state == 'TIMEOUT' +def test_submit_unqualified_hostnames(make_exec_ctx, make_job, local_only): + make_exec_ctx( + system='testsys', + options={ + 'systems/partitions/sched_options/unqualified_hostnames': True + } + ) + hostname = socket.gethostname().split('.')[0] + minimal_job = make_job(sched_opts={'part_name': 'login'}) + minimal_job.prepare('true') + minimal_job.submit() + minimal_job.wait() + assert minimal_job.nodelist == [hostname] + + def test_submit_job_array(make_job, slurm_only, exec_ctx): job = make_job(sched_access=exec_ctx.access) job.options = ['--array=0-1'] From 4088f2b7f1326fd109e32240a8ed31b5c3365813 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Aug 2024 16:24:39 +0300 Subject: [PATCH 65/69] Make perf. reference selectable as a column --- reframe/frontend/reporting/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index 9500c4ec13..3c952f8770 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -516,6 +516,7 @@ def _group_testcases(testcases, group_by, extra_cols): record = { 'pvar': pvar, 'pval': pval, + 'pref': pref, 'plower': plower, 'pupper': pupper, 'punit': punit, From 73af49fa7a31219608efc1bc19c062cfe17d7cb5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 28 Aug 2024 12:01:28 +0200 Subject: [PATCH 66/69] Apply suggestions from code review Co-authored-by: Theofilos Manitaras --- docs/manpage.rst | 8 ++++---- docs/tutorial.rst | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 50387abdbb..1265067a51 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1096,7 +1096,7 @@ Miscellaneous options For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified. - If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run oif the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``). + If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run of the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``). For the exact syntax of ``CMPSPEC``, refer to :ref:`performance-comparisons`. @@ -1297,15 +1297,15 @@ Result storage .. versionadded:: 4.7 -ReFrame stores the results of every session that has exectuted at least one test into a database. -There is only one storage backend supported at the moment and this is Sqlite. +ReFrame stores the results of every session that has executed at least one test into a database. +There is only one storage backend supported at the moment and this is SQLite. The full session information as recorded in a run report file (see :option:`--report-file`) is stored in the database. The test cases of the session are indexed by their run job completion time for quick retrieval of all the test cases that have run in a certain period of time. The database file is controlled by the :attr:`~config.storage.sqlite_db_file` configuration parameter and multiple ReFrame processes can access it safely simultaneously. There are several command-line options that allow users to query the results database, such as the :option:`--list-stored-sessions`, :option:`--list-stored-testcases`, :option:`--describe-stored-session` etc. -Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` options which compare the performance results of the same test cases in different periods of time or from different sessions. +Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` which compare the performance results of the same test cases in different periods of time or from different sessions. Check the :ref:`commands` section for the complete list and details of each option related to the results database. Since the report file information is now kept in the results database, there is no need to keep the report files separately, although this remains the default behavior for backward compatibility. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index f842125069..f0b9540291 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -168,7 +168,7 @@ This can be suppressed by increasing the level at which this information is logg Run reports and performance logging ----------------------------------- -Once a test session finishes, ReFrame stores the detailed session information in database file located under ``$HOME/.reframe/reports``. +Once a test session finishes, ReFrame stores the detailed session information in a database file located under ``$HOME/.reframe/reports``. Past performance data can be retrieved from this database and compared with the current or another run. We detail handling of the results database in section :ref:`inspecting-past-results`. @@ -180,7 +180,7 @@ These files are located by default under ``perflogs/// Date: Wed, 28 Aug 2024 11:56:35 +0200 Subject: [PATCH 67/69] Print info line about results DB file --- reframe/frontend/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 6fa3c51b34..da801dd453 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1158,6 +1158,10 @@ def print_infoline(param, value): print_infoline('output directory', repr(session_info['prefix_output'])) print_infoline('log files', ', '.join(repr(s) for s in session_info['log_files'])) + print_infoline( + 'results database', + repr(osext.expandvars(rt.get_option('storage/0/sqlite_db_file'))) + ) printer.info('') try: logging.getprofiler().enter_region('test processing') From 75763c901e85e87ef88d1ca53b8d766087edeb50 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 28 Aug 2024 11:57:37 +0200 Subject: [PATCH 68/69] Fix coding style --- reframe/frontend/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index da801dd453..e4e3bd1f27 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1000,7 +1000,7 @@ def restrict_logging(): try: printer.table( reporting.performance_compare(options.performance_compare, - namepatt=namepatt) + namepatt=namepatt) ) except errors.ReframeError as err: printer.error(f'failed to generate performance report: {err}') From 2572d1fa0801e4ccce48deb3073102369a7dc220 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 28 Aug 2024 15:29:37 +0200 Subject: [PATCH 69/69] Address remaining PR comments --- docs/manpage.rst | 2 +- docs/tutorial.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 1265067a51..5630174514 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1422,7 +1422,7 @@ The ```` is an absolute timestamp in one of the following ``strpt Optionally, a shift argument can be appended with ``+`` or ``-`` signs, followed by an amount of weeks (``w``), days (``d``), hours (``h``) or minutes (``m``). -For example, the period of the last 10 days can be specified as ``now:now-10d``. +For example, the period of the last 10 days can be specified as ``now-10d:now``. Similarly, the period of the week starting on August 5, 2024 will be specified as ``20240805:20240805+1w``. Environment diff --git a/docs/tutorial.rst b/docs/tutorial.rst index f0b9540291..e2d8bd2a99 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -170,7 +170,7 @@ Run reports and performance logging Once a test session finishes, ReFrame stores the detailed session information in a database file located under ``$HOME/.reframe/reports``. Past performance data can be retrieved from this database and compared with the current or another run. -We detail handling of the results database in section :ref:`inspecting-past-results`. +We explain in detail the handling of the results database in section :ref:`inspecting-past-results`. By default, the session information is also saved in a JSON report file under ``$HOME/.reframe/reports``. The latest report is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given.