From 9a312f6577e6c3eeb75319a32c6c479cf0154236 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 13 Nov 2024 18:31:48 +0530 Subject: [PATCH 1/4] use the default sut folder name supplied if cm-sut-json is not there in root --- .../customize.py | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 57475a9b51..72b7c17a8a 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -221,29 +221,9 @@ def generate_submission(i): run_config = sut_info["run_config"] new_res = f"{system}-{implementation}-{device}-{framework}-{run_config}" else: - parts = res.split("-") - if len(parts) > 5: #result folder structure used by CM script - system = parts[0] if system == 'default' else system - implementation = parts[1] - device = parts[2] - framework = parts[3] - framework_version = parts[4] - run_config = parts[5] - - print('* System: {}'.format(system)) - print('* Implementation: {}'.format(implementation)) - print('* Device: {}'.format(device)) - print('* Framework: {}'.format(framework)) - print('* Framework Version: {}'.format(framework_version)) - print('* Run Config: {}'.format(run_config)) - - new_res = system + "-" + "-".join(parts[1:]) - - # Override framework and framework versions from the folder name - system_meta_default['framework'] = framework + " " + framework_version - else: - print(parts) - return {'return': 1, 'error': f"The required details for generating the inference submission:\n1.hardware_name\n2.implementation\n3.Device\n4.framework\n5.framework_version\n6.run_config\nInclude a cm-sut-info.json or sut-info.json file with the above content in {result_path}"} + new_res = res + + print(f"The SUT folder name for submission generation is: {new_res}") platform_prefix = inp.get('platform_prefix', '') if platform_prefix: From 4be11398830ea188c83e0aa4c8eee057df536a4f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 14 Nov 2024 00:49:40 +0530 Subject: [PATCH 2/4] set pytorch as default backend --- script/get-mlperf-inference-sut-description/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-sut-description/customize.py b/script/get-mlperf-inference-sut-description/customize.py index ec293a5316..d519e6e429 100644 --- a/script/get-mlperf-inference-sut-description/customize.py +++ b/script/get-mlperf-inference-sut-description/customize.py @@ -18,7 +18,7 @@ def preprocess(i): hw_name = env['CM_HW_NAME'] - backend = env.get('CM_MLPERF_BACKEND', '') + backend = env.get('CM_MLPERF_BACKEND', 'pytorch') # default is set to pytorch backend_version = env.get('CM_MLPERF_BACKEND_VERSION', '') sut_suffix = '' backend_name = '' From aaa7b5ce1a054fe0948c2c423da4a1ccb89a1b3c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 14 Nov 2024 00:55:52 +0530 Subject: [PATCH 3/4] revert commit https://github.com/mlcommons/cm4mlops/pull/537/commits/4be11398830ea188c83e0aa4c8eee057df536a4f --- script/get-mlperf-inference-sut-description/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-sut-description/customize.py b/script/get-mlperf-inference-sut-description/customize.py index d519e6e429..1da58c7d14 100644 --- a/script/get-mlperf-inference-sut-description/customize.py +++ b/script/get-mlperf-inference-sut-description/customize.py @@ -18,7 +18,7 @@ def preprocess(i): hw_name = env['CM_HW_NAME'] - backend = env.get('CM_MLPERF_BACKEND', 'pytorch') # default is set to pytorch + backend = env.get('CM_MLPERF_BACKEND', '') backend_version = env.get('CM_MLPERF_BACKEND_VERSION', '') sut_suffix = '' backend_name = '' From cf30e25aa7da54b3f641ccdef9f4bcd952ebbdbd Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 14 Nov 2024 01:02:49 +0530 Subject: [PATCH 4/4] set pytorch as default framework --- script/generate-mlperf-inference-submission/customize.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 72b7c17a8a..580d8a8d51 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -88,6 +88,11 @@ def generate_submission(i): results = [f for f in os.listdir(results_dir) if not os.path.isfile(os.path.join(results_dir, f))] system_meta_default = state['CM_SUT_META'] + + # set pytorch as the default framework + if system_meta_default['framework'] == '': + system_meta_default['framework'] = "pytorch" + system_meta = {} if 'CM_MLPERF_SUBMISSION_SYSTEM_TYPE' in env: system_meta['system_type'] = env['CM_MLPERF_SUBMISSION_SYSTEM_TYPE']