From 71db29b2eb3c69ede214a16d45deed082dcdd2dc Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:17:07 +0530 Subject: [PATCH 01/23] Commit for obtaining platform details --- .../customize.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 3e9bcd3763..43c5380d35 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -238,6 +238,20 @@ def generate_submission(env, state, inp, submission_division): compliance_path = os.path.join(path_submission, "compliance", sub_res) system_path = os.path.join(path_submission, "systems") submission_system_path = system_path + + if env.get('CM_GET_PLATFORM_DETAILS', '') == "yes": + cm_input = {'action': 'run', + 'automation': 'script', + 'tags': 'get,platform,details', + 'adr': '', + 'print_deps': True, + 'env': {'CM_PLATFORM_DETAILS_FILE_PATH':f"{measurement_path}/system_info.txt"}, + 'quiet': True + } + r = cm.access(cm_input) + if r['return'] > 0: + return r + if not os.path.isdir(submission_system_path): os.makedirs(submission_system_path) system_file = os.path.join(submission_system_path, sub_res+".json") From b49e1776b6638f7ee446688c0adbbd148ed8ed38 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:18:39 +0530 Subject: [PATCH 02/23] added option to get platform details --- script/generate-mlperf-inference-submission/_cm.json | 1 + 1 file changed, 1 insertion(+) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 3eb04d96b9..cab362359e 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -69,6 +69,7 @@ "submitter": "CM_MLPERF_SUBMITTER", "preprocess": "CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR", "preprocess_submission": "CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR", + "get_platform_details": "CM_GET_PLATFORM_DETAILS", "tar": "CM_TAR_SUBMISSION_DIR" }, "default_env": { From 35aa2ac5cb1b275aa84bf391c9ea167c1269f08e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:38:38 +0530 Subject: [PATCH 03/23] fix typo --- script/generate-mlperf-inference-submission/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 43c5380d35..59a9e9c64d 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -2,7 +2,7 @@ import os import json import shutil -import cmind +import cmind import sys from tabulate import tabulate import mlperf_utils @@ -248,7 +248,7 @@ def generate_submission(env, state, inp, submission_division): 'env': {'CM_PLATFORM_DETAILS_FILE_PATH':f"{measurement_path}/system_info.txt"}, 'quiet': True } - r = cm.access(cm_input) + r = cmind.access(cm_input) if r['return'] > 0: return r From 1081135e1c9b40f85136776008aa669cc5984840 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:42:04 +0530 Subject: [PATCH 04/23] create measurements path if not present --- script/generate-mlperf-inference-submission/customize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 59a9e9c64d..e07923f2aa 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -239,6 +239,9 @@ def generate_submission(env, state, inp, submission_division): system_path = os.path.join(path_submission, "systems") submission_system_path = system_path + if not os.path.isdir(measurement_path): + os.makedirs(measurement_path) + if env.get('CM_GET_PLATFORM_DETAILS', '') == "yes": cm_input = {'action': 'run', 'automation': 'script', From f9db49a5af1da5b6d78ddf5127457f856685b43a Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 22 Nov 2024 13:22:07 +0530 Subject: [PATCH 05/23] looks for file with precendence --- .../customize.py | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index e07923f2aa..2b32cea7e2 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -2,7 +2,7 @@ import os import json import shutil -import cmind +import cmind import sys from tabulate import tabulate import mlperf_utils @@ -239,22 +239,6 @@ def generate_submission(env, state, inp, submission_division): system_path = os.path.join(path_submission, "systems") submission_system_path = system_path - if not os.path.isdir(measurement_path): - os.makedirs(measurement_path) - - if env.get('CM_GET_PLATFORM_DETAILS', '') == "yes": - cm_input = {'action': 'run', - 'automation': 'script', - 'tags': 'get,platform,details', - 'adr': '', - 'print_deps': True, - 'env': {'CM_PLATFORM_DETAILS_FILE_PATH':f"{measurement_path}/system_info.txt"}, - 'quiet': True - } - r = cmind.access(cm_input) - if r['return'] > 0: - return r - if not os.path.isdir(submission_system_path): os.makedirs(submission_system_path) system_file = os.path.join(submission_system_path, sub_res+".json") @@ -288,6 +272,8 @@ def generate_submission(env, state, inp, submission_division): print('* MLPerf inference model: {}'.format(model)) for scenario in scenarios: + # the system_info.txt is copied from the mode directory if found, else it would be looked under scenario directory + system_info_from_mode = False results[model][scenario] = {} result_scenario_path = os.path.join(result_model_path, scenario) submission_scenario_path = os.path.join(submission_model_path, scenario) @@ -424,7 +410,11 @@ def generate_submission(env, state, inp, submission_division): files.append(f) elif f in [ "README.md", "README-extra.md", "cm-version-info.json", "os_info.json", "cpu_info.json", "pip_freeze.json", "system_info.txt" ] and mode == "performance": shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, f)) + if f == "system_info.txt": + system_info_from_mode = True if f == "system_info.txt" and not platform_info_file: + # the first system_info.txt would be taken as platform info file for a specific model to be placed in + # measurements-model folder when generating the final submission platform_info_file = os.path.join(result_mode_path, f) elif f in [ "console.out" ]: shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, mode+"_"+f)) @@ -441,6 +431,10 @@ def generate_submission(env, state, inp, submission_division): p_target = os.path.join(submission_results_path, f) shutil.copy(os.path.join(result_mode_path, f), p_target) + if os.path.exists(os.path.join(result_scenario_path, "system_info.txt")) and not system_info_from_mode: + shutil.copy(os.path.join(result_scenario_path, "system_info.txt"), os.path.join(submission_measurement_path, f)) + if not platform_info_file: + platform_info_file = os.path.join(result_scenario_path, "system_info.txt") readme_file = os.path.join(submission_measurement_path, "README.md") if not os.path.exists(readme_file): @@ -455,24 +449,39 @@ def generate_submission(env, state, inp, submission_division): with open(readme_file, mode='a') as f: f.write(result_string) - #Copy system_info.txt to the submission measurements model folder if any scenario performance run has it + # Copy system_info.txt to the submission measurements model folder if any scenario performance run has it sys_info_file = None - if os.path.exists(os.path.join(result_model_path, "system_info.txt")): - sys_info_file = os.path.join(result_model_path, "system_info.txt") - elif platform_info_file: + if platform_info_file: sys_info_file = platform_info_file + elif os.path.exists(os.path.join(result_model_path, "system_info.txt")): + sys_info_file = os.path.join(result_model_path, "system_info.txt") + if sys_info_file: - model_platform_info_file = sys_info_file + # The first system_info file found would also be taken as the one for copying to SUT folder + if not model_platform_info_file: + model_platform_info_file = sys_info_file shutil.copy(sys_info_file, os.path.join(measurement_model_path, "system_info.txt")) #Copy system_info.txt to the submission measurements folder if any model performance run has it sys_info_file = None - if os.path.exists(os.path.join(result_path, "system_info.txt")): - sys_info_file = os.path.join(result_path, "system_info.txt") - elif model_platform_info_file: + if model_platform_info_file: sys_info_file = model_platform_info_file + elif os.path.exists(os.path.join(result_path, "system_info.txt")): + sys_info_file = os.path.join(result_path, "system_info.txt") + if sys_info_file: shutil.copy(sys_info_file, os.path.join(measurement_path, "system_info.txt")) + else: + if env.get('CM_GET_PLATFORM_DETAILS', '') == "yes": + cm_input = {'action': 'run', + 'automation': 'script', + 'tags': 'get,platform,details', + 'env': {'CM_PLATFORM_DETAILS_FILE_PATH': os.path.join(measurement_path, "system_info.txt")}, + 'quiet': True + } + r = cmind.access(cm_input) + if r['return'] > 0: + return r with open(system_file, "w") as fp: From 63719154f468800fa8a04b2d4e5a5f26c2c5d364 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 22 Nov 2024 19:50:17 +0530 Subject: [PATCH 06/23] improved logic --- .../customize.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 6803e9cf99..1a38d58af8 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -431,8 +431,6 @@ def generate_submission(env, state, inp, submission_division): files.append(f) elif f in [ "README.md", "README-extra.md", "cm-version-info.json", "os_info.json", "cpu_info.json", "pip_freeze.json", "system_info.txt", "cm-deps.png", "cm-deps.mmd" ] and mode == "performance": shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, f)) - if f == "system_info.txt": - system_info_from_mode = True if f == "system_info.txt" and not platform_info_file: # the first system_info.txt would be taken as platform info file for a specific model to be placed in # measurements-model folder when generating the final submission @@ -452,10 +450,9 @@ def generate_submission(env, state, inp, submission_division): p_target = os.path.join(submission_results_path, f) shutil.copy(os.path.join(result_mode_path, f), p_target) - if os.path.exists(os.path.join(result_scenario_path, "system_info.txt")) and not system_info_from_mode: + if os.path.exists(os.path.join(result_scenario_path, "system_info.txt")): shutil.copy(os.path.join(result_scenario_path, "system_info.txt"), os.path.join(submission_measurement_path, f)) - if not platform_info_file: - platform_info_file = os.path.join(result_scenario_path, "system_info.txt") + platform_info_file = os.path.join(result_scenario_path, "system_info.txt") readme_file = os.path.join(submission_measurement_path, "README.md") if not os.path.exists(readme_file): @@ -472,10 +469,11 @@ def generate_submission(env, state, inp, submission_division): # Copy system_info.txt to the submission measurements model folder if any scenario performance run has it sys_info_file = None - if platform_info_file: - sys_info_file = platform_info_file - elif os.path.exists(os.path.join(result_model_path, "system_info.txt")): + + if os.path.exists(os.path.join(result_model_path, "system_info.txt")): sys_info_file = os.path.join(result_model_path, "system_info.txt") + elif platform_info_file: + sys_info_file = platform_info_file if sys_info_file: # The first system_info file found would also be taken as the one for copying to SUT folder @@ -485,10 +483,11 @@ def generate_submission(env, state, inp, submission_division): #Copy system_info.txt to the submission measurements folder if any model performance run has it sys_info_file = None - if model_platform_info_file: - sys_info_file = model_platform_info_file - elif os.path.exists(os.path.join(result_path, "system_info.txt")): + + if os.path.exists(os.path.join(result_path, "system_info.txt")): sys_info_file = os.path.join(result_path, "system_info.txt") + elif model_platform_info_file: + sys_info_file = model_platform_info_file if sys_info_file: shutil.copy(sys_info_file, os.path.join(measurement_path, "system_info.txt")) From efc9371ab7ed90bd9735ba372361c84287cb3959 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 22 Nov 2024 19:54:57 +0530 Subject: [PATCH 07/23] precedence correction --- script/generate-mlperf-inference-submission/customize.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 1a38d58af8..464a49d0ba 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -432,7 +432,7 @@ def generate_submission(env, state, inp, submission_division): elif f in [ "README.md", "README-extra.md", "cm-version-info.json", "os_info.json", "cpu_info.json", "pip_freeze.json", "system_info.txt", "cm-deps.png", "cm-deps.mmd" ] and mode == "performance": shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, f)) if f == "system_info.txt" and not platform_info_file: - # the first system_info.txt would be taken as platform info file for a specific model to be placed in + # the first found system_info.txt would be taken as platform info file for a specific model to be placed in # measurements-model folder when generating the final submission platform_info_file = os.path.join(result_mode_path, f) elif f in [ "console.out" ]: @@ -476,9 +476,7 @@ def generate_submission(env, state, inp, submission_division): sys_info_file = platform_info_file if sys_info_file: - # The first system_info file found would also be taken as the one for copying to SUT folder - if not model_platform_info_file: - model_platform_info_file = sys_info_file + model_platform_info_file = sys_info_file shutil.copy(sys_info_file, os.path.join(measurement_model_path, "system_info.txt")) #Copy system_info.txt to the submission measurements folder if any model performance run has it From 93e8eadeaeeb11156286a21a68169c74a54e6249 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 22 Nov 2024 14:36:44 +0000 Subject: [PATCH 08/23] Update customize.py --- script/generate-mlperf-inference-submission/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 464a49d0ba..6516451fd2 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -432,7 +432,7 @@ def generate_submission(env, state, inp, submission_division): elif f in [ "README.md", "README-extra.md", "cm-version-info.json", "os_info.json", "cpu_info.json", "pip_freeze.json", "system_info.txt", "cm-deps.png", "cm-deps.mmd" ] and mode == "performance": shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, f)) if f == "system_info.txt" and not platform_info_file: - # the first found system_info.txt would be taken as platform info file for a specific model to be placed in + # the first found system_info.txt will be taken as platform info file for a specific model to be placed in # measurements-model folder when generating the final submission platform_info_file = os.path.join(result_mode_path, f) elif f in [ "console.out" ]: From 49e68c45120495ce510bc9f69972e9cc2597d0b4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 20:20:10 +0530 Subject: [PATCH 09/23] revert platform details input --- script/generate-mlperf-inference-submission/_cm.json | 1 - 1 file changed, 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index cab362359e..3eb04d96b9 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -69,7 +69,6 @@ "submitter": "CM_MLPERF_SUBMITTER", "preprocess": "CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR", "preprocess_submission": "CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR", - "get_platform_details": "CM_GET_PLATFORM_DETAILS", "tar": "CM_TAR_SUBMISSION_DIR" }, "default_env": { From f458fb0dfcc77944539669a45455e379e2d22ea8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 20:21:41 +0530 Subject: [PATCH 10/23] add platform details input mapping --- script/generate-mlperf-inference-submission/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/generate-mlperf-inference-submission/_cm.yaml b/script/generate-mlperf-inference-submission/_cm.yaml index 9e6200556b..32003a1b33 100644 --- a/script/generate-mlperf-inference-submission/_cm.yaml +++ b/script/generate-mlperf-inference-submission/_cm.yaml @@ -83,6 +83,7 @@ input_mapping: submitter: CM_MLPERF_SUBMITTER sw_notes_extra: CM_MLPERF_SUT_SW_NOTES_EXTRA tar: CM_TAR_SUBMISSION_DIR + get_platform_details: CM_GET_PLATFORM_DETAILS post_deps: - enable_if_env: CM_RUN_MLPERF_ACCURACY: From bdf8d9e261c7b521d9e9f166c4f0b36ee7cc512d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 20:52:17 +0530 Subject: [PATCH 11/23] add open-case-1 and 2 --- .../test-cm-based-submission-generation.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 68f46267aa..a439914bb5 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -19,12 +19,22 @@ jobs: python-version: [ "3.12" ] division: ["closed", "open", "closed-open"] category: ["datacenter", "edge"] - case: ["closed", "closed-no-compliance", "closed-power", "closed-failed-power-logs", "case-3", "case-7", "case-8"] + case: ["closed", "closed-no-compliance", "closed-power", "closed-failed-power-logs", "case-1", "case-2", "case-3", "case-7", "case-8"] action: ["run", "docker"] exclude: - os: macos-latest - os: windows-latest - category: "edge" + - case: case-1 + division: closed + - case: case-1 + division: closed-open + - case: case-2 + division: closed + - case: case-2 + division: closed-open + + steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 1f0214827e6772cee84d6e018129273e165cce8e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:09:34 +0530 Subject: [PATCH 12/23] add case 5 and 6 --- .../test-cm-based-submission-generation.yml | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index a439914bb5..d4c489b52f 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -19,7 +19,7 @@ jobs: python-version: [ "3.12" ] division: ["closed", "open", "closed-open"] category: ["datacenter", "edge"] - case: ["closed", "closed-no-compliance", "closed-power", "closed-failed-power-logs", "case-1", "case-2", "case-3", "case-7", "case-8"] + case: ["closed", "closed-no-compliance", "closed-power", "closed-failed-power-logs", "case-1", "case-2", "case-3", "case-5", "case-6", "case-7", "case-8"] action: ["run", "docker"] exclude: - os: macos-latest @@ -33,8 +33,15 @@ jobs: division: closed - case: case-2 division: closed-open + - case: case-5 + division: closed + - case: case-5 + division: closed-open + - case: case-6 + division: closed + - case: case-6 + division: closed-open - steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -73,5 +80,17 @@ jobs: # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args + + if [[ "${{ matrix.case }}" -eq 5 || "${{ matrix.case }}" -eq 6 ]]; then + # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds + if [[ $exit_status -ne 0 ]]; then + exit 0 + else + exit 1 + fi + else + # For other cases, exit with the original status + test $exit_status -eq 0 || exit 1 + fi echo "::endgroup::" From bde755c9812371fca5d2a5fad163fc8affe78718 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:13:16 +0530 Subject: [PATCH 13/23] fix typo --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index d4c489b52f..4a1eabfbd1 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -81,7 +81,7 @@ jobs: echo "::group::$description" cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args - if [[ "${{ matrix.case }}" -eq 5 || "${{ matrix.case }}" -eq 6 ]]; then + if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds if [[ $exit_status -ne 0 ]]; then exit 0 From e0f1a5c695dc2b5e6b61fa203fd5fd7aa7da1fee Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:14:24 +0530 Subject: [PATCH 14/23] Add exit status --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 4a1eabfbd1..c1c8314460 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -80,7 +80,7 @@ jobs: # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args - + exit_status=$? if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds if [[ $exit_status -ne 0 ]]; then From 813615896bbba29350dcfc988a4285a8735020da Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:16:31 +0530 Subject: [PATCH 15/23] Update test-cm-based-submission-generation.yml --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index c1c8314460..3e7e342286 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -79,7 +79,7 @@ jobs: fi # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" - cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args + eval cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args exit_status=$? if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds From 6747bd155bd89c1932b8f93e678febc90a17c32f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:29:21 +0530 Subject: [PATCH 16/23] test commit --- .github/workflows/test-cm-based-submission-generation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 3e7e342286..78d3532a0b 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -81,6 +81,7 @@ jobs: echo "::group::$description" eval cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args exit_status=$? + echo $exit_status if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds if [[ $exit_status -ne 0 ]]; then From 8ad1f875a9505e4baf8f7778097c81193321138b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:34:35 +0530 Subject: [PATCH 17/23] test commit --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 78d3532a0b..21f92509c1 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -81,7 +81,7 @@ jobs: echo "::group::$description" eval cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args exit_status=$? - echo $exit_status + echo "Exit status for the job $description $exit_status" if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds if [[ $exit_status -ne 0 ]]; then From 97fd3940da106c40c67d16e6d94cbfcb00daa215 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:44:02 +0530 Subject: [PATCH 18/23] test commit --- .github/workflows/test-cm-based-submission-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 21f92509c1..4048e71193 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -81,17 +81,17 @@ jobs: echo "::group::$description" eval cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args exit_status=$? - echo "Exit status for the job $description $exit_status" + echo "Exit status for the job ${description} ${exit_status}" if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds - if [[ $exit_status -ne 0 ]]; then + if [[ ${exit_status} -ne 0 ]]; then exit 0 else exit 1 fi else # For other cases, exit with the original status - test $exit_status -eq 0 || exit 1 + test ${exit_status} -eq 0 || exit 1 fi echo "::endgroup::" From f50e302d67155672432b7bc12ab1db4e8785f866 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:52:31 +0530 Subject: [PATCH 19/23] test commit --- .github/workflows/test-cm-based-submission-generation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 4048e71193..6d06c40f79 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -57,6 +57,7 @@ jobs: git clone -b submission-generation-tests https://github.com/mlcommons/inference.git submission_generation_tests - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} run: | + case=${{ matrix.case }} if [ "${{ matrix.case }}" == "case-3" ]; then description="Submission generation (model_mapping.json not present but model name matches with official one)" elif [ "${{ matrix.case }}" == "case-7" ]; then @@ -82,7 +83,7 @@ jobs: eval cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args exit_status=$? echo "Exit status for the job ${description} ${exit_status}" - if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then + if [[ "${case}" == "case-5" || "${case}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds if [[ ${exit_status} -ne 0 ]]; then exit 0 From dbe1d66f00ace5e4d3c53a2336a916056dd55270 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:15:40 +0530 Subject: [PATCH 20/23] continue on error --- .github/workflows/test-cm-based-submission-generation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 6d06c40f79..b148450959 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -12,6 +12,7 @@ on: jobs: submission_generation: runs-on: ${{ matrix.os }} + continue-on-error: true strategy: fail-fast: false matrix: From 3ede08f633b99937ac1af08f939dc12586489468 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:18:23 +0530 Subject: [PATCH 21/23] Update test-cm-based-submission-generation.yml --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index b148450959..03e1433981 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -12,7 +12,6 @@ on: jobs: submission_generation: runs-on: ${{ matrix.os }} - continue-on-error: true strategy: fail-fast: false matrix: @@ -57,6 +56,7 @@ jobs: run: | git clone -b submission-generation-tests https://github.com/mlcommons/inference.git submission_generation_tests - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} + continue-on-error: true run: | case=${{ matrix.case }} if [ "${{ matrix.case }}" == "case-3" ]; then From ff6e8274dced9fe6f6399c88f0600b3a442987ac Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 22 Nov 2024 22:30:49 +0530 Subject: [PATCH 22/23] code clean --- .../workflows/test-cm-based-submission-generation.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 03e1433981..4bd9d94304 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -58,7 +58,6 @@ jobs: - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} continue-on-error: true run: | - case=${{ matrix.case }} if [ "${{ matrix.case }}" == "case-3" ]; then description="Submission generation (model_mapping.json not present but model name matches with official one)" elif [ "${{ matrix.case }}" == "case-7" ]; then @@ -81,19 +80,19 @@ jobs: fi # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" - eval cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args + cm ${{ matrix.action }} script --tags=generate,inference,submission --adr.submission-checker-src.tags=_branch.dev --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args exit_status=$? echo "Exit status for the job ${description} ${exit_status}" - if [[ "${case}" == "case-5" || "${case}" == "case-6" ]]; then + if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then # For cases 5 and 6, exit status should be 0 if cm command fails, 1 if it succeeds if [[ ${exit_status} -ne 0 ]]; then exit 0 else - exit 1 + exit ${exit_status} fi else # For other cases, exit with the original status - test ${exit_status} -eq 0 || exit 1 + test ${exit_status} -eq 0 || exit ${exit_status} fi echo "::endgroup::" From 6eefe2faa982ad10d820d249a6bef9a3c9e54fdf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 22 Nov 2024 17:08:12 +0000 Subject: [PATCH 23/23] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1d0ba9ea18..267577d47e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0 +0.4.1