Support version in preprocess-submission, cleanups for coco2014 script (

#76) * Dont use 'install' sub directory for coco2014 * Update test-scc24-sdxl.yaml * Update test-mlperf-inference-mixtral.yml * Support version in preprocess-mlperf-inference-submission
mlcommons · Dec 23, 2024 · cf575d0 · cf575d0
1 parent d28df7e
commit cf575d0
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 10 deletions.
diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml
@@ -5,7 +5,7 @@ name: MLPerf inference MIXTRAL-8x7B
 
 on:
   schedule:
-    - cron: "08 23 * * *"   # 30th minute and 20th hour => 20:30 UTC => 2 AM IST 
+    - cron: "59 19 * * *"   # 30th minute and 20th hour => 20:30 UTC => 2 AM IST 
 
 jobs:
   build_reference:

diff --git a/.github/workflows/test-nvidia-mlperf-inference-implementations.yml b/.github/workflows/test-nvidia-mlperf-inference-implementations.yml
@@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations
 
 on:
   schedule:
-    - cron: "08 01 * * *" #to be adjusted
+    - cron: "08 01 * * */3" #to be adjusted
 
 jobs:
   run_nvidia:

diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml
@@ -2,7 +2,7 @@ name: MLPerf inference SDXL (SCC)
 
 on:
   schedule:
-    - cron: "56 22 * * *"
+    - cron: "34 19 * * *"
 
 jobs:
   build_reference:

diff --git a/script/get-dataset-coco2014/customize.py b/script/get-dataset-coco2014/customize.py
@@ -27,14 +27,14 @@ def postprocess(i):
             os.getcwd(), 'install', 'sample_ids.txt')
         print(env['CM_COCO2014_SAMPLE_ID_PATH'])
     if env.get('CM_DATASET_CALIBRATION', '') == "no":
-        env['CM_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install')
+        env['CM_DATASET_PATH_ROOT'] = os.getcwd()
         # env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data')
         env['CM_DATASET_CAPTIONS_DIR_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'captions')
+            os.getcwd(), 'captions')
         env['CM_DATASET_LATENTS_DIR_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'latents')
+            os.getcwd(), 'latents')
     else:
         env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'calibration', 'data')
+            os.getcwd(), 'calibration', 'data')
 
     return {'return': 0}
diff --git a/script/get-dataset-coco2014/run.sh b/script/get-dataset-coco2014/run.sh
@@ -5,8 +5,7 @@ python3() {
 export -f python3
 
 CUR=${PWD}
-mkdir -p install
-INSTALL_DIR=${CUR}/install
+INSTALL_DIR=${CUR}
 
 cd ${CM_RUN_DIR}
 

diff --git a/script/preprocess-mlperf-inference-submission/_cm.yaml b/script/preprocess-mlperf-inference-submission/_cm.yaml
@@ -22,6 +22,7 @@ deps:
 input_mapping:
   input: CM_MLPERF_INFERENCE_SUBMISSION_DIR
   submission_dir: CM_MLPERF_INFERENCE_SUBMISSION_DIR
+  version: CM_MLPERF_SUBMISSION_CHECKER_VERSION
   submitter: CM_MLPERF_SUBMITTER
 tags:
 - run

diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py
@@ -29,8 +29,11 @@ def preprocess(i):
         print(f"Cleaning {submission_processed}")
         shutil.rmtree(submission_processed)
 
+    version = env.get('CM_MLPERF_SUBMISSION_CHECKER_VERSION', '')
+    x_version = ' --version ' + version + ' ' if version != '' else ''
+
     CMD = env['CM_PYTHON_BIN'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission",
-                                                     "preprocess_submission.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --output '" + submission_processed + "'"
+                                                     "preprocess_submission.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --output '" + submission_processed + "'" + x_version
     env['CM_RUN_CMD'] = CMD
 
     return {'return': 0}