Merge pull request #579 from GATEOverflow/mlperf-inference

Improve docker detached mode error capture
mlcommons · Nov 21, 2024 · b770ea4 · b770ea4
2 parents e27bede + 1fcec02
commit b770ea4
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 4 deletions.
diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py
@@ -235,6 +235,11 @@ def preprocess(i):
                     env['CM_REQUIRE_SDXL_MODEL_DOWNLOAD'] = 'yes'
                     cmds.append(f"make download_model BENCHMARKS='{model_name}'")
                     break
+            if scenario.lower() == "singlestream":
+                ammo_model_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'models', 'SDXL', 'ammo_models', 'unetxl.int8', 'unet.onnx')
+                if not os.path.exists(ammo_model_path):
+                    env['CM_REQUIRE_SDXL_MODEL_DOWNLOAD'] = 'yes'
+                    cmds.append(f"make download_model BENCHMARKS='{model_name}'")
         else:
             return {'return':0}
 
@@ -252,7 +257,7 @@ def preprocess(i):
             cmds.append(f"make preprocess_data BENCHMARKS='{model_name}'")
 
     else:
-        scenario=env['CM_MLPERF_LOADGEN_SCENARIO'].lower()
+        scenario=scenario.lower()
 
         if env['CM_MLPERF_LOADGEN_MODE'] == "accuracy":
             test_mode = "AccuracyOnly"

diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py
@@ -214,8 +214,8 @@ def postprocess(i):
         if existing_container_id:
             CMD = f"ID={existing_container_id} && docker exec $ID bash -c '" + run_cmd + "'"
         else:
-            CONTAINER="docker run -dt "+ run_opts + " --rm " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + " bash"
-            CMD = "ID=`" + CONTAINER + "` && docker exec $ID bash -c '" + run_cmd + "'"
+            CONTAINER=f"docker run -dt {run_opts} --rm  {docker_image_repo}/{docker_image_name}:{docker_image_tag} bash"
+            CMD = f"ID=`{CONTAINER}` && docker exec $ID bash -c '{run_cmd}'"
 
             if False and str(env.get('CM_KEEP_DETACHED_CONTAINER', '')).lower() not in [ 'yes', "1", 'true' ]:
                 CMD +=  " && docker kill $ID >/dev/null"
@@ -232,7 +232,18 @@ def postprocess(i):
         record_script({'cmd':CMD, 'env': env})
 
         print ('')
-        docker_out = subprocess.check_output(CMD, shell=True).decode("utf-8")
+        # Execute the command
+        try:
+            result = subprocess.run(CMD, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+            print("Command Output:", result.stdout)
+        except subprocess.CalledProcessError as e:
+            print("Error Occurred!")
+            print(f"Command: {e.cmd}")
+            print(f"Return Code: {e.returncode}")
+            print(f"Error Output: {e.stderr}")
+            return {'return': 1, 'error': e.stderr}
+
+        docker_out = result.stdout
         #if docker_out != 0:
         #    return {'return': docker_out, 'error': 'docker run failed'}