From eac5a0a97dc8c3b3b56712dd926a492cc9e22080 Mon Sep 17 00:00:00 2001 From: sahilavaran Date: Mon, 23 Dec 2024 10:44:22 +0000 Subject: [PATCH] support --outdirname for ml models, partially fixed #63 --- script/get-ml-model-3d-unet-kits19/_cm.yaml | 8 ++++++++ script/get-ml-model-bert-large-squad/_cm.yaml | 4 ++++ script/get-ml-model-dlrm-terabyte/_cm.yaml | 2 ++ script/get-ml-model-gptj/_cm.yaml | 2 ++ script/get-ml-model-huggingface-zoo/_cm.yaml | 2 ++ script/get-ml-model-llama2/_cm.yaml | 2 ++ script/get-ml-model-mixtral/_cm.yaml | 2 ++ script/get-ml-model-retinanet-nvidia/_cm.yaml | 2 ++ script/get-ml-model-retinanet/_cm.yaml | 4 ++++ script/get-ml-model-stable-diffusion/_cm.yaml | 2 ++ script/get-ml-model-tiny-resnet/_cm.yaml | 2 ++ 11 files changed, 32 insertions(+) diff --git a/script/get-ml-model-3d-unet-kits19/_cm.yaml b/script/get-ml-model-3d-unet-kits19/_cm.yaml index 7dc7f5b06..658f306a7 100644 --- a/script/get-ml-model-3d-unet-kits19/_cm.yaml +++ b/script/get-ml-model-3d-unet-kits19/_cm.yaml @@ -43,6 +43,8 @@ variations: extra_cache_tags: 3d-unet,medical-imaging force-cache: true tags: download,file,download-file,_wget + force_env_keys: + - CM_OUTDIRNAME env: CM_ML_MODEL_ACCURACY: '0.86170' CM_ML_MODEL_FILE: 3dunet_kits19_128x128x128_dynbatch.onnx @@ -60,6 +62,8 @@ variations: extra_cache_tags: 3d-unet,medical-imaging force-cache: true tags: download,file,download-file,_wget + force_env_keys: + - CM_OUTDIRNAME env: CM_ML_MODEL_ACCURACY: '0.86170' CM_ML_MODEL_FILE: 3dunet_kits19_pytorch.ptc @@ -73,6 +77,8 @@ variations: extra_cache_tags: 3d-unet,medical-imaging force-cache: true tags: download-and-extract,_wget,_extract + force_env_keys: + - CM_OUTDIRNAME env: CM_ML_MODEL_ACCURACY: '0.86170' CM_ML_MODEL_FILE: 3dunet_kits19_pytorch_checkpoint.pth @@ -92,6 +98,8 @@ variations: extra_cache_tags: 3d-unet,medical-imaging force-cache: true tags: download-and-extract,_wget,_extract + force_env_keys: + - CM_OUTDIRNAME env: CM_ML_MODEL_ACCURACY: '0.86170' CM_ML_MODEL_FILE: 3dunet_kits19_128x128x128.tf diff --git a/script/get-ml-model-bert-large-squad/_cm.yaml b/script/get-ml-model-bert-large-squad/_cm.yaml index e5b4d11bb..e81819279 100644 --- a/script/get-ml-model-bert-large-squad/_cm.yaml +++ b/script/get-ml-model-bert-large-squad/_cm.yaml @@ -13,6 +13,8 @@ new_env_keys: - CM_ML_MODEL* post_deps: - tags: get,dataset-aux,squad-vocab + force_env_keys: + - CM_OUTDIRNAME prehook_deps: - env: CM_DOWNLOAD_FINAL_ENV_NAME: CM_ML_MODEL_FILE_WITH_PATH @@ -25,6 +27,8 @@ prehook_deps: CM_ML_MODEL_BERT_PACKED: - 'yes' tags: download-and-extract + force_env_keys: + - CM_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - CM_PACKAGE_URL diff --git a/script/get-ml-model-dlrm-terabyte/_cm.yaml b/script/get-ml-model-dlrm-terabyte/_cm.yaml index 6227a9a17..4d5c93f1b 100644 --- a/script/get-ml-model-dlrm-terabyte/_cm.yaml +++ b/script/get-ml-model-dlrm-terabyte/_cm.yaml @@ -25,6 +25,8 @@ prehook_deps: names: - dae tags: download-and-extract + force_env_keys: + - CM_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - CM_PACKAGE_URL diff --git a/script/get-ml-model-gptj/_cm.yaml b/script/get-ml-model-gptj/_cm.yaml index 25e8deca4..7d645976a 100644 --- a/script/get-ml-model-gptj/_cm.yaml +++ b/script/get-ml-model-gptj/_cm.yaml @@ -29,6 +29,8 @@ prehook_deps: names: - dae tags: download-and-extract + force_env_keys: + - CM_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - CM_DOWNLOAD_URL diff --git a/script/get-ml-model-huggingface-zoo/_cm.yaml b/script/get-ml-model-huggingface-zoo/_cm.yaml index d10c3f448..154675bc1 100644 --- a/script/get-ml-model-huggingface-zoo/_cm.yaml +++ b/script/get-ml-model-huggingface-zoo/_cm.yaml @@ -38,6 +38,8 @@ variations: - env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ML_MODEL_PATH tags: get,git,repo,_lfs + force_env_keys: + - CM_OUTDIRNAME update_tags_from_env_with_prefix: _repo.https://huggingface.co/: - CM_MODEL_ZOO_STUB diff --git a/script/get-ml-model-llama2/_cm.yaml b/script/get-ml-model-llama2/_cm.yaml index 2ff45866b..fe082718e 100644 --- a/script/get-ml-model-llama2/_cm.yaml +++ b/script/get-ml-model-llama2/_cm.yaml @@ -26,6 +26,8 @@ prehook_deps: names: - hf-zoo tags: get,ml-model,huggingface,zoo,_clone-repo + force_env_keys: + - CM_OUTDIRNAME print_env_at_the_end: LLAMA2_CHECKPOINT_PATH: LLAMA2 checkpoint path tags: diff --git a/script/get-ml-model-mixtral/_cm.yaml b/script/get-ml-model-mixtral/_cm.yaml index 2542d4dc7..5b61ef9ca 100644 --- a/script/get-ml-model-mixtral/_cm.yaml +++ b/script/get-ml-model-mixtral/_cm.yaml @@ -22,6 +22,8 @@ prehook_deps: names: - hf-zoo tags: get,ml-model,huggingface,zoo,_clone-repo + force_env_keys: + - CM_OUTDIRNAME print_env_at_the_end: MIXTRAL_CHECKPOINT_PATH: MIXTRAL checkpoint path tags: diff --git a/script/get-ml-model-retinanet-nvidia/_cm.yaml b/script/get-ml-model-retinanet-nvidia/_cm.yaml index 4e114e43d..7f6880322 100644 --- a/script/get-ml-model-retinanet-nvidia/_cm.yaml +++ b/script/get-ml-model-retinanet-nvidia/_cm.yaml @@ -14,6 +14,8 @@ deps: - tags: get,mlperf,training,src,_nvidia-retinanet - tags: get,mlperf,inference,src - tags: get,ml-model,retinanet,_pytorch,_fp32,_weights + force_env_keys: + - CM_OUTDIRNAME - enable_if_env: CM_TORCH_DEVICE: cpu tags: get,generic-python-lib,_torch diff --git a/script/get-ml-model-retinanet/_cm.yaml b/script/get-ml-model-retinanet/_cm.yaml index 90e937000..8da05da0e 100644 --- a/script/get-ml-model-retinanet/_cm.yaml +++ b/script/get-ml-model-retinanet/_cm.yaml @@ -28,6 +28,8 @@ prehook_deps: CM_TMP_ML_MODEL_RETINANET_NO_NMS: - 'yes' tags: download-and-extract + force_env_keys: + - CM_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - CM_PACKAGE_URL @@ -80,6 +82,8 @@ variations: extra_cache_tags: retinanet,training,patch,file force_cache: true tags: download,file,_url.https://raw.githubusercontent.com/arjunsuresh/ck-qaic/main/package/model-onnx-mlperf-retinanet-no-nms/remove-nms-and-extract-priors.patch + force_env_keys: + - CM_OUTDIRNAME - env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_MLPERF_TRAINING_REPO_PATCHED_PATH CM_GIT_PATCH_FILEPATHS: <<>> diff --git a/script/get-ml-model-stable-diffusion/_cm.yaml b/script/get-ml-model-stable-diffusion/_cm.yaml index b2326daff..ae9ee2757 100644 --- a/script/get-ml-model-stable-diffusion/_cm.yaml +++ b/script/get-ml-model-stable-diffusion/_cm.yaml @@ -28,6 +28,8 @@ prehook_deps: names: - hf-zoo tags: get,ml-model,huggingface,zoo,_clone-repo,_model-stub.stabilityai/stable-diffusion-xl-base-1.0 + force_env_keys: + - CM_OUTDIRNAME - enable_if_env: CM_DOWNLOAD_TOOL: - rclone diff --git a/script/get-ml-model-tiny-resnet/_cm.yaml b/script/get-ml-model-tiny-resnet/_cm.yaml index 4f8406e29..791ecccee 100644 --- a/script/get-ml-model-tiny-resnet/_cm.yaml +++ b/script/get-ml-model-tiny-resnet/_cm.yaml @@ -22,6 +22,8 @@ prehook_deps: env: CM_EXTRACT_EXTRACTED_FILENAME: <<>> tags: download-and-extract + force_env_keys: + - CM_OUTDIRNAME update_tags_from_env_with_prefix: _url.: - CM_PACKAGE_URL