From 22de184da0167aceb73113d0167b4beb62539270 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 26 Sep 2024 12:07:12 +0530 Subject: [PATCH 001/658] gh action self hosted - dlrm --- .../workflows/test-mlperf-inference-dlrm.yml | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-dlrm.yml diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml new file mode 100644 index 0000000000..bb4f4f15bb --- /dev/null +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -0,0 +1,85 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference GPT-J + +on: + pull_request: + branches: [ "main1", "dev1" ] + paths: + - '.github/workflows/test-mlperf-inference-gptj.yml' + - '**' + - '!**.md' + +jobs: + build_reference: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + backend: [ "pytorch" ] + device: [ "cpu", "cuda" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Test MLPerf Inference DLRM-v2 reference implementation + run: | + cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=0.01 --rerun + + build_nvidia: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + backend: [ "tensorrt" ] + device: [ "cuda" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Test MLPerf Inference DLRM-v2 NVIDIA implementation + run: | + cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 + + build_intel: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + backend: [ "pytorch" ] + device: [ "cpu" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Test MLPerf Inference DLRM-v2 INTEL implementation + run: | + cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 \ No newline at end of file From cb5e446c16e72a84fb5e8d04827993974ac2ffb6 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 26 Sep 2024 12:16:05 +0530 Subject: [PATCH 002/658] added clean tag --- .github/workflows/test-mlperf-inference-dlrm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index bb4f4f15bb..6a6d00571f 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -34,7 +34,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference DLRM-v2 reference implementation run: | - cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=0.01 --rerun + cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --clean build_nvidia: @@ -58,7 +58,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference DLRM-v2 NVIDIA implementation run: | - cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 + cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --clean build_intel: @@ -82,4 +82,4 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference DLRM-v2 INTEL implementation run: | - cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 \ No newline at end of file + cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --clean \ No newline at end of file From ee55ade430d3c6c8bc6cfea301d659eab53bc908 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 26 Sep 2024 12:30:02 +0530 Subject: [PATCH 003/658] initial commit for self hosted mixtral gha --- .../test-mlperf-inference-mixtral.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-mixtral.yml diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml new file mode 100644 index 0000000000..a5c3f452e8 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MLPerf inference MIXTRAL-8x7B + +on: + schedule: + - cron: "1 2 * * *" + +jobs: + build_reference: + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64 ] + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + backend: [ "pytorch" ] + device: [ "cpu", "cuda" ] + + steps: + - name: Install dependencies + run: | + source gh_action/bin/deactivate || python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation + run: | + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean From 27ce632c0852c38da12427ec5b62965387a76ab4 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 26 Sep 2024 12:43:47 +0530 Subject: [PATCH 004/658] added submission necessary tags --- .../workflows/test-mlperf-inference-dlrm.yml | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index 6a6d00571f..c5809f632e 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -1,20 +1,16 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: MLPerf inference GPT-J +name: MLPerf inference DLRM-v2 on: - pull_request: - branches: [ "main1", "dev1" ] - paths: - - '.github/workflows/test-mlperf-inference-gptj.yml' - - '**' - - '!**.md' + schedule: + - cron: "1 2 * * *" jobs: build_reference: - - runs-on: ubuntu-latest + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64 ] strategy: fail-fast: false matrix: @@ -23,22 +19,19 @@ jobs: device: [ "cpu", "cuda" ] steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install cmind + source gh_action/bin/deactivate || python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference DLRM-v2 reference implementation run: | - cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean build_nvidia: - - runs-on: ubuntu-latest + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64 ] strategy: fail-fast: false matrix: @@ -58,11 +51,11 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference DLRM-v2 NVIDIA implementation run: | - cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean build_intel: - - runs-on: ubuntu-latest + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64 ] strategy: fail-fast: false matrix: @@ -82,4 +75,4 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference DLRM-v2 INTEL implementation run: | - cm run script --tags=run-mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --clean \ No newline at end of file + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean \ No newline at end of file From 51346cf0ae9969e5cc928e67b2775c190282d9df Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:17:05 +0530 Subject: [PATCH 005/658] Updated device tag and cron job schedule --- .github/workflows/test-mlperf-inference-mixtral.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index a5c3f452e8..9de8f2f70c 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -5,12 +5,12 @@ name: MLPerf inference MIXTRAL-8x7B on: schedule: - - cron: "1 2 * * *" + - cron: "30 20 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST jobs: build_reference: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64 ] + runs-on: [ self-hosted, GO-i9, linux, x64 ] strategy: fail-fast: false matrix: @@ -24,7 +24,8 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + python3 -m pip install cm4mlops + cm pull repo - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean From e609dcfd34993616378f266a194065c10c591af1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:28:51 +0530 Subject: [PATCH 006/658] Rescheduled cron job + modified system tags --- .../workflows/test-mlperf-inference-dlrm.yml | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index c5809f632e..27025acde6 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -5,12 +5,12 @@ name: MLPerf inference DLRM-v2 on: schedule: - - cron: "1 2 * * *" + - cron: "30 21 * * *" jobs: build_reference: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64 ] + runs-on: [ self-hosted, GO-i9, linux, x64 ] strategy: fail-fast: false matrix: @@ -24,14 +24,15 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + python3 -m pip install cm4mlops + cm pull repo - name: Test MLPerf Inference DLRM-v2 reference implementation run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean build_nvidia: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64 ] + runs-on: [ self-hosted, GO-i9, linux, x64 ] strategy: fail-fast: false matrix: @@ -40,22 +41,20 @@ jobs: device: [ "cuda" ] steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + source gh_action/bin/deactivate || python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM + python3 -m pip install cm4mlops + cm pull repo - name: Test MLPerf Inference DLRM-v2 NVIDIA implementation run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean build_intel: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64 ] + runs-on: [ self-hosted, GO-i9, linux, x64 ] strategy: fail-fast: false matrix: @@ -64,15 +63,13 @@ jobs: device: [ "cpu" ] steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + source gh_action/bin/deactivate || python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM + python3 -m pip install cm4mlops + cm pull repo - name: Test MLPerf Inference DLRM-v2 INTEL implementation run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean \ No newline at end of file + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean From 1fe3afb553799fef3ce18599da5a4d0f120e20d4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Oct 2024 23:22:33 +0530 Subject: [PATCH 007/658] Added run.bat for push-mlperf-inference-results-to-github --- script/authenticate-github-cli/run.sh | 4 ++-- .../run.bat | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 script/push-mlperf-inference-results-to-github/run.bat diff --git a/script/authenticate-github-cli/run.sh b/script/authenticate-github-cli/run.sh index b17d52e4b3..58c52dad6b 100644 --- a/script/authenticate-github-cli/run.sh +++ b/script/authenticate-github-cli/run.sh @@ -7,8 +7,8 @@ #${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency -echo "Running: " -echo "${CM_RUN_CMD}" +echo "Running gh auth: " #Not printing as it can contain secret +#echo "${CM_RUN_CMD}" echo "" if [[ ${CM_FAKE_RUN} != "yes" ]]; then diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat new file mode 100644 index 0000000000..fa677e026d --- /dev/null +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -0,0 +1,20 @@ +@echo off + +cd "%CM_GIT_CHECKOUT_PATH%" +git pull +git add * + +REM Check if the CM_MLPERF_INFERENCE_SUBMISSION_DIR variable is set +if defined CM_MLPERF_INFERENCE_SUBMISSION_DIR ( + robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_CHECKOUT_PATH%" /MIR + git add * +) + +REM Check if the previous command was successful +if %errorlevel% neq 0 exit /b %errorlevel% + +git commit -a -m "%CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE%" +git push + +REM Check if the previous command was successful +if %errorlevel% neq 0 exit /b %errorlevel% From 9f7a911b619b3f181109c49715b846877d0069e5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Oct 2024 23:41:51 +0530 Subject: [PATCH 008/658] Added windows installation for gh --- script/install-github-cli/run.bat | 1 + 1 file changed, 1 insertion(+) create mode 100644 script/install-github-cli/run.bat diff --git a/script/install-github-cli/run.bat b/script/install-github-cli/run.bat new file mode 100644 index 0000000000..2ec8b97186 --- /dev/null +++ b/script/install-github-cli/run.bat @@ -0,0 +1 @@ +choco install gh From 2073132ce13a431a1ef2360b446914e48ccb7699 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Oct 2024 19:12:48 +0100 Subject: [PATCH 009/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 5867abbf68..f45e962131 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -47,14 +47,17 @@ jobs: if: matrix.os != 'windows-latest' run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + - name: Install GitHub CLI + if: matrix.os == 'windows-latest-off' + run: choco install gh # for Chocolatey (Windows package manager) - name: Push Results if: github.repository_owner == 'gateoverflow' env: USER: "GitHub Action" EMAIL: "admin@gateoverflow.com" run: | - git config --global user.name "$USER" - git config --global user.email "$EMAIL" + git config --global user.name "${{ env.USER }}" + git config --global user.email "${{ env.EMAIL }}" git config --global credential.https://github.com.helper "" git config --global credential.https://github.com.helper "!gh auth git-credential" git config --global credential.https://gist.github.com.helper "" From 4e924dada2a8c7d7240a6a22c17ef6af34347b76 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 4 Oct 2024 18:21:13 +0000 Subject: [PATCH 010/658] Results from R50 GH action --- cm-run-script-versions.json | 602 +++++++++++++++++++++++++++++ hardware/gh_windows.json | 26 ++ mlperf-inference-submission.tar.gz | Bin 0 -> 19288 bytes summary.csv | 2 + summary.json | 32 ++ tmp-run.bat | 36 ++ tmp-state.json | 3 + version_info.json | 602 +++++++++++++++++++++++++++++ 8 files changed, 1303 insertions(+) create mode 100644 cm-run-script-versions.json create mode 100644 hardware/gh_windows.json create mode 100644 mlperf-inference-submission.tar.gz create mode 100644 summary.csv create mode 100644 summary.json create mode 100644 tmp-run.bat create mode 100644 tmp-state.json create mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json new file mode 100644 index 0000000000..08ddbd1975 --- /dev/null +++ b/cm-run-script-versions.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "onnx,fp32", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_resnet50,_onnxruntime,_cpu,_offline,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "resnet50,onnxruntime,cpu,offline,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json new file mode 100644 index 0000000000..b7cf960dbb --- /dev/null +++ b/hardware/gh_windows.json @@ -0,0 +1,26 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "host_memory_configuration": "undefined", + "host_networking": "Gig Ethernet", + "host_network_card_count": "1", + "host_networking_topology": "N/A", + "host_processor_interconnect": "", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "status": "available", + "submitter": "cTuning", + "sw_notes": "Automated by MLCommons CM", + "system_type": "edge", + "system_type_detail": "edge server" +} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..252ae3709cc23b28380a6560cbd92ddd57b4229e GIT binary patch literal 19288 zcmcedV{oUz^WbCi8(ZJl+SsaQ4|cwIWvek2>6w+r!(Goec7k3*eQ?8Z2b%&BaExA?o#yZ^@3s|U;TXa za^enr?NP{kImKQ-`)Md}y6f#aYYj18&=dR_+9 z&Y5TY(y?7~bg%(mo-&STa}F)EV41tG_qNS1i&56dbEh-$^^;Md?BLEtd0PkYeA1grbNTwkmCN^< zC+F-IppZ!?;UXJ6)?&wB!=UlvUAW{rD?RbRDE z+-vnnePoe^^#8CD2L1Yzy-+qYDLZ+49Y4SBPYqGhjh3PT&3C7k*sWsQF0{@}Ly|ha%z3l{8z! z+P(*OhEFyl#gc;!^o?fTu9dd-lt9>0H0;%e>+*^9Ph}bMv5gOVRn(pq6K-%5 zpmn}X0r$7QZL;xlWn>PScV#A=`k(2@JIE8{n&qJrc~M;PcZS?w^`9ToTY)7V^hrxL z76Hc0psx7xHruJwVnyuC9)22p$=ov}@Qblhnlg!|NuO-I@VP%v+>iuX%hnApFuGiS zulZ!IH9!P60(tBme8=zeKivQj-Mi#?HGT&d`a?%p3K=#Ptvn($kku2zumI3}&v>_S?ML-Jc+0Kk@dtHKSF^9kmuyQZJeV@e5r* zMl*9kc0NuT$aa|+{ESa-uKuNmeALZ|kWAfWt6~Z9=u~hPv1aYaW)TAG*W&Hhc zb_hiA9eGz`L@QgC~okK*Mmrd z4O;lw@4b!n(bk(7S<0S<(0;!CW2bhv@hzjo9;&_Ce^xCpzp9_}(lK##v{&p);bXA- zLx2R5=kfI`;PETZ|Em&YpoI8x#;=5I`R>b^H!tyPp%A2Ylz%(#ODnYS;OoSLA9k>x z2#S)mQ$x$Y`;8H2;r#P{2eyj$HF1V=PoTwzn}|{oDzXppwN_;9zlMmuiFfbA1+NqM zYRBBO5B;i?f9X&2aN9LpNCZXLfujBUAdEtu!WWJVP{e7`s_lt}i-Tf(#dhP7%WcDj5Dd{Jp{^xDkZdwtMErCA1 zT++Wud8nJKu2}h(z@nFN3#HLv69I%zzSmFcPYo5;5hk5guIEsJ zNoeuetOyu`-tXJ;zbBYBE{J;GKd*K>Z=mgay`S*)jCnzcL>h%5ncPlosv%{CiqJ z6zIZ?e*{iE7WimBZUGKY=;S#$aG=;@dK@f9sq=^fPsV@@Pv8oDO85Ix&j@V`J*$z# z>evIp4<#Sb0ZGpH(N?fe4}4vf!aNL7*ar_KGb8rhUu4KG5njbDpAUjNcg6!N;gwuwZJ z3~T`<^K7X~BaKXQDj4IF$Gp_(DI=Jk*b)_z2*)x7K_L`rag9POct-E?vHARi_rMl9 zBOH$3Od%;sxs(ynw#g;1jQgZ!wr?{e-ji^q^E#sysMIT^C_4LqXP`VBy=ww&68fzr z*?UzkdS@XCdPT60I4i1zXCgcrxn^-c$Df7CAyP!+pkM=%9~p5B4u5JjBi0UQoN{Pz z$OnenJeo#NM&r2%Z071IaLe^r7$sIX#H(h(`aLm8mT8zZv#~8v-kR!oQyZ9~X;Y;(4lC6h)^;^ENWoK?ZL9u- zNU{lHos-T{$C34+^ho^BoC(J`_QVaMB5T7*qJexi&0xm~J2n*yk4Ao=tESD?ojhLf zTdO+%xo`)=Vh+2I877vE!$4+&WdR@VhZJZO0~0lw4kij<<#;_7sROFH@na({c(U|b zq%$Pq-%BDrQ^*LtK2SkvsG#U70U;$_plC{#)R;|Ka${_Quy-q2(U@~fJu$5rgj1~0 zuztkQso=l~C$j?@b3KE=$l4@1PiUI`#X!3dW2W*KRl!$8?F?N|GS~)%Q}vv{J2817 zVbq3%rrROd!H>p4|3G*^-RB-bC6_Z}R7SI3h6eeY1O`=%f&S)AiN=T?AHzoaiv-fg z=%{OGSvg#lhhabz7s&=GUrCxKOVO8$Z=(bVgS(VP4G5c9gxM7JYMi8+hqmaBd%NPXW z(C@V*FbJc_V+80FNTue7;55Kj31g0f#e{Ka- z(OAkAt0Q2Se#rqau%~7DvTlzTLR^Su`eg;FBre(YF&pBTw_mSc0OZIb13yEi<&F|W zChVw}&>|hxsreLFpq_r$+r@^6_1|r~#|%j02AfY}eFj~;U88(jD%^1ad~95=m%zO; zr$q|Tual{XB5-T&PN{_xw;wq_e-NoI(BROoJX8x6AjyNhRvT!*&u>FBJtNI7U9iiD z7EWFQGmy*hBJV`dCMbTRGJgyk-gau@l{*k@<0p3@=>&FIGSmbghmD`^*UcGnNSJdi zFxL^8GnEDs{mQg;p}h1DKsEGnHW(QkSHt4j?gR3g$ga{~PH?_?2rw(PCMBt@9{=q* z$ybc9qojU8$MAJlwQ7ZIMbp@Yq?`SIgI;aW+q7xFaDYo7^smnB<WDiVe8zz~gm6@e^;tnqgXy~@1%vh^EY1AvJa-pD|6B)O zLeDZcxv54~cHd%kw@ULfb00b>(DV zGDAB{mX~ix{rT&E$KUS+-+J^zMI{U(;d;wS!)2qQLImsr^K z)p>mUo_p}*^nV)0c9A(eXJ&E({q5!ot_0X!CHpoD@jBlY7fC?TcgO`h<>UxZcoVnS zc{^q02mpfnb|CjLY+B~UNU(X`5(ZDlHHOO_H}8m#+8V}nP`Y)Qxobm`6}4_xJqaA^ z$UQn(8g|bce&&Hs8g$+W`DUR{oEPKRpJTbv8VY@k(f!&?F0xsA#oo8V5ronFUZm)J zP-Fzz0V7xJ^P;!%Pn?fs!a@`dmyWh<2VKxdV~Qf0P?0BmsCR_$h$h2@q~%4GXG8KIT#VsjYBrzB+G)dfN{Z^6vh{3gDu6Qyl}um zR^;Q_XhbK15BTQ^;VL~?=|Y1U(G`VRN3+wQB|<(~nxL_uA3?ARS}RW0)4jqjrh#h! zGzaH|LB=FeBq716#fpXWIFn=S1%1ekklBbIsSs`}6Zohw!cF=pLJ^?@0^LORVIG8N z6+;z^fy$tokSzo~&nUYnB{(P{4V>YDY~_3w_3?L9gdNa}jVSD}THuj`iV^uCh2l8< zp}oZMiWnllXA5Dg0OMiG3AXvxfJWg7gg?ah@Y0|x2$@1aq`4q4#c5D8Hs<_GFd!mc zgcY2mASqHGj0G$j)h7&u$%wlJZak9(rZlgXrVSJd*nQX|lPJ=dxUUek=vx>^-|)Ju zFzPHyyqF?$#Co(7)Bu^%Dl<_rU~EiFbuPTXz%YaO?KaF%gcv82HdGxvF`usAoLG2- zibE0XUY<;4Uyr^|5;PL*5BEA85dDZ?9CTN>iOYf{fPonIvOqjM02rVS0S5_Sw*qIw zPPbOroObR&;M0RLGuZl;=xBVc4=DNk6tMl=^{C9$GD{N`!z`whOqvv{0l7X0d!3uq3 zp_Ou5ZPmyV>oQz#S-pScxt#}R3<~Vjuh|5zEQ2*c!8sA1Oh>Z@) z251ro&&OYp-XiN);utEyO_@kybT}uI`6HBoAn^hQ@U@n-s+PtQGdwY^R(p{&c>Xca zcm@qZ*{rxN4y~V;9?an}p3n6o;-G?fzZ?UA_N)0uaea~x4>8NNH3k7OF~{me z=*b=;NTlKQBt6l`=t9~KpEaaV6l1Ta$rVf=A_^(S7E18caB9}wsZ~!MbeJgmHdw^b z`Qbb0(C33M-e}=AGRiBxkcIR%6Ka?_!K9q4Q7hFvC^5VyB)`|}ZapR`x;Q>5{(h;$>JTk~^eC=9yUOwH zZ9K=2$XgiS;m&v=gvCTeG z7X9i_ucHm2i^4?X&WHVx5${MR-g76&b>=$}|xkb&_u9BqW(0SSb8_buDW`1Mu>x`yd6`E zG!4wkzh#>VlaynD+g6oQ+b%{Z_OhG{5|I644|bwSlqYi{scQQ_w8! zmY*io_S1-3FNZbjhnMn*>l`J*R_@+rvpuvAbA!tt5kgxKJ5b%JtYuIu(olt;GBVTLr*~>A(ihsXw={~Ai+H}G}=%J7b6Dn^=wMr{>n{6uyU1o&% z+`Ycur9T-3*kbpaU;JdQUj2k&pR*pm2z?^Vhbt(#18W%n-E{BTa8Kp!5+>y9>E9Lq z7)9py#RiD{aQ@V{ekko^%%B-@lf%$jAcGlCpPNPF%<(vZk^rd~jP&Ttv(C;F_tqX4XxYPzmP8;+s#L zffCEkA$Y}bac-OqLEh>W2u&5x&cPDF1GXj0SFYUqgPau)mYti-KYyLG%KgZA>&hXI z1z}I}s&Q&onjiojAn1~pe<*C0pVq0NnxA$nIEn>v6gofb;432&5=O;g>EbJMX$N$& znD8#r`~(>N1ROrN*^sr$4=9wZJ9D#`ghTFMwIuS80!6NVsEl5z0q{)1wI-gC`Y{U9 zlcpz<7Eno0QT_AhTioDU-{BDJ=dNT?YhYl7oV>XCTfrAZZSq=Q;Dy_LDeC8|x{Xon zh$zAn`Qo$?(;cSbNpn!W=k&^GMd-HYEZ`(UboCmuUDbuGm#m$4B}hL-m}ccT5ekXa_pkMx~2fJb>p+(8!k1R; z%zB320z~aBqF!(17PTo4Za5T2|w%scm&M4Y1{)a`Wf5b*CdCfd~>jbgPM$*w`@+_%oJh&v) zx)GFDhgXAYS7Rvd&`$ZeR}?~>ETYhAGwUB(A|ns;OA2$>$=E+bEqsoe2(%9}N#u)X zn((v_#uVGE`F{plEbeP|y5YJrVJ7vb+Yz1Ox(gDa<$6FFk0j)Wdo=&vBF1jTN`hD| zyZXp;^kx@Qqx%%aI z8Lr6ZypQm61B2Yn*OJ?X?P@M;)wR?ZfR=bQfYm8Z1*CZJ^v1&euBq3c-ix{!{S1vJ z;o3M#4JnXeKz+k1pd^$k6gE*M*YmITM9OIk=E zlXLnz;8U=X5dy?~lJ%x>;N^tR32U?49j$PKeLojf zDrlucVTLM@_2=v7?!pv^bt*+gzuN7W`(0}-QKr!7On|ENOEr%j^;S}3(+yj;26lB? z(kzEay-y~t)c`UZJ`BRZEl1qBC3|0x&I7h{Xs^QQ$(Wx}@om-o?CxaCk&*@Wp(8OG z&fLu9q%)$)8~v68xObaLONgAf^C_%xwvIu|bJMb}LCetBP_!03ZO!}CMg+{3 z3mIp19^3S23Aa#3mbH`A9@kY%32v%D!}o+3h_RGeucuKU!b-42-Bx5d>5TF zJ|f-3XbWiav0}0V=3V+ zHo{|8IpDG!jqf>5UT3`>VFKYiD^}m80Y{lNdKCZ8Tdp4)Wu}zN+>&GTfx!#zVfX~cOmOCpYyDT(6 z);ucQCS=bI<+B$dIs{ae3S#J~GC_pRS+;p;Zo_+umAonEl{wkWZ_Sqx;$0-og!jTx)}}{8 z2P@SgL%ls8M5minJS>rZwcV~Zld!{~Ci|bUK*0I4N9t%|Un@!T-{s#O0rn(PWpp8B z>Ks})XmW8I5^C9~mW8E{Az=nA1u}n~ch0pq9;eO<8LubKLwmSLR?)C`?M}0lN*-jv zoH(AR%72FuZipJ~xNJ*!6-Ny9RN!DvCZ#+&n^-12_WegL18-+!KXv}>j4GsIrBEHq zhGXH<36`8-(Zr(t(BV+oL#k6TJKiGtEn3$dP2u%UEcn!{Rk#r z-|O+S8r1y89Da{+6MR?>uVp0%DFnb)nZvhj*$|PmOrpIV*sW^MzFHij`iNo5&ohUP zijW!Fcd+bblqvsAKK(1PBahSZYvTznWYmflpk27#Qx>Fbf(KMmo4L(Q!q{XR!5Pl2 zs-dF>*qW3P29D#}wXpWZ(#`EHIxFZ4VLFbw2xW}6e4B}QkbABrBq6FMDM8F_{-Jnx z28GqZmhQ25cf_3EH7X3pDLrJwu@}@6FRAtJDJlI-{1~DaM}1a|R`Y1vcmMPHR~=!F zV|t5Puk!?Qj^k>}v5ke5*{4mLpYfaVt>}v<&qLs2k#-#j4;f*VnN@*gI*5GT?G$)2 zZaFP`CQ9y6m)=#acn@ZwZmPN4I%u7uAkS*&f99uoyty=2{&{eMcqMYz->7Z`<|&g;5PmEvdkQz4eGaq zKB}zpyQCi=_`np}-!k!xw&&=@yDsfq40Pt{IcxD3eB>{lB zmq)t+L^pV=X+00l1E@|yltJCQUU@8oQTM+&{D+3#uAiWWWO!s7Lh&;1yR z{$tOlxpf4Jn(=i@7lL9}m(_7O;(i%=;T|SB#ivBL2h|Bl*_F^!Rl_8OyJJJ756`hW zb^168ANQEFAax2di&ljiSjA2f`Ovi$hhb9DN@1)!!(c=*mdm+Ncy^_^rIfw?k= z)rG;gA?o1-Ff2UbgF){8-oTJNZ*DFj3fGElkA_5J>C;~>I~g><)9Nw)9!1ZF#4TB; zpWc%4GzHLMFvtUsb%6PU*JAxddMX}wJiFDbb^AS33MW0-Em%_{1(hvJ&W`wn3A`1S z=F1g3B3c-ZbK6t8?2}{yCo|Y9ZN3G*K+*0qy54&aV!Or8qj2H4%On+<)7ec{5|Xic za+iXtI|SCva*m=(gp5%rT0lj#8-{_H4n+qY3p%4#BX@)7(Tejtn&z1tmcQ3?V0ids z-Ymnxq6%%sT#UPG{#HH*$B^v{{s;x>zW0J)rg^rIJQJ#kT%bL%M5>ocAYXwRYC?Qp z&FEo7__wkqQ3?4dWxCHSp*IJAa$L%#v3mTVgm0MdJi-Twb95ZsPTnjdj7zS0EKLd{ z^x68J7OQG5yOwa$8Yj21UEK2b>b^aNsK;?eFy2$?yQI$n!uKgI$GdFYHYwmhO!o$> z45WKhw>{ZzHYe9U(9wOO`;BUn!@(CfeuA6my6sZ9=uX7|>$R}frV=-h;JWz zj+X-m;*aodLu;PMVe#g8ZGNBj?uQ8Uso1OLZ2GK>b#ppM&Eu^G_homAf(!aNrMWX_ z1@-UCiVJN0Ahv_qUh2btm9>r%zPn=JZ6MTcRvQL4Ju3r1?1BVvp$+#ERtVVnlk3$wmf-JJuCt9}H{AQ5;uHTGCV z=|(TL+DRV1$HmP8?97l<(f7F+msa;ebA77HO9I|khky0^FeQL@p4udwOiBaZN+GO~ z`|S?1yub|KnM4x%ICRhcR^KeDH0%e@Uhr6@Fg%trJ0FjqyE(ZiuP)HaI00JMU3aO+ zQ7KK@Qz-j@&3riHr#UW3HymQHi=VN?i zI&j~<@7V8E<))o*C5t6Cwrh*$e*69kj-2S{Y^^Kto-<9I+=uXD;xc7Gz^x7o=~f>{ z*+q@Q24!do-fJmyK-pE;m?Yo@6GDH+;iq)NzUU=QpWUYRTuDMtYT_&4PN%r`Kkmm0 zW9GL2_4p%02H%T~cCy`O)^+#y^{+2Ajp_2Cq>!#h)1aTGgSpEGo`e4+&s*K*+gs!^ zxb2d>$CW9Uh$erEIyJ+42*_%>FvB?$@8;smjPdk17w=)b{V^Zo zh36IRf9DWdRZ;xVm|K%j^>pl_)o=Vbp0V|~4I2k9i^Lh>)j9kD)I6&`g+h;;xQ6Ps zbXX+o1|G`&4E0RCk@2$P;2R%a5KCB))57%f&K7Q%&06=F;cape*>+JXK1tV;BHfNo zXPv3|n^^T2D_K3WUPcDw>d0bffA&8N?Kx&up82L>c|-+`x`2ndfJS0wcwYCziaEJ+e-QE z4ESK~76)+wcgI6Y+)v;Bwu{W`C{$ucvQPWQ3%weMYQJ9Lbw(ZwfYp255BVNRk^Uw7 zVu8ugSVJD6F08^Og<}i%M#+Uqp{^IaDGQeqwOc)TUrx~kW|Yh3uK zZLdz;D{r5CgHmXF`us`G6_x$2pYrXczb;PNzMc*b50@s=1pZOK9-T^fJ3{>7$)tG8 z$U)fN-~xCa)F|9 zn}}myJse6QAyLT2eu1pdjcK-Cg~JgZhIk)PWs)4~k+U zgxFxx6aWeR^DaGPw7_R#5;jV{?UXgNGWP#UWL*C*L`Hs81?G`&Ujd=mHVibKr(!Op zP4`|Maf|rAsX7ilWU8GfQa_Gp0P1nP_9^@0nti6R;=KgrVW(7>LY+OnwYK z=im$Zh8{`TXb!PBn8K@Q;2e1-j;)GoX5NxA7%iuM;Wvy{DN;gP;2{{}lOoVG1cf1Jj zmY{Q_<{+Uo%7i0Qq&79t4J=36e=10hNaLLIdlc*I2h4;IxeaGSfW(;VETwHN86!ri zV9Sfh*JS4k{|7L`&_%7vdZ5&?B8tPA8ma)iJhcp0O(z^NbLRWL>|h2St&F6rx`Yos`s?W%%FxHHTx zaSdsVTJ!nKn5Fh%#Z4%5OlPR|P#4ZRXc~M8W(#^ZBcxb`YM6kd;xB1*47K9}lUn>| z8oJ1kTj3ZBaX~CmgmByw+l(5DEgjC5py zZN_LytX$c4f1otMltx4-yg}S(nbls=00Im4M5hF^j>^BapdKeV(`q+!%oe+1tQZHg z;V}_!wRx9Xu$&ahX_~wohe`oSk)n?^{)LVsUo@sd!?enf10Mp2hA%6Iq!yH@CrjKo!kBY1ZMy>Z2t?+=yyO!yv*{prJ7zq*9ja|!IpcWNVoY(`a==DEP>l6;5CSk_N4m?py z4WtK2VOLm}#c6+Pl}drtIawaah8d{9XPpq6TBg&OB}!swwxrsFJdaK%S#jI{4?qJ0 zAq}q|hoIQs2`eyNO zyRZ&X6grCRKl0_nv-SdBdM$(CjD zChA+|5#(syRZa~T%~+FSA(H1nKM12pl~WZFJC7RNb#MHkXcrF~ywiI<1}&!eNU)AIT75b-MdsE%G(0A<{lR6~pli}3|$gqfZ_GtR(-H)K4_4IqCW_Jn|+_g~GP zo_{L8G2cA4><)PK;FKaV{F~5wxY12QIz3#L+Te@ZjDJ2qwqm9Tt{u!D zIk3Pm2*loyuEjs9g@Np3uNNcNVZl*dA!CbrJ5PIaB7q`XxUovgYp|SSN!Eb@VZqWa z=H-I#eSg&@`?$G3IQcF7C#OIM+6~mfkH}F49 z%a)Jcc|1tuw`R7EOORj)$FFg7b-!z{#Eu*wZ{oPs3rbqbdo^8zwaoS1B&iqG}}PCU-w;=@~4SP>ybd_4%sj1>bQw2Z32dPVx7 z&V<>7J)j|lxsl|C;Y)EmRIsgvN)&G~(M2g;cO#_%ow1g0hUZtfLTt2 zB5K-?CUN(wrDKkOIH-ZO?FHYr+dShj%Ck*Q*e@ds0y)wr2>u3v0$suaivsp?7s{)| z0TB*?xw}q*1lH#*Nl!c;Am?$$6vgDcs-lSW34ZKs0$v9_^KVc)3Ec^;T+~pR3T;dw zAD=4}0kputA6Cy0qu>+-3^!4T7W)^JULOorqJgC%btDE0QKGX4va55v3=#mWX&oO)wcyodRH21s7UT{p#N<#q*zxc8}IWvSVt0%Fohvf2`Z(Sy32|ZAaS<`8sk9=p?Bl| z!nOz&jh;`HUsY5sD6F)dfDghTniYfzI0Y;0C$tL2mIPRc#Utw%DmyPyyhhJHnQRf95=LU-@-~UTqFS%;`Qu}89PITP!S-rd z4V>}V4kUkf@j~=LsnVctZ2XP(Uu%Tg8E1z`I=w9}i=m(ZThRVPa-ai_y!i>c0eas6 zN7~Eg{{T3QahbI8$dB?RLyNpYBYN0@&FUplg>?@N%fG#WOx}2VYmeamNm^9)WWwr1 zH_p+N9}tzSzy6A67O6{mye_#d(qYzQ0OwWMnrw%dsg*;1`VcGbanfYi~A6 z{VQoxp?SOC2`o$uz5S2hsR1i@M~%;p8(8c3!BTOMTt$Dp~&*f?@j;x3%MuuL>?z6 zCN9lf(73bg!!M<|&i?8vW4<6B>8k|gUn}dU_DUCY-rL*J+#oThv{|EZ+OA$r^Pj{? z3&ZNez#KMr^+$D^o9YNjWYN;FpuTOLXB?P6@BPXeUNU=}+_0aqu8{qVQ+fIPo9#dK zl1HBku0{{`Ad2KFTW@PMI*=YgiEwJz30~u*ouEhl&x|V{RpO zNKdOrc&||WLe5p62tA6HZ z%l~iPS;^}Dx%5cgMY4TkwjF4WNAr_SUwKxfxyRYTEwN`8xnKDcDh-TNBdrfK(!h5f zve+TS!S&kucIwH)U3pE4d{6Dib%`eaNd2ysHkpdCds4BuMQdPkSj8V_deGm(6dn>Y z?4*my)HP_iB|UU1FEvijHPj{?o1%Glzg#%E`3Ozr!Ztb!%d^e3HOW*;G9r@u*)O@5 zR~kay-XAWt$n(s*ojTIN`FhijH&eo@<$hOd z&A^RU(a0v_k+iFE917~LsuRXXJ|7FIjI!%klvI(_{-mSO1UEV9b@`n5c+oeI73hz6 zHO@}C8P?3r;7y{FqT|HH9J&K%v8YRr`+K6sPOCsD@Oir@B9NzGe)ecgWHWzo?>I@> z)mHZfD9U3Lr6x3&Qd$MQefZ!F+9nE}{%Fi|NBx7r-kq7*pR?4R;%!&d>TFL?TaCYI z@uzw1M}`K+KZbQ*5kpCexB3r_DYW=5IxvM)a+XHkM8^U_3iVp3JT*@4s3lE07Xa>_ zH9{)YfT~3HuqU?kAlq}ElLW0A$#{a;ha;`=dy$v z+HeYSdxwV0Vom;I^*0rrVRu{gC5f8uN=Y-SkM2w;I_i0Bqmpv!pi%k~!kO+*0}jRW z&Db9<*K3c1cczMF7~lCC6vlYggZUTIWS0sp5huzB-a$qb9+c5JOxMszf8qx^%h0mn zkTVWLws8o?x-bZpZ;@^dGFJ9gX25Ej^Ho;lbvva72!It?z)RJ3l_Pak{z@9gnJX>n z?K_L~XCoA@T+z_7Sz>~NTNAls?zi(pYXJxDnd=FQ-QSe&)lc<#=!>a=(2aXGv}1Nt zD}l1K?MOF!#Z`%O7Giq#t$=J04z|G{jx^uT=KOB;Nnb(VZZ5*dtme}kiM6L!=w2`~ zyp@ZjA7*~PT9PdJuM|$&dP%dUA4UP|L{cBvGSrM5y1S(a=u3EXJ#A4|aY!~>!wpld zR`-bQep6M4Cq|#g{|+b*K2~@aUUAxYstkEYigF}I({iFBJ*cXh9Bq^{%=YGyS8L5R zF*f)E+%;Et-)3FRo{NxteDuthC6x{ndvNf}3>>+rK50-RM=V`RH^OtCuL>{`M?PZDM`{TzBE^ z)`m0S94afs9s@G1$(rZB1xjxvj|}S8KAj`jt_h3`7jdGc_1tT`-Ntq6^h5JSs)*tJ ziuJTb+{LkOdhH`py=^M-QUaxupHMZmTc4c2Rf-?%>{7G_t^K3#471ypU3#rL=$)6- zUFM)i9Lk3NP4Cvs8=lW5$rtl^uqPqMeZ3t{zt^1`KclNcFy#<^Z{crnI8qX*(@@h2-7+S+_ESy0$||dQ+V=i)Zkr z-F`gp;)_*R9kEhxUkjV@lv#74xi$aMv0ZD9=)_^^Tvv4g#nt;JB!2}BOtIzS?reH6 z?rJQf?CX-QZc=pwE@-$6EW!Td2>krBbi?7rIa>za)CqG)aS*969OLC`#z-{n4c^t* zv*YBb<+{1a*}BywtuhwwjQ5)}g{E<(p}gh|+o~s)@h5OVppJCmZTN{ve<%Gz#APG$ z!n8{gerk}ZMMpa2dbOP~-5Lp4#z>w>!(jD=(EYhwEc1Yk?uNP9`iVCW**a;jSsXdD3-KdAH##xk{xW zr%!Vljl(Dy&FsvG^S2ibYCXL{H2#BXgG1Qdgw;TM!l;_iM#@aJ37*YWu-%}8*qmw6 z4=HV{>ED}EdG-h64hvX$l7@KeEk@_5-nJftE1NO1;&Knmhwj1hCn_$-*Olv=k9(Wz zN|c5dS6etM=jr^7*7%=pV~oGUb12`=>aKjoYav~f*M09PSG0}{Bh-DL zhr#-Ff9`lDdtZDk$p@DJ-R762Hh%58#2&5W$L09Fjgwq&PSzrL>leI?pB2YE+)6zU z^1ln$ocu2Dr#VnZYQxZ*03|wGxib_~-&ihQ>ilpg(`}g>46p7??8&{#-^y)>^(J`f zs9xRLJrfK)Zv<#AMQBhOy$3AB%=D%M+=A{Ef~+|JU$`JURA>A#?P>J)btHiSZmaF- zx?g^NI+HQlZrrdJK%4Jlg8U$JW^iKgtw5fy`-y{YrCDFWUyVY9ggWj)x+ey1TVJ5u z=}E&LZgW!Vl$zh4;7ZNYe&G(gm*2a@A7>16*6s5Z?63@nwq8C_>V3fM%3mTy_UqHNCQ%T6BUtC1FfuoJ z*o9v&>nVfKwP#{_Cg=GpX7c6k$%|m)DQ`*t$Gcxm@cG~GEcSm49^Q6A;z5CysH!J} zEEcbL`C8DoPcH)NQM*#N-$A6yoA+;y+Vghwp*R>n?afah_temkP zC-{RQ*MX&G3`5{vapYnCZ+Nc|oIppXg!SDQzkj|UN50?jAG(Z_kKImKbfjT3Da#8h zD>yK0w3tv#np~)H7#3qxQ{8oy*GMpJC?@|vufRJ8He4ujjY=s!ey?!L?Nu7VY2x9& zRH2M$b24~R#D>@_7@Nc#j*w;3&*moQd;@^2xrI^w2s~Atl|L9@x*tg>GP4PAF+^K$ zjp2Vui+J``e^cqq6r!~2lddg@{gNi&rIGq}=aE?m*|HSN)gD>*qi%2fI)k*FuKB^r z{5%>Zj_|nX&Ew~9VatKKai@Q|HyPIa8JF8WP{}z{zx}S(F-b@tPWLR5qtSj<4Z5sE z7SH>$twy_;#61`)(J2}mSJ_}=`yOr*3Tt|7xuSNX=ItNH`^H}b-6P_sblEf}0V+sz-30M##4 ztIrC2#N%Bi11?2aT)Y1kkO^=0DTUN=$Q5`6TJa{`k}Bw8?8o&CER2i^xz;9PF$4Aq zEtfg6bDJ+9&w6SZM_ic@{xB5@8S*gRXjHAaZ=b(;I|$X?G_bN|6{>;LP8R7nmdYpz zA6O)rVP+$}uf5>X3KPaL*9u%F9&_X5YiYHTLMg==_`^aP5rwF*_`g5)-+((cvTv%n zO~!rbz|L$n7$p^SG#3L9iVVeRgxrrql6*Xz4D!Xi_L$HL)E!n+>?`Qqhr5&-yQ!OY zmzZqu)56ZN!!T|lPt7z@UBc2PL%N8s8#h0Zh#5DHyo+bYfBhcB1I%$5*q;{32)dvW zx(_t%!1*kJH^H!~`>+Y9@M1DVT?%T)v|QJ+OU;^OSP%TeW{wxtsv)eY^5x4{!DU3S zgo5AOmXd>Hk_Xd8g`a!%7bJvGhysn>+p1}+PW95v?LyUtecL6&c=!=q>d~?JsMDQW z4)SH^U~nimSm=G*CQzFR*Cb`V7>^KLcC}tw)VdJ)DGryNqBn(D_GsP|(OQ?cElu6Q zZ6PFF^4gb#H!W}LjIPN7vU({Q>ms}KMA5PN(N31K2fmggNg%L;I>q2JMXUID)BP^M zhF#Sh)NXQ1wM$vAU$+;vOtJp%0<38=!r^R@WEJQUz+HD2KTN=RtE!@^)vlUNtJ+Zn zlCvMd;-bo7UvSmd;NOC^CVjg`Fr=F=dk+`ma04wLC)!g1Z4S3ulL6(M$xsb9z~54B zYr3|ITNAcc{m1?f%@A;hzG#)w+YS8=9sgyjOS9L5E*tFOhvZ^uSf}u^?k4)dw~H!x zIZg6Ef$WTvS-JAfu_&x%!^MZuhn+WW{E$`ygN~w>0Sr8#7gi2`uE#DTY}Y)%B4JE2 zX&gbMX*3)Ut2&F@ae)KZq^b=*3e1NgSbE5V2EQrEsc1Qzs>?(uE{)ncXq%nm(j&dH zY}Y1TnWk$K)@IP3YB?UL6>v+c#?!8C-If|_2Y%J%`qN3#Ztwg#l&eG6HhV+tw=_&$ z$}9Mrz|mr8N8oan=hMX~uboa?K-RqBf8&tnAfoBT#u#y{Gk4jCZgqN0OE>$|tTvGS zuthu9-rZ^ZXIi#R|6CjYl@W>mXvwM32>8rYe;{9>*Gt*Ic%+)HY`#)Yk zefwv4`S)ff`snoJ-8&4xl<(ewEKck^$w3j{y*qscKiJt6ejiPtuYdP0OJD%??p2I&8B3Wn_U>JJwYsP-?tEIpAP^UFEwd~e`tIH9dNTdpyJpqmyLTTW`R?7DdJ?}p zy03HMcdGxh;cQUMt^L_tlgFb@dwR zTCGq$l6@uDvYrjA#e7s3ds*WZDd&rEGAZUmJIO~4rE6y!9S!qIvRwas_2Z92aQ~*G z_YIY-FZ-8^;*Q2iH5*MUT!~s|i`P)RTfUS0NIh&o1)^xjh z%hoG&4-lYx>o=1epW7rK*|Ka;+Z+zHoc`nM5YL9A(e%R-J-J4Avr>CwIWsO_tERxq zhQ-{hzgiunwRyzU(c0a7&Yg8fo2!#Im+rz#Wc!k0;Nco_vx_-47L1)}UaF0kH5}m4 z+Z90^SQ+-AqpGgRdVTS?r>|bF$h~UD+=7EyR!f2e{QvUR#=S7kZ7E0P@Z*w;(fY3! zm%N9TUlj$;^8!7`MyTVp93bu>fTF1$YscSz?dDSN^>bHi52!Qnm*-EPy?Wjs=hyZB zf-`sh7mK3b&ibzhU-Eg2p%VKMXdR*qV5p}-Z;v0m_W0g!>RWRMsOXa*KnWi$hf;19T%`0@MZ+eso5 zp7XR9ys@(&hyrxEdhp*9#!k5S1~~BMGvSgCuk8iB40blNqX?S?H80L02+#fG9K5kg}&wu_iP0I5}5Agzo<*VTM zuR-kwVuEZO%*R1J><*!)^Pb_lF$1~@2y{xae3jxG66c^ z%LGp@?5eCG#w}4TYt}qjo(vR2W`DwTcrxVa-qPmv4Sr3~g2O8l>*~X=R%w8p9&Em@a}3NfY;6EEjD29X$=u zz%?5&x7l!J_tNa8K>(YwWN7&f`Yc%CGT2rkQWsD*V%RQ7AHW8I6a_1Bs-to+o=7IB z8I4dTIFF1zX?gO;4T)|f*DIwv2y&bB*C5uq`4?M4S6Z)7dA&4NC&4XxV=Sn#Os^5| z#u(u0xu(9~1y8eVF;B9KRm}lVlVsUHPlA7H@FM@&gBQT+IYuv6Ns$%VXj;N=`S5*R zXzVn{=c?@2YfBc^ihtA05aWu5<%%4v4SZI^A_18nbk7%fKArz}8RNcw z{Vx-4{1*&)=l^| Date: Fri, 4 Oct 2024 19:30:59 +0100 Subject: [PATCH 011/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index f45e962131..9d3ad69466 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -47,9 +47,6 @@ jobs: if: matrix.os != 'windows-latest' run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet - - name: Install GitHub CLI - if: matrix.os == 'windows-latest-off' - run: choco install gh # for Chocolatey (Windows package manager) - name: Push Results if: github.repository_owner == 'gateoverflow' env: @@ -64,4 +61,4 @@ jobs: git config --global credential.https://gist.github.com.helper "!gh auth git-credential" cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action" --quiet + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet From 78f336c28ae5e6e723e681028edf3447e7916dac Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 00:07:12 +0530 Subject: [PATCH 012/658] Added an error check for push-mlperf-inference-results to avoid push to unintended repos --- script/push-mlperf-inference-results-to-github/_cm.json | 2 +- script/push-mlperf-inference-results-to-github/run.bat | 6 ++++++ script/push-mlperf-inference-results-to-github/run.sh | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/_cm.json b/script/push-mlperf-inference-results-to-github/_cm.json index 132b590b69..bff6447fb3 100644 --- a/script/push-mlperf-inference-results-to-github/_cm.json +++ b/script/push-mlperf-inference-results-to-github/_cm.json @@ -31,7 +31,7 @@ } ], "default_env": { - "CM_MLPERF_RESULTS_GIT_REPO_URL": "https://github.com/ctuning/mlperf_inference_submissions_v4.0" + "CM_MLPERF_RESULTS_GIT_REPO_URL": "https://github.com/mlcommons/mlperf_inference_submissions_v4.0" }, "input_mapping": { "repo_url": "CM_MLPERF_RESULTS_GIT_REPO_URL", diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index fa677e026d..bcfd791647 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -1,5 +1,11 @@ @echo off +REM Check if CM_GIT_CHECKOUT_PATH is set +if not defined CM_GIT_CHECKOUT_PATH ( + echo "Error: CM_GIT_CHECKOUT_PATH is not set." + exit /b 1 +) + cd "%CM_GIT_CHECKOUT_PATH%" git pull git add * diff --git a/script/push-mlperf-inference-results-to-github/run.sh b/script/push-mlperf-inference-results-to-github/run.sh index ac3a50d9f4..c1d66f0278 100644 --- a/script/push-mlperf-inference-results-to-github/run.sh +++ b/script/push-mlperf-inference-results-to-github/run.sh @@ -1,5 +1,11 @@ #!/bin/bash +# Check if CM_GIT_CHECKOUT_PATH is set +if [ -z "${CM_GIT_CHECKOUT_PATH}" ]; then + echo "Error: CM_GIT_CHECKOUT_PATH is not set." + exit 1 +fi + cd "${CM_GIT_CHECKOUT_PATH}" git pull git add * From 948842c8d6526129221006d7b4461b54043932c5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 00:08:05 +0530 Subject: [PATCH 013/658] Revert "Results from R50 GH action" This reverts commit 4e924dada2a8c7d7240a6a22c17ef6af34347b76. --- cm-run-script-versions.json | 602 ----------------------------- hardware/gh_windows.json | 26 -- mlperf-inference-submission.tar.gz | Bin 19288 -> 0 bytes summary.csv | 2 - summary.json | 32 -- tmp-run.bat | 36 -- tmp-state.json | 3 - version_info.json | 602 ----------------------------- 8 files changed, 1303 deletions(-) delete mode 100644 cm-run-script-versions.json delete mode 100644 hardware/gh_windows.json delete mode 100644 mlperf-inference-submission.tar.gz delete mode 100644 summary.csv delete mode 100644 summary.json delete mode 100644 tmp-run.bat delete mode 100644 tmp-state.json delete mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json deleted file mode 100644 index 08ddbd1975..0000000000 --- a/cm-run-script-versions.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "onnx,fp32", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_resnet50,_onnxruntime,_cpu,_offline,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "resnet50,onnxruntime,cpu,offline,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json deleted file mode 100644 index b7cf960dbb..0000000000 --- a/hardware/gh_windows.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "accelerator_frequency": "", - "accelerator_host_interconnect": "N/A", - "accelerator_interconnect": "N/A", - "accelerator_interconnect_topology": "", - "accelerator_memory_capacity": "N/A", - "accelerator_memory_configuration": "N/A", - "accelerator_model_name": "N/A", - "accelerator_on-chip_memories": "", - "accelerators_per_node": "0", - "cooling": "air", - "division": "open", - "host_memory_configuration": "undefined", - "host_networking": "Gig Ethernet", - "host_network_card_count": "1", - "host_networking_topology": "N/A", - "host_processor_interconnect": "", - "host_storage_type": "SSD", - "hw_notes": "", - "number_of_nodes": "1", - "status": "available", - "submitter": "cTuning", - "sw_notes": "Automated by MLCommons CM", - "system_type": "edge", - "system_type_detail": "edge server" -} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz deleted file mode 100644 index 252ae3709cc23b28380a6560cbd92ddd57b4229e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19288 zcmcedV{oUz^WbCi8(ZJl+SsaQ4|cwIWvek2>6w+r!(Goec7k3*eQ?8Z2b%&BaExA?o#yZ^@3s|U;TXa za^enr?NP{kImKQ-`)Md}y6f#aYYj18&=dR_+9 z&Y5TY(y?7~bg%(mo-&STa}F)EV41tG_qNS1i&56dbEh-$^^;Md?BLEtd0PkYeA1grbNTwkmCN^< zC+F-IppZ!?;UXJ6)?&wB!=UlvUAW{rD?RbRDE z+-vnnePoe^^#8CD2L1Yzy-+qYDLZ+49Y4SBPYqGhjh3PT&3C7k*sWsQF0{@}Ly|ha%z3l{8z! z+P(*OhEFyl#gc;!^o?fTu9dd-lt9>0H0;%e>+*^9Ph}bMv5gOVRn(pq6K-%5 zpmn}X0r$7QZL;xlWn>PScV#A=`k(2@JIE8{n&qJrc~M;PcZS?w^`9ToTY)7V^hrxL z76Hc0psx7xHruJwVnyuC9)22p$=ov}@Qblhnlg!|NuO-I@VP%v+>iuX%hnApFuGiS zulZ!IH9!P60(tBme8=zeKivQj-Mi#?HGT&d`a?%p3K=#Ptvn($kku2zumI3}&v>_S?ML-Jc+0Kk@dtHKSF^9kmuyQZJeV@e5r* zMl*9kc0NuT$aa|+{ESa-uKuNmeALZ|kWAfWt6~Z9=u~hPv1aYaW)TAG*W&Hhc zb_hiA9eGz`L@QgC~okK*Mmrd z4O;lw@4b!n(bk(7S<0S<(0;!CW2bhv@hzjo9;&_Ce^xCpzp9_}(lK##v{&p);bXA- zLx2R5=kfI`;PETZ|Em&YpoI8x#;=5I`R>b^H!tyPp%A2Ylz%(#ODnYS;OoSLA9k>x z2#S)mQ$x$Y`;8H2;r#P{2eyj$HF1V=PoTwzn}|{oDzXppwN_;9zlMmuiFfbA1+NqM zYRBBO5B;i?f9X&2aN9LpNCZXLfujBUAdEtu!WWJVP{e7`s_lt}i-Tf(#dhP7%WcDj5Dd{Jp{^xDkZdwtMErCA1 zT++Wud8nJKu2}h(z@nFN3#HLv69I%zzSmFcPYo5;5hk5guIEsJ zNoeuetOyu`-tXJ;zbBYBE{J;GKd*K>Z=mgay`S*)jCnzcL>h%5ncPlosv%{CiqJ z6zIZ?e*{iE7WimBZUGKY=;S#$aG=;@dK@f9sq=^fPsV@@Pv8oDO85Ix&j@V`J*$z# z>evIp4<#Sb0ZGpH(N?fe4}4vf!aNL7*ar_KGb8rhUu4KG5njbDpAUjNcg6!N;gwuwZJ z3~T`<^K7X~BaKXQDj4IF$Gp_(DI=Jk*b)_z2*)x7K_L`rag9POct-E?vHARi_rMl9 zBOH$3Od%;sxs(ynw#g;1jQgZ!wr?{e-ji^q^E#sysMIT^C_4LqXP`VBy=ww&68fzr z*?UzkdS@XCdPT60I4i1zXCgcrxn^-c$Df7CAyP!+pkM=%9~p5B4u5JjBi0UQoN{Pz z$OnenJeo#NM&r2%Z071IaLe^r7$sIX#H(h(`aLm8mT8zZv#~8v-kR!oQyZ9~X;Y;(4lC6h)^;^ENWoK?ZL9u- zNU{lHos-T{$C34+^ho^BoC(J`_QVaMB5T7*qJexi&0xm~J2n*yk4Ao=tESD?ojhLf zTdO+%xo`)=Vh+2I877vE!$4+&WdR@VhZJZO0~0lw4kij<<#;_7sROFH@na({c(U|b zq%$Pq-%BDrQ^*LtK2SkvsG#U70U;$_plC{#)R;|Ka${_Quy-q2(U@~fJu$5rgj1~0 zuztkQso=l~C$j?@b3KE=$l4@1PiUI`#X!3dW2W*KRl!$8?F?N|GS~)%Q}vv{J2817 zVbq3%rrROd!H>p4|3G*^-RB-bC6_Z}R7SI3h6eeY1O`=%f&S)AiN=T?AHzoaiv-fg z=%{OGSvg#lhhabz7s&=GUrCxKOVO8$Z=(bVgS(VP4G5c9gxM7JYMi8+hqmaBd%NPXW z(C@V*FbJc_V+80FNTue7;55Kj31g0f#e{Ka- z(OAkAt0Q2Se#rqau%~7DvTlzTLR^Su`eg;FBre(YF&pBTw_mSc0OZIb13yEi<&F|W zChVw}&>|hxsreLFpq_r$+r@^6_1|r~#|%j02AfY}eFj~;U88(jD%^1ad~95=m%zO; zr$q|Tual{XB5-T&PN{_xw;wq_e-NoI(BROoJX8x6AjyNhRvT!*&u>FBJtNI7U9iiD z7EWFQGmy*hBJV`dCMbTRGJgyk-gau@l{*k@<0p3@=>&FIGSmbghmD`^*UcGnNSJdi zFxL^8GnEDs{mQg;p}h1DKsEGnHW(QkSHt4j?gR3g$ga{~PH?_?2rw(PCMBt@9{=q* z$ybc9qojU8$MAJlwQ7ZIMbp@Yq?`SIgI;aW+q7xFaDYo7^smnB<WDiVe8zz~gm6@e^;tnqgXy~@1%vh^EY1AvJa-pD|6B)O zLeDZcxv54~cHd%kw@ULfb00b>(DV zGDAB{mX~ix{rT&E$KUS+-+J^zMI{U(;d;wS!)2qQLImsr^K z)p>mUo_p}*^nV)0c9A(eXJ&E({q5!ot_0X!CHpoD@jBlY7fC?TcgO`h<>UxZcoVnS zc{^q02mpfnb|CjLY+B~UNU(X`5(ZDlHHOO_H}8m#+8V}nP`Y)Qxobm`6}4_xJqaA^ z$UQn(8g|bce&&Hs8g$+W`DUR{oEPKRpJTbv8VY@k(f!&?F0xsA#oo8V5ronFUZm)J zP-Fzz0V7xJ^P;!%Pn?fs!a@`dmyWh<2VKxdV~Qf0P?0BmsCR_$h$h2@q~%4GXG8KIT#VsjYBrzB+G)dfN{Z^6vh{3gDu6Qyl}um zR^;Q_XhbK15BTQ^;VL~?=|Y1U(G`VRN3+wQB|<(~nxL_uA3?ARS}RW0)4jqjrh#h! zGzaH|LB=FeBq716#fpXWIFn=S1%1ekklBbIsSs`}6Zohw!cF=pLJ^?@0^LORVIG8N z6+;z^fy$tokSzo~&nUYnB{(P{4V>YDY~_3w_3?L9gdNa}jVSD}THuj`iV^uCh2l8< zp}oZMiWnllXA5Dg0OMiG3AXvxfJWg7gg?ah@Y0|x2$@1aq`4q4#c5D8Hs<_GFd!mc zgcY2mASqHGj0G$j)h7&u$%wlJZak9(rZlgXrVSJd*nQX|lPJ=dxUUek=vx>^-|)Ju zFzPHyyqF?$#Co(7)Bu^%Dl<_rU~EiFbuPTXz%YaO?KaF%gcv82HdGxvF`usAoLG2- zibE0XUY<;4Uyr^|5;PL*5BEA85dDZ?9CTN>iOYf{fPonIvOqjM02rVS0S5_Sw*qIw zPPbOroObR&;M0RLGuZl;=xBVc4=DNk6tMl=^{C9$GD{N`!z`whOqvv{0l7X0d!3uq3 zp_Ou5ZPmyV>oQz#S-pScxt#}R3<~Vjuh|5zEQ2*c!8sA1Oh>Z@) z251ro&&OYp-XiN);utEyO_@kybT}uI`6HBoAn^hQ@U@n-s+PtQGdwY^R(p{&c>Xca zcm@qZ*{rxN4y~V;9?an}p3n6o;-G?fzZ?UA_N)0uaea~x4>8NNH3k7OF~{me z=*b=;NTlKQBt6l`=t9~KpEaaV6l1Ta$rVf=A_^(S7E18caB9}wsZ~!MbeJgmHdw^b z`Qbb0(C33M-e}=AGRiBxkcIR%6Ka?_!K9q4Q7hFvC^5VyB)`|}ZapR`x;Q>5{(h;$>JTk~^eC=9yUOwH zZ9K=2$XgiS;m&v=gvCTeG z7X9i_ucHm2i^4?X&WHVx5${MR-g76&b>=$}|xkb&_u9BqW(0SSb8_buDW`1Mu>x`yd6`E zG!4wkzh#>VlaynD+g6oQ+b%{Z_OhG{5|I644|bwSlqYi{scQQ_w8! zmY*io_S1-3FNZbjhnMn*>l`J*R_@+rvpuvAbA!tt5kgxKJ5b%JtYuIu(olt;GBVTLr*~>A(ihsXw={~Ai+H}G}=%J7b6Dn^=wMr{>n{6uyU1o&% z+`Ycur9T-3*kbpaU;JdQUj2k&pR*pm2z?^Vhbt(#18W%n-E{BTa8Kp!5+>y9>E9Lq z7)9py#RiD{aQ@V{ekko^%%B-@lf%$jAcGlCpPNPF%<(vZk^rd~jP&Ttv(C;F_tqX4XxYPzmP8;+s#L zffCEkA$Y}bac-OqLEh>W2u&5x&cPDF1GXj0SFYUqgPau)mYti-KYyLG%KgZA>&hXI z1z}I}s&Q&onjiojAn1~pe<*C0pVq0NnxA$nIEn>v6gofb;432&5=O;g>EbJMX$N$& znD8#r`~(>N1ROrN*^sr$4=9wZJ9D#`ghTFMwIuS80!6NVsEl5z0q{)1wI-gC`Y{U9 zlcpz<7Eno0QT_AhTioDU-{BDJ=dNT?YhYl7oV>XCTfrAZZSq=Q;Dy_LDeC8|x{Xon zh$zAn`Qo$?(;cSbNpn!W=k&^GMd-HYEZ`(UboCmuUDbuGm#m$4B}hL-m}ccT5ekXa_pkMx~2fJb>p+(8!k1R; z%zB320z~aBqF!(17PTo4Za5T2|w%scm&M4Y1{)a`Wf5b*CdCfd~>jbgPM$*w`@+_%oJh&v) zx)GFDhgXAYS7Rvd&`$ZeR}?~>ETYhAGwUB(A|ns;OA2$>$=E+bEqsoe2(%9}N#u)X zn((v_#uVGE`F{plEbeP|y5YJrVJ7vb+Yz1Ox(gDa<$6FFk0j)Wdo=&vBF1jTN`hD| zyZXp;^kx@Qqx%%aI z8Lr6ZypQm61B2Yn*OJ?X?P@M;)wR?ZfR=bQfYm8Z1*CZJ^v1&euBq3c-ix{!{S1vJ z;o3M#4JnXeKz+k1pd^$k6gE*M*YmITM9OIk=E zlXLnz;8U=X5dy?~lJ%x>;N^tR32U?49j$PKeLojf zDrlucVTLM@_2=v7?!pv^bt*+gzuN7W`(0}-QKr!7On|ENOEr%j^;S}3(+yj;26lB? z(kzEay-y~t)c`UZJ`BRZEl1qBC3|0x&I7h{Xs^QQ$(Wx}@om-o?CxaCk&*@Wp(8OG z&fLu9q%)$)8~v68xObaLONgAf^C_%xwvIu|bJMb}LCetBP_!03ZO!}CMg+{3 z3mIp19^3S23Aa#3mbH`A9@kY%32v%D!}o+3h_RGeucuKU!b-42-Bx5d>5TF zJ|f-3XbWiav0}0V=3V+ zHo{|8IpDG!jqf>5UT3`>VFKYiD^}m80Y{lNdKCZ8Tdp4)Wu}zN+>&GTfx!#zVfX~cOmOCpYyDT(6 z);ucQCS=bI<+B$dIs{ae3S#J~GC_pRS+;p;Zo_+umAonEl{wkWZ_Sqx;$0-og!jTx)}}{8 z2P@SgL%ls8M5minJS>rZwcV~Zld!{~Ci|bUK*0I4N9t%|Un@!T-{s#O0rn(PWpp8B z>Ks})XmW8I5^C9~mW8E{Az=nA1u}n~ch0pq9;eO<8LubKLwmSLR?)C`?M}0lN*-jv zoH(AR%72FuZipJ~xNJ*!6-Ny9RN!DvCZ#+&n^-12_WegL18-+!KXv}>j4GsIrBEHq zhGXH<36`8-(Zr(t(BV+oL#k6TJKiGtEn3$dP2u%UEcn!{Rk#r z-|O+S8r1y89Da{+6MR?>uVp0%DFnb)nZvhj*$|PmOrpIV*sW^MzFHij`iNo5&ohUP zijW!Fcd+bblqvsAKK(1PBahSZYvTznWYmflpk27#Qx>Fbf(KMmo4L(Q!q{XR!5Pl2 zs-dF>*qW3P29D#}wXpWZ(#`EHIxFZ4VLFbw2xW}6e4B}QkbABrBq6FMDM8F_{-Jnx z28GqZmhQ25cf_3EH7X3pDLrJwu@}@6FRAtJDJlI-{1~DaM}1a|R`Y1vcmMPHR~=!F zV|t5Puk!?Qj^k>}v5ke5*{4mLpYfaVt>}v<&qLs2k#-#j4;f*VnN@*gI*5GT?G$)2 zZaFP`CQ9y6m)=#acn@ZwZmPN4I%u7uAkS*&f99uoyty=2{&{eMcqMYz->7Z`<|&g;5PmEvdkQz4eGaq zKB}zpyQCi=_`np}-!k!xw&&=@yDsfq40Pt{IcxD3eB>{lB zmq)t+L^pV=X+00l1E@|yltJCQUU@8oQTM+&{D+3#uAiWWWO!s7Lh&;1yR z{$tOlxpf4Jn(=i@7lL9}m(_7O;(i%=;T|SB#ivBL2h|Bl*_F^!Rl_8OyJJJ756`hW zb^168ANQEFAax2di&ljiSjA2f`Ovi$hhb9DN@1)!!(c=*mdm+Ncy^_^rIfw?k= z)rG;gA?o1-Ff2UbgF){8-oTJNZ*DFj3fGElkA_5J>C;~>I~g><)9Nw)9!1ZF#4TB; zpWc%4GzHLMFvtUsb%6PU*JAxddMX}wJiFDbb^AS33MW0-Em%_{1(hvJ&W`wn3A`1S z=F1g3B3c-ZbK6t8?2}{yCo|Y9ZN3G*K+*0qy54&aV!Or8qj2H4%On+<)7ec{5|Xic za+iXtI|SCva*m=(gp5%rT0lj#8-{_H4n+qY3p%4#BX@)7(Tejtn&z1tmcQ3?V0ids z-Ymnxq6%%sT#UPG{#HH*$B^v{{s;x>zW0J)rg^rIJQJ#kT%bL%M5>ocAYXwRYC?Qp z&FEo7__wkqQ3?4dWxCHSp*IJAa$L%#v3mTVgm0MdJi-Twb95ZsPTnjdj7zS0EKLd{ z^x68J7OQG5yOwa$8Yj21UEK2b>b^aNsK;?eFy2$?yQI$n!uKgI$GdFYHYwmhO!o$> z45WKhw>{ZzHYe9U(9wOO`;BUn!@(CfeuA6my6sZ9=uX7|>$R}frV=-h;JWz zj+X-m;*aodLu;PMVe#g8ZGNBj?uQ8Uso1OLZ2GK>b#ppM&Eu^G_homAf(!aNrMWX_ z1@-UCiVJN0Ahv_qUh2btm9>r%zPn=JZ6MTcRvQL4Ju3r1?1BVvp$+#ERtVVnlk3$wmf-JJuCt9}H{AQ5;uHTGCV z=|(TL+DRV1$HmP8?97l<(f7F+msa;ebA77HO9I|khky0^FeQL@p4udwOiBaZN+GO~ z`|S?1yub|KnM4x%ICRhcR^KeDH0%e@Uhr6@Fg%trJ0FjqyE(ZiuP)HaI00JMU3aO+ zQ7KK@Qz-j@&3riHr#UW3HymQHi=VN?i zI&j~<@7V8E<))o*C5t6Cwrh*$e*69kj-2S{Y^^Kto-<9I+=uXD;xc7Gz^x7o=~f>{ z*+q@Q24!do-fJmyK-pE;m?Yo@6GDH+;iq)NzUU=QpWUYRTuDMtYT_&4PN%r`Kkmm0 zW9GL2_4p%02H%T~cCy`O)^+#y^{+2Ajp_2Cq>!#h)1aTGgSpEGo`e4+&s*K*+gs!^ zxb2d>$CW9Uh$erEIyJ+42*_%>FvB?$@8;smjPdk17w=)b{V^Zo zh36IRf9DWdRZ;xVm|K%j^>pl_)o=Vbp0V|~4I2k9i^Lh>)j9kD)I6&`g+h;;xQ6Ps zbXX+o1|G`&4E0RCk@2$P;2R%a5KCB))57%f&K7Q%&06=F;cape*>+JXK1tV;BHfNo zXPv3|n^^T2D_K3WUPcDw>d0bffA&8N?Kx&up82L>c|-+`x`2ndfJS0wcwYCziaEJ+e-QE z4ESK~76)+wcgI6Y+)v;Bwu{W`C{$ucvQPWQ3%weMYQJ9Lbw(ZwfYp255BVNRk^Uw7 zVu8ugSVJD6F08^Og<}i%M#+Uqp{^IaDGQeqwOc)TUrx~kW|Yh3uK zZLdz;D{r5CgHmXF`us`G6_x$2pYrXczb;PNzMc*b50@s=1pZOK9-T^fJ3{>7$)tG8 z$U)fN-~xCa)F|9 zn}}myJse6QAyLT2eu1pdjcK-Cg~JgZhIk)PWs)4~k+U zgxFxx6aWeR^DaGPw7_R#5;jV{?UXgNGWP#UWL*C*L`Hs81?G`&Ujd=mHVibKr(!Op zP4`|Maf|rAsX7ilWU8GfQa_Gp0P1nP_9^@0nti6R;=KgrVW(7>LY+OnwYK z=im$Zh8{`TXb!PBn8K@Q;2e1-j;)GoX5NxA7%iuM;Wvy{DN;gP;2{{}lOoVG1cf1Jj zmY{Q_<{+Uo%7i0Qq&79t4J=36e=10hNaLLIdlc*I2h4;IxeaGSfW(;VETwHN86!ri zV9Sfh*JS4k{|7L`&_%7vdZ5&?B8tPA8ma)iJhcp0O(z^NbLRWL>|h2St&F6rx`Yos`s?W%%FxHHTx zaSdsVTJ!nKn5Fh%#Z4%5OlPR|P#4ZRXc~M8W(#^ZBcxb`YM6kd;xB1*47K9}lUn>| z8oJ1kTj3ZBaX~CmgmByw+l(5DEgjC5py zZN_LytX$c4f1otMltx4-yg}S(nbls=00Im4M5hF^j>^BapdKeV(`q+!%oe+1tQZHg z;V}_!wRx9Xu$&ahX_~wohe`oSk)n?^{)LVsUo@sd!?enf10Mp2hA%6Iq!yH@CrjKo!kBY1ZMy>Z2t?+=yyO!yv*{prJ7zq*9ja|!IpcWNVoY(`a==DEP>l6;5CSk_N4m?py z4WtK2VOLm}#c6+Pl}drtIawaah8d{9XPpq6TBg&OB}!swwxrsFJdaK%S#jI{4?qJ0 zAq}q|hoIQs2`eyNO zyRZ&X6grCRKl0_nv-SdBdM$(CjD zChA+|5#(syRZa~T%~+FSA(H1nKM12pl~WZFJC7RNb#MHkXcrF~ywiI<1}&!eNU)AIT75b-MdsE%G(0A<{lR6~pli}3|$gqfZ_GtR(-H)K4_4IqCW_Jn|+_g~GP zo_{L8G2cA4><)PK;FKaV{F~5wxY12QIz3#L+Te@ZjDJ2qwqm9Tt{u!D zIk3Pm2*loyuEjs9g@Np3uNNcNVZl*dA!CbrJ5PIaB7q`XxUovgYp|SSN!Eb@VZqWa z=H-I#eSg&@`?$G3IQcF7C#OIM+6~mfkH}F49 z%a)Jcc|1tuw`R7EOORj)$FFg7b-!z{#Eu*wZ{oPs3rbqbdo^8zwaoS1B&iqG}}PCU-w;=@~4SP>ybd_4%sj1>bQw2Z32dPVx7 z&V<>7J)j|lxsl|C;Y)EmRIsgvN)&G~(M2g;cO#_%ow1g0hUZtfLTt2 zB5K-?CUN(wrDKkOIH-ZO?FHYr+dShj%Ck*Q*e@ds0y)wr2>u3v0$suaivsp?7s{)| z0TB*?xw}q*1lH#*Nl!c;Am?$$6vgDcs-lSW34ZKs0$v9_^KVc)3Ec^;T+~pR3T;dw zAD=4}0kputA6Cy0qu>+-3^!4T7W)^JULOorqJgC%btDE0QKGX4va55v3=#mWX&oO)wcyodRH21s7UT{p#N<#q*zxc8}IWvSVt0%Fohvf2`Z(Sy32|ZAaS<`8sk9=p?Bl| z!nOz&jh;`HUsY5sD6F)dfDghTniYfzI0Y;0C$tL2mIPRc#Utw%DmyPyyhhJHnQRf95=LU-@-~UTqFS%;`Qu}89PITP!S-rd z4V>}V4kUkf@j~=LsnVctZ2XP(Uu%Tg8E1z`I=w9}i=m(ZThRVPa-ai_y!i>c0eas6 zN7~Eg{{T3QahbI8$dB?RLyNpYBYN0@&FUplg>?@N%fG#WOx}2VYmeamNm^9)WWwr1 zH_p+N9}tzSzy6A67O6{mye_#d(qYzQ0OwWMnrw%dsg*;1`VcGbanfYi~A6 z{VQoxp?SOC2`o$uz5S2hsR1i@M~%;p8(8c3!BTOMTt$Dp~&*f?@j;x3%MuuL>?z6 zCN9lf(73bg!!M<|&i?8vW4<6B>8k|gUn}dU_DUCY-rL*J+#oThv{|EZ+OA$r^Pj{? z3&ZNez#KMr^+$D^o9YNjWYN;FpuTOLXB?P6@BPXeUNU=}+_0aqu8{qVQ+fIPo9#dK zl1HBku0{{`Ad2KFTW@PMI*=YgiEwJz30~u*ouEhl&x|V{RpO zNKdOrc&||WLe5p62tA6HZ z%l~iPS;^}Dx%5cgMY4TkwjF4WNAr_SUwKxfxyRYTEwN`8xnKDcDh-TNBdrfK(!h5f zve+TS!S&kucIwH)U3pE4d{6Dib%`eaNd2ysHkpdCds4BuMQdPkSj8V_deGm(6dn>Y z?4*my)HP_iB|UU1FEvijHPj{?o1%Glzg#%E`3Ozr!Ztb!%d^e3HOW*;G9r@u*)O@5 zR~kay-XAWt$n(s*ojTIN`FhijH&eo@<$hOd z&A^RU(a0v_k+iFE917~LsuRXXJ|7FIjI!%klvI(_{-mSO1UEV9b@`n5c+oeI73hz6 zHO@}C8P?3r;7y{FqT|HH9J&K%v8YRr`+K6sPOCsD@Oir@B9NzGe)ecgWHWzo?>I@> z)mHZfD9U3Lr6x3&Qd$MQefZ!F+9nE}{%Fi|NBx7r-kq7*pR?4R;%!&d>TFL?TaCYI z@uzw1M}`K+KZbQ*5kpCexB3r_DYW=5IxvM)a+XHkM8^U_3iVp3JT*@4s3lE07Xa>_ zH9{)YfT~3HuqU?kAlq}ElLW0A$#{a;ha;`=dy$v z+HeYSdxwV0Vom;I^*0rrVRu{gC5f8uN=Y-SkM2w;I_i0Bqmpv!pi%k~!kO+*0}jRW z&Db9<*K3c1cczMF7~lCC6vlYggZUTIWS0sp5huzB-a$qb9+c5JOxMszf8qx^%h0mn zkTVWLws8o?x-bZpZ;@^dGFJ9gX25Ej^Ho;lbvva72!It?z)RJ3l_Pak{z@9gnJX>n z?K_L~XCoA@T+z_7Sz>~NTNAls?zi(pYXJxDnd=FQ-QSe&)lc<#=!>a=(2aXGv}1Nt zD}l1K?MOF!#Z`%O7Giq#t$=J04z|G{jx^uT=KOB;Nnb(VZZ5*dtme}kiM6L!=w2`~ zyp@ZjA7*~PT9PdJuM|$&dP%dUA4UP|L{cBvGSrM5y1S(a=u3EXJ#A4|aY!~>!wpld zR`-bQep6M4Cq|#g{|+b*K2~@aUUAxYstkEYigF}I({iFBJ*cXh9Bq^{%=YGyS8L5R zF*f)E+%;Et-)3FRo{NxteDuthC6x{ndvNf}3>>+rK50-RM=V`RH^OtCuL>{`M?PZDM`{TzBE^ z)`m0S94afs9s@G1$(rZB1xjxvj|}S8KAj`jt_h3`7jdGc_1tT`-Ntq6^h5JSs)*tJ ziuJTb+{LkOdhH`py=^M-QUaxupHMZmTc4c2Rf-?%>{7G_t^K3#471ypU3#rL=$)6- zUFM)i9Lk3NP4Cvs8=lW5$rtl^uqPqMeZ3t{zt^1`KclNcFy#<^Z{crnI8qX*(@@h2-7+S+_ESy0$||dQ+V=i)Zkr z-F`gp;)_*R9kEhxUkjV@lv#74xi$aMv0ZD9=)_^^Tvv4g#nt;JB!2}BOtIzS?reH6 z?rJQf?CX-QZc=pwE@-$6EW!Td2>krBbi?7rIa>za)CqG)aS*969OLC`#z-{n4c^t* zv*YBb<+{1a*}BywtuhwwjQ5)}g{E<(p}gh|+o~s)@h5OVppJCmZTN{ve<%Gz#APG$ z!n8{gerk}ZMMpa2dbOP~-5Lp4#z>w>!(jD=(EYhwEc1Yk?uNP9`iVCW**a;jSsXdD3-KdAH##xk{xW zr%!Vljl(Dy&FsvG^S2ibYCXL{H2#BXgG1Qdgw;TM!l;_iM#@aJ37*YWu-%}8*qmw6 z4=HV{>ED}EdG-h64hvX$l7@KeEk@_5-nJftE1NO1;&Knmhwj1hCn_$-*Olv=k9(Wz zN|c5dS6etM=jr^7*7%=pV~oGUb12`=>aKjoYav~f*M09PSG0}{Bh-DL zhr#-Ff9`lDdtZDk$p@DJ-R762Hh%58#2&5W$L09Fjgwq&PSzrL>leI?pB2YE+)6zU z^1ln$ocu2Dr#VnZYQxZ*03|wGxib_~-&ihQ>ilpg(`}g>46p7??8&{#-^y)>^(J`f zs9xRLJrfK)Zv<#AMQBhOy$3AB%=D%M+=A{Ef~+|JU$`JURA>A#?P>J)btHiSZmaF- zx?g^NI+HQlZrrdJK%4Jlg8U$JW^iKgtw5fy`-y{YrCDFWUyVY9ggWj)x+ey1TVJ5u z=}E&LZgW!Vl$zh4;7ZNYe&G(gm*2a@A7>16*6s5Z?63@nwq8C_>V3fM%3mTy_UqHNCQ%T6BUtC1FfuoJ z*o9v&>nVfKwP#{_Cg=GpX7c6k$%|m)DQ`*t$Gcxm@cG~GEcSm49^Q6A;z5CysH!J} zEEcbL`C8DoPcH)NQM*#N-$A6yoA+;y+Vghwp*R>n?afah_temkP zC-{RQ*MX&G3`5{vapYnCZ+Nc|oIppXg!SDQzkj|UN50?jAG(Z_kKImKbfjT3Da#8h zD>yK0w3tv#np~)H7#3qxQ{8oy*GMpJC?@|vufRJ8He4ujjY=s!ey?!L?Nu7VY2x9& zRH2M$b24~R#D>@_7@Nc#j*w;3&*moQd;@^2xrI^w2s~Atl|L9@x*tg>GP4PAF+^K$ zjp2Vui+J``e^cqq6r!~2lddg@{gNi&rIGq}=aE?m*|HSN)gD>*qi%2fI)k*FuKB^r z{5%>Zj_|nX&Ew~9VatKKai@Q|HyPIa8JF8WP{}z{zx}S(F-b@tPWLR5qtSj<4Z5sE z7SH>$twy_;#61`)(J2}mSJ_}=`yOr*3Tt|7xuSNX=ItNH`^H}b-6P_sblEf}0V+sz-30M##4 ztIrC2#N%Bi11?2aT)Y1kkO^=0DTUN=$Q5`6TJa{`k}Bw8?8o&CER2i^xz;9PF$4Aq zEtfg6bDJ+9&w6SZM_ic@{xB5@8S*gRXjHAaZ=b(;I|$X?G_bN|6{>;LP8R7nmdYpz zA6O)rVP+$}uf5>X3KPaL*9u%F9&_X5YiYHTLMg==_`^aP5rwF*_`g5)-+((cvTv%n zO~!rbz|L$n7$p^SG#3L9iVVeRgxrrql6*Xz4D!Xi_L$HL)E!n+>?`Qqhr5&-yQ!OY zmzZqu)56ZN!!T|lPt7z@UBc2PL%N8s8#h0Zh#5DHyo+bYfBhcB1I%$5*q;{32)dvW zx(_t%!1*kJH^H!~`>+Y9@M1DVT?%T)v|QJ+OU;^OSP%TeW{wxtsv)eY^5x4{!DU3S zgo5AOmXd>Hk_Xd8g`a!%7bJvGhysn>+p1}+PW95v?LyUtecL6&c=!=q>d~?JsMDQW z4)SH^U~nimSm=G*CQzFR*Cb`V7>^KLcC}tw)VdJ)DGryNqBn(D_GsP|(OQ?cElu6Q zZ6PFF^4gb#H!W}LjIPN7vU({Q>ms}KMA5PN(N31K2fmggNg%L;I>q2JMXUID)BP^M zhF#Sh)NXQ1wM$vAU$+;vOtJp%0<38=!r^R@WEJQUz+HD2KTN=RtE!@^)vlUNtJ+Zn zlCvMd;-bo7UvSmd;NOC^CVjg`Fr=F=dk+`ma04wLC)!g1Z4S3ulL6(M$xsb9z~54B zYr3|ITNAcc{m1?f%@A;hzG#)w+YS8=9sgyjOS9L5E*tFOhvZ^uSf}u^?k4)dw~H!x zIZg6Ef$WTvS-JAfu_&x%!^MZuhn+WW{E$`ygN~w>0Sr8#7gi2`uE#DTY}Y)%B4JE2 zX&gbMX*3)Ut2&F@ae)KZq^b=*3e1NgSbE5V2EQrEsc1Qzs>?(uE{)ncXq%nm(j&dH zY}Y1TnWk$K)@IP3YB?UL6>v+c#?!8C-If|_2Y%J%`qN3#Ztwg#l&eG6HhV+tw=_&$ z$}9Mrz|mr8N8oan=hMX~uboa?K-RqBf8&tnAfoBT#u#y{Gk4jCZgqN0OE>$|tTvGS zuthu9-rZ^ZXIi#R|6CjYl@W>mXvwM32>8rYe;{9>*Gt*Ic%+)HY`#)Yk zefwv4`S)ff`snoJ-8&4xl<(ewEKck^$w3j{y*qscKiJt6ejiPtuYdP0OJD%??p2I&8B3Wn_U>JJwYsP-?tEIpAP^UFEwd~e`tIH9dNTdpyJpqmyLTTW`R?7DdJ?}p zy03HMcdGxh;cQUMt^L_tlgFb@dwR zTCGq$l6@uDvYrjA#e7s3ds*WZDd&rEGAZUmJIO~4rE6y!9S!qIvRwas_2Z92aQ~*G z_YIY-FZ-8^;*Q2iH5*MUT!~s|i`P)RTfUS0NIh&o1)^xjh z%hoG&4-lYx>o=1epW7rK*|Ka;+Z+zHoc`nM5YL9A(e%R-J-J4Avr>CwIWsO_tERxq zhQ-{hzgiunwRyzU(c0a7&Yg8fo2!#Im+rz#Wc!k0;Nco_vx_-47L1)}UaF0kH5}m4 z+Z90^SQ+-AqpGgRdVTS?r>|bF$h~UD+=7EyR!f2e{QvUR#=S7kZ7E0P@Z*w;(fY3! zm%N9TUlj$;^8!7`MyTVp93bu>fTF1$YscSz?dDSN^>bHi52!Qnm*-EPy?Wjs=hyZB zf-`sh7mK3b&ibzhU-Eg2p%VKMXdR*qV5p}-Z;v0m_W0g!>RWRMsOXa*KnWi$hf;19T%`0@MZ+eso5 zp7XR9ys@(&hyrxEdhp*9#!k5S1~~BMGvSgCuk8iB40blNqX?S?H80L02+#fG9K5kg}&wu_iP0I5}5Agzo<*VTM zuR-kwVuEZO%*R1J><*!)^Pb_lF$1~@2y{xae3jxG66c^ z%LGp@?5eCG#w}4TYt}qjo(vR2W`DwTcrxVa-qPmv4Sr3~g2O8l>*~X=R%w8p9&Em@a}3NfY;6EEjD29X$=u zz%?5&x7l!J_tNa8K>(YwWN7&f`Yc%CGT2rkQWsD*V%RQ7AHW8I6a_1Bs-to+o=7IB z8I4dTIFF1zX?gO;4T)|f*DIwv2y&bB*C5uq`4?M4S6Z)7dA&4NC&4XxV=Sn#Os^5| z#u(u0xu(9~1y8eVF;B9KRm}lVlVsUHPlA7H@FM@&gBQT+IYuv6Ns$%VXj;N=`S5*R zXzVn{=c?@2YfBc^ihtA05aWu5<%%4v4SZI^A_18nbk7%fKArz}8RNcw z{Vx-4{1*&)=l^| Date: Fri, 4 Oct 2024 18:44:54 +0000 Subject: [PATCH 014/658] Results from R50 GH action on windows-latest --- cm-run-script-versions.json | 602 +++++++++++++++++++++++++++++ hardware/gh_windows.json | 26 ++ mlperf-inference-submission.tar.gz | Bin 0 -> 19309 bytes summary.csv | 2 + summary.json | 32 ++ tmp-run.bat | 36 ++ tmp-state.json | 3 + version_info.json | 602 +++++++++++++++++++++++++++++ 8 files changed, 1303 insertions(+) create mode 100644 cm-run-script-versions.json create mode 100644 hardware/gh_windows.json create mode 100644 mlperf-inference-submission.tar.gz create mode 100644 summary.csv create mode 100644 summary.json create mode 100644 tmp-run.bat create mode 100644 tmp-state.json create mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json new file mode 100644 index 0000000000..cff706099b --- /dev/null +++ b/cm-run-script-versions.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_fp32,_onnx": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "fp32,onnx", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_resnet50,_cpu,_offline,_onnxruntime,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "resnet50,cpu,offline,onnxruntime,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json new file mode 100644 index 0000000000..b7cf960dbb --- /dev/null +++ b/hardware/gh_windows.json @@ -0,0 +1,26 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "host_memory_configuration": "undefined", + "host_networking": "Gig Ethernet", + "host_network_card_count": "1", + "host_networking_topology": "N/A", + "host_processor_interconnect": "", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "status": "available", + "submitter": "cTuning", + "sw_notes": "Automated by MLCommons CM", + "system_type": "edge", + "system_type_detail": "edge server" +} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..39adcfa3828de3c5ee68e6f1047f82dfab819b05 GIT binary patch literal 19309 zcma%ibBw1!*Jazb`D@#@&1v_vZQHgrZQC}dF>Twnt?l>wHoM7YH`%|ca&xPms{5Q< zb(1=WFbWDNfdj}42=vm&!wGk+zVu61?1aa8rhb}$0m?;JcQJbAYF;ssuYN9iDRCRN z_AunVjC{AB?IaX1)%ALnxe6aIXo3k9zW^-XT21UHv9Xz5nfOdMyg|ble-Mv2y$E#Jha=y*NI9>du>;%#jxS`C40?i2p4x(g^w8Xklvy z0Q0)f4I_!<{_=}8L;Q-fQKmm~wWe6~9oF~za>8(V`_|0yc|m*=z==*;wdUkH|E|oE zcdY`w2`?~*{T4g8(W?vRgR-4W(#_fC{Py#BV1$Z2YROuW4w0a+$|dCccocfxxHm)m zgzVRduE2c>cwhG$PCmQ%K6ZcF@$*npbN$G8A&iLq&fUMgdAabuG2k34CH%MFt{WlF zu8pTakra3Tw+JWpa)aQ!YrT0-nsS}CT8oDpAx@1*ij=;UZSK7+UHJ1QvZNERK2b?^FQ-2b}TC7X! z@PO7KQ7I9(!^Xi@ow-3uvHhe&?+@c%Z$9gE5<@K8KHAeku?b7=3D37{7w9x9)|oXY zLoqlpP!&gZhmLGcby&4fgPy|{Mau8-8U!!XlbZpW?57HYjXNGLw#PS4UKQ{yzx%aI z|4f!<(8&mL$~4?VET_>Tp4m|d&A|R{*nQ*Z`4ZFdy-nuI)>a^`L9q$=XfJc#uFFTm{J-zd5;LK1h?e1u@aq%2c(YcwpCh0F{2u`V(u0{KAno(kD zNc*Z&=$lPHdL9sj{W}iVudS&#%@xFVUbH#9UXN~+JF3XOyey2xFfc*L&OeLHA8!TR zx$^QLvu0g{kAeFpo|1}mXKP8%WA@2Kq@A8YB@G~1w}&RjO`h#bd8JX7-15|!kf&ny z8H}7h=v~J?vvKb<<>p#WWAT+&Hus+}{F%9$Z>I6Ji~S#Do0ReV!1y?c5%0sm zp1Coq7AO%wI?iKJclB#gL5B5(GxefT64fenNZ}IOCG>m|rjs=#5 zzf50x&zuRpBg`NP*y*tQZhY#PFWx7dq>=^oj21jdIdaR;v^6W zou};{Ru#T8fh2q1G%i&}TtDNI6*m_BFha7w{183`xVStX?+y|DV0^jWW|waN&a#e8 zUl`wPO`V@OGjMw5Mq{|>$UY73YzJNE?!~dO8Nm&Pd@{23@xLI0S|8Gmoh`tFQBV+Z znPE|fK0kG5tEL&WEDLx)f3EktZ6j{^KA&;*5CnN2N~QFT>&1p1_DvnDz=2?ll{MWE z_`bzYP{?g~JQ^|KEf&*3UzuTJgbeVe%kG9G-ng(&>=vdfFBR=$f~$VNofn4rLe2fP zPe$}7f}P7HD}<+YA3bD(jT3oI%|IQK558al?>!q7me>U5j-u{NTM^~>dt@E{hl^m4 z5X>AjOGJgx2N$Vs*`GaSY}FtLX&WNgv}B)u2dO@D&z}tO-+gd3$9G!cNbUD?Zfx)S z*Iz^l3JGcz?wC7CdvX|JjRX+1wGwu;jTK%w)r!RMLy7Lf5kHTwhtxq+5xE>wU>o4MXA))I}|!Ea*oG#!3lw!ape;+@ghv(HMl< zSQnIw(UPH5b2L9YI)%%O6=z-LP7oT>**2%($aFT!?o}7~xXX zD_hYpU?FzQW*i;>8xq!r$c|{q7&c4=Sy~_zBZ^>>LCpplgdqlQ8!h2hk}5DZN3~*I zuF#kcN7o|aoF*C4A|sDNC50!+#Yi1d#t3!z%WRS9-3&vIi)W>6+{auJ^Q#>x0^&qi z7r=e!0ERX2JW76!T|dC9V1#4>#$lvf+}A_fNWMSQRRxuBYmigy64Ds8>ix!$`Rmh? zn?UG@)Jr)E#snKAP;VDglHqyQX~RjeGG&{+fszCh zY7wFE264kB7CS|KFihxUonnkSO8?gUdhFysmOFn&>7o%xl1w#5=VWa4mjg?I@*_D% zxzh4X^LV)gGk~TAw;DJ6QP@fhQ|d!E?UCYT?HSRz1XD^A>jRmvu?K~=fq`DgYG`O} zn$sT}K`!={5=#0kejOx^fvqG3(Mj(q24qu_AkB&vIplEM%^qax=9!=%+s_&+O=#Hu z8X_44U}nho^+J*l%L1uZVT>G#vdCXgL#EJGhDwm9kw>GOyBLDkX&w(BI9Sq|drhG9 zagRG{qef>yDDq5gb4*U=IRkI_C!k}Q$6GSiFoTand8b(bOt2hrnS>lU#_`C<%%NLj zoRI_VE1(Ww?X~_k)AL zWlV6E|233_7LqlJAz*MR%z-?NlEev$w1HfimPhnkVlyHmr~j$6*2sWsR#Us-HLRB- z!P3MbAX5{9*3V+;%Rx_wuxjjD+5|Q)pX9_ESV602N3K&ifS7<9CEfQxF431BB!OOL zSrVsF*D8?$sdKd0mkrZbfz3Q7GOZoR zQm&5yU-_m4MMjxX5X!hXSq^f?x8(f?l@% zg-7aW@HEu$V`iu!!jkgykVmRFvpE1}d3RE%{Jf8>Ec0kj+T1rinjQfc-_^-El?5O4 z-2uXg`enCtRora=0SgOxm;=(3H%rH*eji?TKx8<0bD$?WDef5+)rE&kGsLFm+6w_D z<-|47Q9)^zL!#JK+ny)jFFfoyzZclPF|iAtV{yn_Bq6Kl5eB_1DkiSH<9j_i4Tkt| zu#OdKeL59+aB3JF2BO{mb@sY9uzfQK*?u?;>HuL039-m;x3DnS1-h=RMdfNwh#L5+ zAa*I84P@rVS5~>XT__IqOG&PL}<5vG=QOxOeCUf}Ui*C=^&5hwWkydO2Y78?a#MjpXkxc;g9MtpNvvfH55 zgOlcnu&+Yz;fB`<>2xrezXqROr_=M4!ayR8FOKpy7|GowPj0lsWvdk?Czp^U@7ugz{PwM_j?7n-gN~z(8wAloF0W!pFCB31Z>M z{)>`QTgTw#IohIB@wTd8>6UhtY6z(XPbaD{PZU_x$9LY?*=6v{-Sx1*r& zn&`(vp+0j>Y~a`LQmES)uxX!MSRfWE@@mDy*mSi#i|^|Q5pa4hF9!v&{1!YxSL8_l z!woQg!6$CK|8Wc;y@tx_E-68P&eL`_e^-#hM`3I|8MftTbCCr4>jphztD*u47H8}R zKTWTq5(XCEhU@Q58nc95E;3+tjjYk#Zk67A_3jJ$iLr!LC4^a3V#dxGnf_t9e1+># z__;Ig5^!kR`{_om5%;%Y1jy1w^* z)`)SgAKQH6H>n)hwQ8J*A2pgW$Oyr*ILnlmVk{h6vPqu%7Ao>}{fwX*Ep^}@qfrPn zO39Hu0we$trWwene=i@^F5y8~1vVgHjmufSiXLB*^#wmt)gVp`g&&lS_&|RJP7*X% zR6$TokR&dTA0Ax9hk4*G4+OetNCVtAED(M!#60*00VdcP@6SJC0Z>1fpqMrSm@N=y zl3_n3L`(6QR2ZD{0FW1{fthF#nP8zvI6-*u5Jh{q6x{hiA1svN4o1~e3viPs)d(doGFqCyeG8epl|Cdpnon; z8I8u6A8>p_aCStLh6eyMPZ0!#u+3YAa}V(j^2ReksPGSMKspq8AOTv;gCd5Q5s!@r zy0@Ik!=RPXr}C#s9)ed6_8xW?gx(oe(#$h}yBEezBu)yjv=)n2F7$-(2n!Zwqiipe z^an7?3C<^!cjI{Im!Aggh%o2DfjYv?0rmI?9}q!}nkq5_ufQAt;vc<35k;6#^@`qFKZ(HBFUzLX#e(zVCM3k^8N}B7rLJiC{7D60LL7{+OMLe-cAdL%q^3V!C261-` zZOHSX%%Mg}%F;z`#JNCs6DTjz5tX5hjp-`PhZO7TWaGVG2J499;U>{VDuJaGQ+HU9 zh>enQDSH;cNrwaz1!+J+Kq7FO zfpOp_*vdA`B;t{pQJ@07UrC5g6!wXnnolbtlCTa5f+xq#2#F+&%%H>$gF=BE0B&%Ga)7D(eh0t=u;a=$ z_FcqdfFmd(o4d^N8l{D)0f(R(i{<@dIwP#H%P*{T)6*XsiovTX+H$QMAA(vLLs{`3G7a7!UICPwJA1W3uxn_)Z~MN82P$gwkb?Fek>R52b2R9&YtW6+}X0= zMxq=PMKBj4(Ev%jeY47S!p#0Hj~tM+`fmGmc10g|GVq_rXg0zYw)XrxtJuF??uaws zQ_oo-56!hwZmKOCc{1m~w`!r@xgjX-$H$isHN`VQJ2`u-6t7@e9Y(JRaCpiC>hHaP z z@_YFEZ|F1N=7ezoQHHj&w0b`fD(Lw>SHXh?g@O56AT7lLZ2%D~&%z6>=vcR7$JHFc zdf7{g?$X6;)X4V`3SJ0ow>r?CeF6n+qXd^6z@s|2FZFMzQn*Hv;DdnN${}kYlB8gW z&t^ugIp=0d;^E+FZIyJIZ+{FL+63=jDFui5hwgERk-nY?1!bm>OAg&3?ILl643hso zRaCx(in~t@3k&NaCV+`6)?J4`8WhJNHtNFP12Q4&A!fg|U9sma)(E?64T(9cyEQCy z8$61?ypE}l;IVGeNBthI_n6_MoA!O~O^Le2y0%@p25Gap@s14MMo5ub#Q)M=(I2lKBT`cILk%nes&P(xVg5ZR^~+oLnKoqvUk<-(H9%S z%IgzvkhGtg*D=K~V~3oQck-TCIVn1j@(N#H72-K98l5(s=Gyce%4> zU!cUnHdF1lnBB?3u;z6_9Tzs+CHT1S*}<2!1FKj9fciVx@-n%eo#mzQMGEg0>8l*) zJI1nhm;6y4c}jL#zn)Cyb`f3!4rQ6wumobQyt~kpj<25Gm=w_IC^m-N-6N3aP1vp0 zUaB64TNMNB%0@#siItC&?Mw%S+akG{o8?Ro6}ocgI!n-#ZblqI%asD$8TkwQNpqKr zYZ)8W2A$Tp9^ZSCx83-#x#u=p{0!Q;HcOzX1cdbA^Su74T}uC-)p`DT5b2Yw&ln-s&v_Q*7x&{3@?p5-z>7T&#vD!xY@kL z7ua(+*yCFyMKhQ04Bzwp?fFf0?Psrd%XzP9g zT%c(J+S~8J4^rR1bRPXm_*DUR68aT`H|5C^{k41)q)yDkjx0u4`~oRmWTbA)OYgMH z%sHVI-Qv5(tih@`Cu_|ACZn_3lVMH*wC>6$z1uiu$6i>ItH_KFR3-CXmzs{|*_Ou*l0ls3te<4Y&yB|)>LX|}DMMfCtbpR{N$Z|-jjM2J+Lb}>?c#XkD3;*b0!!xx77nXSv3$ve4~-Hz z8oztVRXm=ul0v+nuhyQ3)R{<>+iHZ{?tg6w^!mAp3Vlq2vpX11u*Om%w2dbo^wF9FdM$;RyUcUxY8i()C#}Xe(zSG`jahV` zru1?JXBp0c#HVC)1xF%uK=-~KMYIrOdaZfBEvP;nZrJ%%b{K!pZc3YB7uCD|ySR6?u$!E+%b zqy{g6KU3@npLiCD^O#4ln^Uxa`ZBXMmGX0Bivq?0RBw3Qv~a9Yvr9g6mh`n5wzCyU zh4|IW=`xPLYi6+NxOo_fZR||X3uh2mMsvqB`{gMbfHgUH_rY+x?m}Gy+jK2+6y<04 z!-*J`d7_K^(i~}BC1z0~d7qcW*qB%WZ;)LrGLpAxpGJflFS#J|9DQQae9Lmz;^mlM zGB^&Ym$slzL5s?X#BMLu6qQyfSO(|iH%J$;^WinV=b$V2rAh)bs4k|qU0T6IHjm}) zxb7vOKJ#aGPRXO)GGlOl2=Ei8jjE zNJY%`!gj^0J!C<340xY5pg%BM*b)Rt+X=asFjde0zgu(L1che7Lj-iafy91-m{ zR@e&fCvM9rD%bzcI>b}N4sM(@D>fNDVqtQ4)3Ifr-pZCr&;3O{-^&;>vxVwxr}vtN zct>*mBqKqt@R1zcAYvz5rUUtsf-{fSE+t=k1 z0a-<6qTe8uwgqU8Xc@wqa?kJTDjn!_`GS zw5TLc`T#3gVHt8QTvrWL;1f}k!-$M`XkZ8w8e9j!A%_qo>lhRe^xEYolA|`WBJG|Y zNCvOZXeO#CR1Yyn=q@hCIB+P%$O%k})DfpfbnxZauEsbz$)}U9Ew?hF9AN;mrui7# z9c;zL(!KCxP|iwj8FMX2>TQ)zRi8}pp7TsQunGVY@_cC==UWppE6I~4)6Uwzn`LgIo)U+e+{X~z z;JPq-8OVo#6Z|>F!`2+FO+^z~iGP=yq;7>!IGeL=W^fGmOJw8+QsU@ja}Nqr8{g^UsmW0iNV0>-2M~igHP1GK(PpW7pQoE3ckA z81C^9MM_;$sU(#P#rno1IHjn!bHg$hzChdF^=fy$V?ukf!uK?uRk-X&Oq*Sfg zu2;^*#&_Jxs%G@PB&ZE8uPT`h@CH6tUYuxhcW+NAtRMv_@B znQ0u$EAeh45;%oiFUJlmuoLT2#NW)=5(6k{E~_o82IxK9|Y z{5T$9dI=#(&e6NKGmPHjN%Br`emUHgk&mBJ%CS5|kL>T0($Ua#KlNLF$z`5?iUoQ= zYBbaK)HsKK5Z>cHeu$G##eXAS4CJ;Z$Dep$$n`J=_wL9!KXrPM29E%xOf&VZH3ke;VP+`C!>NONgQc|3s6p1>mgwG@!#IUQH}GK9@ajJa z(R8*N*uR;_p)16|XDMR1<@_|n0w5XhBHJy0XilyQB1OgK*Nqu#k-`zZhcb{|I!;`4 z+h|-|j`2`oso=v++DvaPXhgoujv*qxPF*FK-m>91m;}@8D9WIyJf{=ulpliTIFJji z%pt0?2iA@+ypt?d!9$`bF zp_fyQM39NWq7QuCnf5`Ofqxn?38vcmZ3&Q<2U&1wCF|B_$d(DnN@2SrmPwj^mPs>v z(q%RvZtvIx&s2D)t)S-SAt-tzz$-%>jC)Z+-+7DQx%;+BtmLGSEb%i>KeSyDLnV!D z;xNbh$npTZD+~AM?EvZLfins>&;yuNFp-s9_#Us_vk|!FS@oF4a$}SRR0G9Slewpt znj2cFD<6Si8?c?&G&^!mmT3Bk2jYN8?fRI~?s1O1e7uxy1xHTRK{`qY;ig*&474Xr zLmY!@PL(l7(#9e|Vv_VpKk+~iBVSeU11JA!nlbFZsm)N_r;;aa_uz;ubzwK*jFov+ zH=J1dgNM7}W?8!T=JgSnff!F*ZrEckldvpJ;dixZ=QtuIJMQsXUV?DkM_aCf8PfNI z1r%=9XIV)oM{8(ZN-N$$IF7T)6Kav8`u<2!>Ox&`4dk>4`Y5=+QA<^^v`Vj!Sr4LW zp7Y>&Io(7gB=;0ewIDL5#cXetno+KX7Q-9V46ugxT6Wh zTgmzg=D$P%fVy)g4-yh9IVh)KiRm$AJI^)X=bJ@%ulYl`oXN4kss~#x$UY;TjXDmp z=kgKKVw&#LgM3K6|11kG#VjUu#+W@eHhH~}?Pqte;vmR59^|{?-KYF0+nj^&p0H;S z?^PU{Cvu=iIeisGu-(7b+ubfT$CfzN)%l_@XC2;%6_G@nvu|^{=Il7Yj4rd4#SE%9Aa$p?hpB+?Xx-+ydsdXG3DjmZ&b@`7t2En`Y0Ckx=%akDgom zfU7Z4w1=vHO{5O@wIoMyg?}O*a3iPQ;d{>Wm|Vq4{O}g5RSes0F_Gob(2^e&fVV+Ze!pbo^)CVR z(GcTo5)j|Ws^cEhYzXOw3e8W5`HEM#{q-ietLhq}T0msvJa6010oJ7MZ@pVgc#ifW4i zJ)*prr-sIAmZU{qER&^yyH5*J zNOYqr(AaKYQ*PFA;G4L|nkF3tqk4+ovmmw)kzaObigzg4$;X)*5#;=IuxB3&e=x%) z?Y!J(mK)<-xD`r3s?VE6Y(;(gQB`UMuYz_-_R*!hlBSh$pGy!pJI@6_@0=vCa2(d>#TI(oRk`}MNk?~`%uEZ&4Ng`8PB1M zcFI+vo*2$SdNkow(F36ACRILfd^&}aioGh4so^7PC${T|TWAFT)q!k>Jtvk$Fetp_ zfva43JO?60vgK^a{t(SDx=J4X65NA~$JLYhfU)5u7lwPAn)O3Y3GX3JAoH{le}BjY zK)Pq@E?-Lhoc)NxypMIEqFr1L$?|-Z?a0^RzuR|bIZ$qvItj#Rq=qtGD}H-M6p997 z_PGBPco?eVe8+FBvKv^bsGt`H)Vhaqt;1dmSy|S?46^qVyHY53B{VmBFAa(J@~7h9 zdfH=DTwvsS`eX1OjY}%j+Fl3g7XnIj!<~GUR%u|Hv`0n=KcTs-&0z%jY0LB$q8SPJ zJ|x_{F4R9RoWHGXD~&lV>79D|QEj`we@RV|9Dk|5#JD~8`o_NNDagzJ?)H8i%nj6w z^=-nG$m=S!NX%Uw+v(0Gt(VAKxlWoL*yCA64Dv(l@rx8#T8Nf;;kpQ_HkYUS12EE_ zO01IK6ts5)A#rBU$}%8--`e|p#%$#7KPT*Y|GM1iyoR*v^?Jn9ixZl$q;Uj!fEyR` z+2r-6N>sAg<@;#rD)+q{G32a}(MH*rW~PM+@nFm#phJww3OKCT2Un8g_MP3~`hbD> zhbYv66?+GaWGn>VVB8b{4yk^d9x{^eJw6d(Odl=;v<;(@8F_NDBKPd#F=TapbL?6p>Py;_y3Ea5h`w#xu$4u zbkC|(E=`h3_!vv@efT5&Ec7!nzS&W=j53~fX!D@fa9GLvDRX+djBr7!oXRCJSd zz&Z=fScQb*A7x#EngWmxWv>?>Jo&$}G%ysf>EQMwP3G8NEzxz@IhR^S{Y_+XD;0P{3={o>vK=mx4)D=5l^f3Y&_pKLALNxehcUx!e0|Vk zuY6l{<_(Qgr(H{_MOr(Mzsrd}kc zx5m*dVMYt4v;vs&-9vzj+b7sh{jvm(0&&O1$rLIthp36z2u+BT{bN8Z!_Xd}3!4KH zOua{RtP}Vr;O+p(1cVpVW7-i^dLcdX*D#LjppYPQf4}Mh(61cnVVGfKV>n13NT8jx zwwk)8MZ=|On7TL#u|FWCzY=7~QS}uNS|}q7EfozKkZ4O-x3D{sLO>g-pgPb`!>mp^d<`vNeFMm)e;; zvb12=2pYlarW!Vuz>G?O7k=l5lslOzb^zJ#WW47CmqYARtJFpJxn+nehZ46UqXM4w zIm<7+6Qfh@*CjEDp-LkKYi0m4GXrp|;A+HC#=xWg4c1hUV^kV7`r&ER2?6+!?7{3! z9wmZ$dz|u|8g;5A4%M_#mYiD5+6Vdx_W(MFSiQvydjL98^AFgw_!E_d(%(KTa3ieP1uERlmeMQr$E1A7?r zG7O-}T4(@vJ%`uK52eRhrGV7vZL&_0tSt@h@rr2SRsPDNl#`4v-NE;Lt^9{h8Pip# zQ>kiP4pAIv!1VvY))etmOq^#OQ| zWmo8;$2ni!1(=jtlahW-9KFRJ=P5?mQcyjkq5C+gTIQNKLM*O$&l`;RPzaxK9!SF$ zpPST&!6vkae9wXX4Ncncwf-9!lE6f~ye3L(8W~~Q`qCfxcW|B4>zG{+NT*s*$U-Lr z>5CB^8k@bf1dm>^^~O;xiu3HWmAwvO4X`K!dJU4X>iviO)Sk79{tA)vEVi$l4po)C zEb%p?`U+97^<>8Mo_G#Xj)BJCY>-rE)~T&;qX1VMQIhrEGinU!_ZE%6sCybviah7_ z&F-!nyuv=lChj=t!6s1C1NM2su>Ko?IlgVEr^YDangp6x+>o`>N)=8|s}}^={*9ub z1%gEzF(n5y6afnB2gaQ;jY6 zmKDrVJmYSwN}g9lL|b;2J2@`GT9Lg657(*@$WgrF&ksm`oZa&YJjAdaw>c_CfF{}O zJghruxmOMYTcZ2OSf7@`M^b#35UTa>=UK!7C#&;QdK3e7a?x2Hou_kjAy>p=mk1E_+$}J!w<Q zqfE1&^YC~VvYF-w=ZkN)h%o*T{B{oG@Ml34Vb2p3;>LLPC+dnyV3{2p#PC3z4oHb7 ztT2rSvpk+*tM70on%!}uR1V?8ka~ez2#yajU?Gx2MC2WjQTG?^>dvvC#5xBUTh@lktkU#6v&tnnq;mKh=&^# ztPnE6aumyeH(()E3|yz!!9b+%psOKAA`s;fV>bovn(cr`+Y>{lLZlRE5Z@DXg*_{B zK(-$c5ew1~=tmmhnzMucCf+6@+|rIoLl+Uz48J4xME18-3`PR|dH;~~_h2(ugiR%@ zL+eAA9?0ytE#T}T3o5aB7+PD76@W3I9lvL02|T}M@+-CQY8n#p3RSv8&m#+HuYc1OE=QIK`mqW zjUNX|y7wm8m4dI%wlhc8b%XcA@96Iwx`hv~6}|9scVOQ#jy99n1LsntMv7SG+wY!q z2WDx)MUhBH)Cyn@CLv3J8KthfylD;t3ZtC#r%VpP9N4=JawlOVC=CtU+T+s}IQFo(!A#71!Au7R8A(rqV zVo$!V0fg&6K;`f|L_7be=8RyHJYu4#JVWtv6b# zv*G#r1{p-JH(>@MMA(05Le)SM^Jx3ch(O}@20?Ld)u6hzfUhmez_F+Vl>{{Rj}qCtEMP)zZFnMlEW!2)p`e+W<~cuS5mWPoTjSRfH$k7)5xncNL_ z{`pu3&kP?L1FO4v%nj*MjTaCXz3uRzUxINm1LFuE?iK4j$8dD0nU+Iu%Oo zyd}VekDAH|k;a@InDHmtBFa|3Iw)ay7_6#0BtRNlk#pn(fH=!T?$}9;A8+|b+~*YP zc=siq3{>qGV4%M}d92XOGldK(UfBf?q(<`(kAQ&N#e)c(r_Fp_xxYJgqDS0~x8V_i zwtc_cu$Vo++yG*3U(d&!-|w@7*QNc+N#i?njg1~agXCGoWeOzT4|VeDxB)ihKQjdI z#Zt!pm&ygsXR!L<_SO?W4^hji_iH~5{u5K${=YRf_uvoT{pt_i)L(iz(;qvY?+n4N zFN9*zn|klxJ2akOH12GZmz4g;y@4F#-ARvWL>iZfRlQw&xeAzO$Bkm(dV1`*zJUU8 z6#Z=<5c5AW0lks5>9r9Xz0u_(4oYf`S2D%bITq+tRkpZT`d-G(V~Ab?$g6uj#16Cf z`|^kHxms^~)OSJ)I-d7$t{jE2x5aovyRUJ@hLf+m-luOuZ~W8G}>0ko^ac z_Uf&Hcjin)vqgQSk?`2FcUIP<_EDp97w?5asiaxPetj$qpDs;jnKNoGZL%xf%ojd) z@kd_6Y!lMIXBD*1&zc~TYA;TbuA6w5% z)t)6jv=uo{%(5xdne~R*+53lsp8dBxR%+0oYpfusz4l){W3Kp$9aukGhp z-rxKwHBy4rSKHD|9k50Mt!!dsV>aayaq>4|lnF|xr<=jbU@~^_`If^KRZ80A-BaHh zWH&qit#+2gZ`z~!48PCro;%!?3;eRhx7Eq;(ng?-4dfzUwE*zgm6=PR=aAC*_!Hh>1uJbFQi&FT{kw|yy-N5 zh7TdmI2d^{Vp9;jYk=a_OdXxLlN=5K$Tu4U3FfKtJCz&JLb zgr8sWBVY9RemL0d>`hf%>U5iN+LOMA^BrA8>bt zB{fKk7ovM~Z%W&PY|OwRZs|O|E(bnoF+Br6J|4Nx*ew;f5tuCR6Mn!%TkF&+tfhax zuF99ZHVI_!e??kzPZP?R`vD$}Xp6fxoC7kM6*=wdUiRoK__*s_Fvr=qE5*v^<3$x_wwuJhU=wF{)F`TUa0)BHGKxJldHc( zbx(7dOhS~4TTJfp+1QzJvB4Dm2HShMJ7#Tb zc5h)iK6hkn+No}je$0$8&q`HPA$Rm#>i0E0g?Ro^hu?3Z%XrI={phTG@b6&bWZ+-m zZe11Lm-V~*zcv4`DyJtN;&tq-OwM=X5EXWi=Y5BQsCz#--!IN+{>TaP#<*i@e1Wg4 z!_4oOajM8eeNKK*O9@aURh@^2-lIao%fCtYxre+`rMF&X%$bq0!Fwb*5~QDYCXdEU z|E81^y1sung29`!dAl+Reuxl+dT<>IEa4XT>fV!co0^wXmbu;)WXBlJB3h`2+{lR( z7n?l2&VDs6$L}-F&J38j*CB#t zHkL>ItI4)ux2)~*!)zL+2YgpJ8t&zJ)OXLH<--bRA4Dj$vdu3c<&Q&rT@TN@COiK& z-c$Ed&S(sL_I21sK-%7bn+SZM)Z}@QoV`)iPG*#E8q_WmKD}{~yswKKbi1H1C`R45JiW#tBtaDr5rc^t1aX9E<=kK=k ztW7Up8JmfF+>)to0fX&eD}r9CZ1SXqvB*X!GP!(v7%BP!sbF-BdPp1HMkHUk z1JBbky8+LKjMk|xb=uIihA~IW*~a!e3x`xNGNxBh%L7RUa4-Cez7(a=FT1l`g^`iQ z-KtFTcHch-Dtl`lo*cIi{B#Q3PH&kUrMj#JX$Ta!6as5Gdltx68RzN6eJ1l%nx)1( z0gYJ{wnI=96F-_e)kjqlL+w5+uA_3}U4+84#b77WxXQp5nq2KM_LVj0&CuP*f^l)1 z#FmAuU+tL!+rvq_B@Fxt1ME#Ue;4SU*KR^<+i~+_{v1`0T|;G0m7Yv&syDZv4z@QJ z$`5cuuOntnS$!&Vuh5Zy0_t3?q#vD=uqQ=L?*|Gs&Hn~bs^2c6 zzymvLdhRKom)C`0 zrx2a#UrIs0hO6=HX}VHD`R8i)2P3Oms--I3-mk?wtP8`CwcW^k_|}Cw`AqPixvyNb z>pT1B1K<|T$Qmk8>*a@@Kud9R{g}uuAP#dOmTduFY(PCqb3Qmu4B7_A0;I&L~S#E519<1V?Fe{ZsuPKLPwYOdIhk%zO9$Fb(46ZM%Z#oa(>m_y%=m zq=>q`C`hW3Z}|Cy6Rkk&l?%dQ_P{Dvq8`F=hyT03^E?vbQq^>!?=hXL&l}$zh8gB6 zYo_Zv9?$EU_i(4^Ns^`hF_$lhBG312RH4tWiEcrkYA*ft8vF{<_OJ?Q9Z8J;{Qz1nzb z;&o~0V#v(>s7U~3V^c!TOp=xY$;fTkzrWzM`*hfYkX)95d=q>@zd!b82lal0t=)c8 zQ;>wAN#1YyygBPB{sE(Swb1|mw-`H0iOt+r&$}zJenjTf6YdqlJnp9&ElsNA9$<_v zp!_e*`t2?-Xb@(oaG4}zJ{m0=V&F}`nLEY~?o>eXdg?xV&Eea)GcT>uHsOBs)4n=` z0xq2|Z;MoW`a08z_v;ReK(DjbSEg?`!u9|Wr={PkPU*LBaOPWW(cW8Y`FCsoH@WdL z{{ph1B0Pmz5TdKQ>+h5xL(T^G0H!yh<6mqv3IqaQ|MNa9Je)4M|ELc}R{XgSWp#hM zLQ3MJthcdBIB8EN?Se_y9vi1lz0KM)YpVwFmM03iFf}$i1%Hj+xrmH5U+14lY04h& zH*4meDjj!Td~8^JZ;yHt>eVjS>war~nlyeNOE=Vff8$MG2euxwo3m1;V+gtM(*}k1 z0#~>I>G^#Zc0=WW{pT_1tLOW~APLfsS3!EzA>;db-@;bnlNofKiBSGM{^0p@M}Uzq zl_e*2@q`3RzCseAq9ac|ofwM(vmD;{dt@6@aVu<>ayrpt3$Z6ta97bfHMymxq=d+y zY1Z64f6pR5nurBMX{nemq?WSI1PGL}i~Dwu=&ryVPRPMQA{<+0h}_NHFolt zJ9`7s9MSnoY#;fmhl;uGk)8|>Bn)}Kb0|54WtTrlB?KkWF5U%FQJEA?y?hr+ryB5R z&s_W$;d>7^qzV@JR9HKBu!G-gVVFqB{NJsWK9~Q*|HTwADXvYFnh)w%lBSO<4#Q%)oGwA*hWVXq+dJ9UiR_5~M zqlY3tUhK~0@JjAWYxA;SYj6D#b!~*T57R|u^Pu`L4L(@=V+k*y@J92lj2B4t zAED--%F$pB4g;10l?br#QVOZ#kSp*CwBk*=B~{SH*pKTOSQr@-a;;6oVg~FJS}t>B z=QdwJp7qo+j<_-*{9!5*GUQ>r(WqK;-#&lyb`Yw&X<%i`DpUifoh;IEER|6bKCnnK z!^}o{Uwgr&6()>ht`)dUJm$v9*V1Ywg;I($@P~yoA_`Gq@qd5pzX5k@WZzVCn~eL= zft}fEFiI-uXf6gI6d8)s2)Q4JB>8wa8RUz3?J=Phs5`8t*jLcI4|gdwc2hU)E-~5Q zr-hwkhhf}Ao|}tKVsC6OoQyeZkMQ;kR?9sd_qO~q>TbjCq+d@dV zbYa+>6S z0@)cSvvTE|V^LVkhKmoQ4?Az%_#v$Z1|3B$0~mNfFRUE?T#sEw*sghiMZ%b5(l~-h z(`YyzR&^G);{pe+NmUzs6qpY~u=J1z4SrLSQ_*rbRhNlSTpG1?&^9~ArAK;Y*{)5x zGELVetj(Z5)p9&gE8vz?ji+7Px-B)<4*aUi^{11f-QM|gC|8HBZT5!PZ)upilvnUK zfuqIHj=<$C&!>w~UOSz(fUJ4L|HdKDK}6GwjWOa@XYR5O-Rkt1mTvZ^S#2QuVT*RI zy}Q%+&$Mit{<${(DYlPE~R1_Ze2_8 z?RRhAV1h_G?_U4y?Vo@6-v9p1-`>1^{%Y|3(^qu`#rxyrX9k0tV;(J+IUN0u*H7R6 z8D9RqnTb9+J$d&I12E;gcOZ)sJ5O>@#CPvbU%?M{Hih3uQ|Rm8y~`39K)rhxSs_f3 zh|p>)QwhJMSF4Na;?Acf3<7Z>*D}kpq3_tc(=>`0D5B2>j-JG;NmBFtvUFFT9;p z)5W~rfpoBS3mwZ1ov+`df!mV z`m%qyDDG&SRI|~v!j-6Xws;N2%hf)Et5cdc4=p|#&!B*F3fO8Vm$TXFXic}9w`{#a z_W%L9w|+Cp@wrX%kuA&iw9VmA%jrM94)JU_8cjbe(UWU*H!HO_mNVn>wQ35yY*@_A z`m5DJTAN2q9j)EX=iFI$w7EKYbLlR;M7A#}1|F^#|C43*fAKxjhPFR2Yqr=yY0s^NPZyaHhf zl0`LbmR|(LJQ>>$)A^6VA5LF=^N&BQR*pbldcH_uz&n0?*_%Di#&SHG&dP84bp+}3 z+-5%pe?NT{yn6W>kPGr@R$di>->%N|uNR?QLiFd8<<#_t)StNrO z$#^)r2)J5KV6)&&WoI))i}ix13xEuiC4ZJe|%$Y4ft|@OvA&o#W1(K;r@7;@o~e#fBy5IX;Pj)dWaVwEMEo3e+_Ck z5EEqMU_K7&VOMZmEPt6TAQg_~f@A`8oiC2<j-^_bQZ z6^55kt>Q#aSo72S<(r=tLtE9n2C4ggTG?ia#;^!3riZlxyCz1(jMkAC7 z&Lg8wTAuuIL!ukW^-Adug4`zkHHfut{>7HimDVd%UN4Q+NpOqa7z=7F(`&@LF$TDL zuBq>L!P6{T%#-Y5RdWE;Bw6;)li;5kyvTp{;03UHj?v3iQe;IonwIceK73yn8vOHr zJ_*1bjbbJLxhnhh+LDE};@>ng#JHkixgrN^V;X$QX-SY*WPv`$##<;Iv|I36M z|K&35{J+ofkQpXAML;K^l8>kOJ-@Z*!Gr$~GKKno0Hy)}A|FHk literal 0 HcmV?d00001 diff --git a/summary.csv b/summary.csv new file mode 100644 index 0000000000..97fdfaf568 --- /dev/null +++ b/summary.csv @@ -0,0 +1,2 @@ +Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.2863,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json new file mode 100644 index 0000000000..0f823304de --- /dev/null +++ b/summary.json @@ -0,0 +1,32 @@ +{ + "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_windows", + "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":17.2863, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":1, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.19.2", + "operating_system":"Windows-2022Server-10.0.20348-SP0", + "notes":"Automated by MLCommons CM v2.3.9. ", + "compliance":1, + "errors":0, + "version":"v4.1", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + } +} \ No newline at end of file diff --git a/tmp-run.bat b/tmp-run.bat new file mode 100644 index 0000000000..652af1eb8f --- /dev/null +++ b/tmp-run.bat @@ -0,0 +1,36 @@ +@echo off + +set PATH=C:\Users\runneradmin\CM\repos\local\cache\6e338fa30e7e4b4e\bin;C:\hostedtoolcache\windows\Python\3.12.6\x64\Scripts;%PATH% +set CM_GIT_BRANCH=main +set CM_GIT_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\a647227f091b481b\repo +set CM_GIT_REPO_CHECKOUT=main +set CM_GIT_REPO_CHECKOUT_FOLDER=repo +set CM_GIT_REPO_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\a647227f091b481b\repo +set CM_GIT_REPO_DEPTH=--depth 5 +set CM_GIT_REPO_NAME=mlperf_inference_test_submissions_v5.0 +set CM_GIT_REPO_PATCH=no +set CM_GIT_REPO_RECURSE_SUBMODULES= --recurse-submodules +set CM_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 +set CM_MLPERF_INFERENCE_SUBMISSION_DIR=C:\Users\runneradmin\CM\repos\local\cache\f6b16c5abc894157\mlperf-inference-submission +set CM_MLPERF_INFERENCE_SUBMISSION_VERSION=4_1-dev +set CM_MLPERF_RESULTS_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 +set CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE=Results from R50 GH action on windows-latest +set CM_PYTHON_BIN=python.exe +set CM_PYTHON_BIN_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64 +set CM_PYTHON_BIN_WITH_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64\python.exe +set CM_PYTHON_CACHE_TAGS=version-3.12.6,non-virtual +set CM_PYTHON_MAJOR_VERSION=3 +set CM_PYTHON_MINOR_VERSION=12 +set CM_PYTHON_PATCH_VERSION=6 +set CM_PYTHON_VERSION=3.12.6 +set CM_QUIET=yes +set CM_TMP_CURRENT_PATH=D:\a\cm4mlops\cm4mlops +set CM_TMP_CURRENT_SCRIPT_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github +set CM_TMP_CURRENT_SCRIPT_REPO_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops +set CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops +set CM_TMP_CURRENT_SCRIPT_WORK_PATH=D:\a\cm4mlops\cm4mlops +set CM_TMP_PIP_VERSION_STRING= +set CM_WINDOWS=yes + + +call "C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github\run.bat" diff --git a/tmp-state.json b/tmp-state.json new file mode 100644 index 0000000000..e03109d44a --- /dev/null +++ b/tmp-state.json @@ -0,0 +1,3 @@ +{ + "docker": {} +} diff --git a/version_info.json b/version_info.json new file mode 100644 index 0000000000..cff706099b --- /dev/null +++ b/version_info.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_fp32,_onnx": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "fp32,onnx", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_resnet50,_cpu,_offline,_onnxruntime,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "resnet50,cpu,offline,onnxruntime,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] From 47575e9fd1f90e93f8be539ff9aab20677c1c303 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 00:23:20 +0530 Subject: [PATCH 015/658] CM_GIT_URL -> CM_GIT_REPO_URL --- script/push-mlperf-inference-results-to-github/run.bat | 10 +++++----- script/push-mlperf-inference-results-to-github/run.sh | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index bcfd791647..7800479ae9 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -1,18 +1,18 @@ @echo off -REM Check if CM_GIT_CHECKOUT_PATH is set -if not defined CM_GIT_CHECKOUT_PATH ( - echo "Error: CM_GIT_CHECKOUT_PATH is not set." +REM Check if CM_GIT_REPO_CHECKOUT_PATH is set +if not defined CM_GIT_REPO_CHECKOUT_PATH ( + echo "Error: CM_GIT_REPO_CHECKOUT_PATH is not set." exit /b 1 ) -cd "%CM_GIT_CHECKOUT_PATH%" +cd "%CM_GIT_REPO_CHECKOUT_PATH%" git pull git add * REM Check if the CM_MLPERF_INFERENCE_SUBMISSION_DIR variable is set if defined CM_MLPERF_INFERENCE_SUBMISSION_DIR ( - robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_CHECKOUT_PATH%" /MIR + robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /MIR git add * ) diff --git a/script/push-mlperf-inference-results-to-github/run.sh b/script/push-mlperf-inference-results-to-github/run.sh index c1d66f0278..1eb4f663e4 100644 --- a/script/push-mlperf-inference-results-to-github/run.sh +++ b/script/push-mlperf-inference-results-to-github/run.sh @@ -1,16 +1,16 @@ #!/bin/bash -# Check if CM_GIT_CHECKOUT_PATH is set -if [ -z "${CM_GIT_CHECKOUT_PATH}" ]; then - echo "Error: CM_GIT_CHECKOUT_PATH is not set." +# Check if CM_GIT_REPO_CHECKOUT_PATH is set +if [ -z "${CM_GIT_REPO_CHECKOUT_PATH}" ]; then + echo "Error: CM_GIT_REPO_CHECKOUT_PATH is not set." exit 1 fi -cd "${CM_GIT_CHECKOUT_PATH}" +cd "${CM_GIT_REPO_CHECKOUT_PATH}" git pull git add * if [[ -n ${CM_MLPERF_INFERENCE_SUBMISSION_DIR} ]]; then - rsync -avz "${CM_MLPERF_INFERENCE_SUBMISSION_DIR}/" "${CM_GIT_CHECKOUT_PATH}/" + rsync -avz "${CM_MLPERF_INFERENCE_SUBMISSION_DIR}/" "${CM_GIT_REPO_CHECKOUT_PATH}/" git add * fi test $? -eq 0 || exit $? From 386e2fae65701403672fe1f0df3308c8907e249a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 00:23:40 +0530 Subject: [PATCH 016/658] Revert "Results from R50 GH action on windows-latest" This reverts commit 7bac0317e16a69e9a1823d5e8725b24860c81d6d. --- cm-run-script-versions.json | 602 ----------------------------- hardware/gh_windows.json | 26 -- mlperf-inference-submission.tar.gz | Bin 19309 -> 0 bytes summary.csv | 2 - summary.json | 32 -- tmp-run.bat | 36 -- tmp-state.json | 3 - version_info.json | 602 ----------------------------- 8 files changed, 1303 deletions(-) delete mode 100644 cm-run-script-versions.json delete mode 100644 hardware/gh_windows.json delete mode 100644 mlperf-inference-submission.tar.gz delete mode 100644 summary.csv delete mode 100644 summary.json delete mode 100644 tmp-run.bat delete mode 100644 tmp-state.json delete mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json deleted file mode 100644 index cff706099b..0000000000 --- a/cm-run-script-versions.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_fp32,_onnx": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "fp32,onnx", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_resnet50,_cpu,_offline,_onnxruntime,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "resnet50,cpu,offline,onnxruntime,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json deleted file mode 100644 index b7cf960dbb..0000000000 --- a/hardware/gh_windows.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "accelerator_frequency": "", - "accelerator_host_interconnect": "N/A", - "accelerator_interconnect": "N/A", - "accelerator_interconnect_topology": "", - "accelerator_memory_capacity": "N/A", - "accelerator_memory_configuration": "N/A", - "accelerator_model_name": "N/A", - "accelerator_on-chip_memories": "", - "accelerators_per_node": "0", - "cooling": "air", - "division": "open", - "host_memory_configuration": "undefined", - "host_networking": "Gig Ethernet", - "host_network_card_count": "1", - "host_networking_topology": "N/A", - "host_processor_interconnect": "", - "host_storage_type": "SSD", - "hw_notes": "", - "number_of_nodes": "1", - "status": "available", - "submitter": "cTuning", - "sw_notes": "Automated by MLCommons CM", - "system_type": "edge", - "system_type_detail": "edge server" -} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz deleted file mode 100644 index 39adcfa3828de3c5ee68e6f1047f82dfab819b05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19309 zcma%ibBw1!*Jazb`D@#@&1v_vZQHgrZQC}dF>Twnt?l>wHoM7YH`%|ca&xPms{5Q< zb(1=WFbWDNfdj}42=vm&!wGk+zVu61?1aa8rhb}$0m?;JcQJbAYF;ssuYN9iDRCRN z_AunVjC{AB?IaX1)%ALnxe6aIXo3k9zW^-XT21UHv9Xz5nfOdMyg|ble-Mv2y$E#Jha=y*NI9>du>;%#jxS`C40?i2p4x(g^w8Xklvy z0Q0)f4I_!<{_=}8L;Q-fQKmm~wWe6~9oF~za>8(V`_|0yc|m*=z==*;wdUkH|E|oE zcdY`w2`?~*{T4g8(W?vRgR-4W(#_fC{Py#BV1$Z2YROuW4w0a+$|dCccocfxxHm)m zgzVRduE2c>cwhG$PCmQ%K6ZcF@$*npbN$G8A&iLq&fUMgdAabuG2k34CH%MFt{WlF zu8pTakra3Tw+JWpa)aQ!YrT0-nsS}CT8oDpAx@1*ij=;UZSK7+UHJ1QvZNERK2b?^FQ-2b}TC7X! z@PO7KQ7I9(!^Xi@ow-3uvHhe&?+@c%Z$9gE5<@K8KHAeku?b7=3D37{7w9x9)|oXY zLoqlpP!&gZhmLGcby&4fgPy|{Mau8-8U!!XlbZpW?57HYjXNGLw#PS4UKQ{yzx%aI z|4f!<(8&mL$~4?VET_>Tp4m|d&A|R{*nQ*Z`4ZFdy-nuI)>a^`L9q$=XfJc#uFFTm{J-zd5;LK1h?e1u@aq%2c(YcwpCh0F{2u`V(u0{KAno(kD zNc*Z&=$lPHdL9sj{W}iVudS&#%@xFVUbH#9UXN~+JF3XOyey2xFfc*L&OeLHA8!TR zx$^QLvu0g{kAeFpo|1}mXKP8%WA@2Kq@A8YB@G~1w}&RjO`h#bd8JX7-15|!kf&ny z8H}7h=v~J?vvKb<<>p#WWAT+&Hus+}{F%9$Z>I6Ji~S#Do0ReV!1y?c5%0sm zp1Coq7AO%wI?iKJclB#gL5B5(GxefT64fenNZ}IOCG>m|rjs=#5 zzf50x&zuRpBg`NP*y*tQZhY#PFWx7dq>=^oj21jdIdaR;v^6W zou};{Ru#T8fh2q1G%i&}TtDNI6*m_BFha7w{183`xVStX?+y|DV0^jWW|waN&a#e8 zUl`wPO`V@OGjMw5Mq{|>$UY73YzJNE?!~dO8Nm&Pd@{23@xLI0S|8Gmoh`tFQBV+Z znPE|fK0kG5tEL&WEDLx)f3EktZ6j{^KA&;*5CnN2N~QFT>&1p1_DvnDz=2?ll{MWE z_`bzYP{?g~JQ^|KEf&*3UzuTJgbeVe%kG9G-ng(&>=vdfFBR=$f~$VNofn4rLe2fP zPe$}7f}P7HD}<+YA3bD(jT3oI%|IQK558al?>!q7me>U5j-u{NTM^~>dt@E{hl^m4 z5X>AjOGJgx2N$Vs*`GaSY}FtLX&WNgv}B)u2dO@D&z}tO-+gd3$9G!cNbUD?Zfx)S z*Iz^l3JGcz?wC7CdvX|JjRX+1wGwu;jTK%w)r!RMLy7Lf5kHTwhtxq+5xE>wU>o4MXA))I}|!Ea*oG#!3lw!ape;+@ghv(HMl< zSQnIw(UPH5b2L9YI)%%O6=z-LP7oT>**2%($aFT!?o}7~xXX zD_hYpU?FzQW*i;>8xq!r$c|{q7&c4=Sy~_zBZ^>>LCpplgdqlQ8!h2hk}5DZN3~*I zuF#kcN7o|aoF*C4A|sDNC50!+#Yi1d#t3!z%WRS9-3&vIi)W>6+{auJ^Q#>x0^&qi z7r=e!0ERX2JW76!T|dC9V1#4>#$lvf+}A_fNWMSQRRxuBYmigy64Ds8>ix!$`Rmh? zn?UG@)Jr)E#snKAP;VDglHqyQX~RjeGG&{+fszCh zY7wFE264kB7CS|KFihxUonnkSO8?gUdhFysmOFn&>7o%xl1w#5=VWa4mjg?I@*_D% zxzh4X^LV)gGk~TAw;DJ6QP@fhQ|d!E?UCYT?HSRz1XD^A>jRmvu?K~=fq`DgYG`O} zn$sT}K`!={5=#0kejOx^fvqG3(Mj(q24qu_AkB&vIplEM%^qax=9!=%+s_&+O=#Hu z8X_44U}nho^+J*l%L1uZVT>G#vdCXgL#EJGhDwm9kw>GOyBLDkX&w(BI9Sq|drhG9 zagRG{qef>yDDq5gb4*U=IRkI_C!k}Q$6GSiFoTand8b(bOt2hrnS>lU#_`C<%%NLj zoRI_VE1(Ww?X~_k)AL zWlV6E|233_7LqlJAz*MR%z-?NlEev$w1HfimPhnkVlyHmr~j$6*2sWsR#Us-HLRB- z!P3MbAX5{9*3V+;%Rx_wuxjjD+5|Q)pX9_ESV602N3K&ifS7<9CEfQxF431BB!OOL zSrVsF*D8?$sdKd0mkrZbfz3Q7GOZoR zQm&5yU-_m4MMjxX5X!hXSq^f?x8(f?l@% zg-7aW@HEu$V`iu!!jkgykVmRFvpE1}d3RE%{Jf8>Ec0kj+T1rinjQfc-_^-El?5O4 z-2uXg`enCtRora=0SgOxm;=(3H%rH*eji?TKx8<0bD$?WDef5+)rE&kGsLFm+6w_D z<-|47Q9)^zL!#JK+ny)jFFfoyzZclPF|iAtV{yn_Bq6Kl5eB_1DkiSH<9j_i4Tkt| zu#OdKeL59+aB3JF2BO{mb@sY9uzfQK*?u?;>HuL039-m;x3DnS1-h=RMdfNwh#L5+ zAa*I84P@rVS5~>XT__IqOG&PL}<5vG=QOxOeCUf}Ui*C=^&5hwWkydO2Y78?a#MjpXkxc;g9MtpNvvfH55 zgOlcnu&+Yz;fB`<>2xrezXqROr_=M4!ayR8FOKpy7|GowPj0lsWvdk?Czp^U@7ugz{PwM_j?7n-gN~z(8wAloF0W!pFCB31Z>M z{)>`QTgTw#IohIB@wTd8>6UhtY6z(XPbaD{PZU_x$9LY?*=6v{-Sx1*r& zn&`(vp+0j>Y~a`LQmES)uxX!MSRfWE@@mDy*mSi#i|^|Q5pa4hF9!v&{1!YxSL8_l z!woQg!6$CK|8Wc;y@tx_E-68P&eL`_e^-#hM`3I|8MftTbCCr4>jphztD*u47H8}R zKTWTq5(XCEhU@Q58nc95E;3+tjjYk#Zk67A_3jJ$iLr!LC4^a3V#dxGnf_t9e1+># z__;Ig5^!kR`{_om5%;%Y1jy1w^* z)`)SgAKQH6H>n)hwQ8J*A2pgW$Oyr*ILnlmVk{h6vPqu%7Ao>}{fwX*Ep^}@qfrPn zO39Hu0we$trWwene=i@^F5y8~1vVgHjmufSiXLB*^#wmt)gVp`g&&lS_&|RJP7*X% zR6$TokR&dTA0Ax9hk4*G4+OetNCVtAED(M!#60*00VdcP@6SJC0Z>1fpqMrSm@N=y zl3_n3L`(6QR2ZD{0FW1{fthF#nP8zvI6-*u5Jh{q6x{hiA1svN4o1~e3viPs)d(doGFqCyeG8epl|Cdpnon; z8I8u6A8>p_aCStLh6eyMPZ0!#u+3YAa}V(j^2ReksPGSMKspq8AOTv;gCd5Q5s!@r zy0@Ik!=RPXr}C#s9)ed6_8xW?gx(oe(#$h}yBEezBu)yjv=)n2F7$-(2n!Zwqiipe z^an7?3C<^!cjI{Im!Aggh%o2DfjYv?0rmI?9}q!}nkq5_ufQAt;vc<35k;6#^@`qFKZ(HBFUzLX#e(zVCM3k^8N}B7rLJiC{7D60LL7{+OMLe-cAdL%q^3V!C261-` zZOHSX%%Mg}%F;z`#JNCs6DTjz5tX5hjp-`PhZO7TWaGVG2J499;U>{VDuJaGQ+HU9 zh>enQDSH;cNrwaz1!+J+Kq7FO zfpOp_*vdA`B;t{pQJ@07UrC5g6!wXnnolbtlCTa5f+xq#2#F+&%%H>$gF=BE0B&%Ga)7D(eh0t=u;a=$ z_FcqdfFmd(o4d^N8l{D)0f(R(i{<@dIwP#H%P*{T)6*XsiovTX+H$QMAA(vLLs{`3G7a7!UICPwJA1W3uxn_)Z~MN82P$gwkb?Fek>R52b2R9&YtW6+}X0= zMxq=PMKBj4(Ev%jeY47S!p#0Hj~tM+`fmGmc10g|GVq_rXg0zYw)XrxtJuF??uaws zQ_oo-56!hwZmKOCc{1m~w`!r@xgjX-$H$isHN`VQJ2`u-6t7@e9Y(JRaCpiC>hHaP z z@_YFEZ|F1N=7ezoQHHj&w0b`fD(Lw>SHXh?g@O56AT7lLZ2%D~&%z6>=vcR7$JHFc zdf7{g?$X6;)X4V`3SJ0ow>r?CeF6n+qXd^6z@s|2FZFMzQn*Hv;DdnN${}kYlB8gW z&t^ugIp=0d;^E+FZIyJIZ+{FL+63=jDFui5hwgERk-nY?1!bm>OAg&3?ILl643hso zRaCx(in~t@3k&NaCV+`6)?J4`8WhJNHtNFP12Q4&A!fg|U9sma)(E?64T(9cyEQCy z8$61?ypE}l;IVGeNBthI_n6_MoA!O~O^Le2y0%@p25Gap@s14MMo5ub#Q)M=(I2lKBT`cILk%nes&P(xVg5ZR^~+oLnKoqvUk<-(H9%S z%IgzvkhGtg*D=K~V~3oQck-TCIVn1j@(N#H72-K98l5(s=Gyce%4> zU!cUnHdF1lnBB?3u;z6_9Tzs+CHT1S*}<2!1FKj9fciVx@-n%eo#mzQMGEg0>8l*) zJI1nhm;6y4c}jL#zn)Cyb`f3!4rQ6wumobQyt~kpj<25Gm=w_IC^m-N-6N3aP1vp0 zUaB64TNMNB%0@#siItC&?Mw%S+akG{o8?Ro6}ocgI!n-#ZblqI%asD$8TkwQNpqKr zYZ)8W2A$Tp9^ZSCx83-#x#u=p{0!Q;HcOzX1cdbA^Su74T}uC-)p`DT5b2Yw&ln-s&v_Q*7x&{3@?p5-z>7T&#vD!xY@kL z7ua(+*yCFyMKhQ04Bzwp?fFf0?Psrd%XzP9g zT%c(J+S~8J4^rR1bRPXm_*DUR68aT`H|5C^{k41)q)yDkjx0u4`~oRmWTbA)OYgMH z%sHVI-Qv5(tih@`Cu_|ACZn_3lVMH*wC>6$z1uiu$6i>ItH_KFR3-CXmzs{|*_Ou*l0ls3te<4Y&yB|)>LX|}DMMfCtbpR{N$Z|-jjM2J+Lb}>?c#XkD3;*b0!!xx77nXSv3$ve4~-Hz z8oztVRXm=ul0v+nuhyQ3)R{<>+iHZ{?tg6w^!mAp3Vlq2vpX11u*Om%w2dbo^wF9FdM$;RyUcUxY8i()C#}Xe(zSG`jahV` zru1?JXBp0c#HVC)1xF%uK=-~KMYIrOdaZfBEvP;nZrJ%%b{K!pZc3YB7uCD|ySR6?u$!E+%b zqy{g6KU3@npLiCD^O#4ln^Uxa`ZBXMmGX0Bivq?0RBw3Qv~a9Yvr9g6mh`n5wzCyU zh4|IW=`xPLYi6+NxOo_fZR||X3uh2mMsvqB`{gMbfHgUH_rY+x?m}Gy+jK2+6y<04 z!-*J`d7_K^(i~}BC1z0~d7qcW*qB%WZ;)LrGLpAxpGJflFS#J|9DQQae9Lmz;^mlM zGB^&Ym$slzL5s?X#BMLu6qQyfSO(|iH%J$;^WinV=b$V2rAh)bs4k|qU0T6IHjm}) zxb7vOKJ#aGPRXO)GGlOl2=Ei8jjE zNJY%`!gj^0J!C<340xY5pg%BM*b)Rt+X=asFjde0zgu(L1che7Lj-iafy91-m{ zR@e&fCvM9rD%bzcI>b}N4sM(@D>fNDVqtQ4)3Ifr-pZCr&;3O{-^&;>vxVwxr}vtN zct>*mBqKqt@R1zcAYvz5rUUtsf-{fSE+t=k1 z0a-<6qTe8uwgqU8Xc@wqa?kJTDjn!_`GS zw5TLc`T#3gVHt8QTvrWL;1f}k!-$M`XkZ8w8e9j!A%_qo>lhRe^xEYolA|`WBJG|Y zNCvOZXeO#CR1Yyn=q@hCIB+P%$O%k})DfpfbnxZauEsbz$)}U9Ew?hF9AN;mrui7# z9c;zL(!KCxP|iwj8FMX2>TQ)zRi8}pp7TsQunGVY@_cC==UWppE6I~4)6Uwzn`LgIo)U+e+{X~z z;JPq-8OVo#6Z|>F!`2+FO+^z~iGP=yq;7>!IGeL=W^fGmOJw8+QsU@ja}Nqr8{g^UsmW0iNV0>-2M~igHP1GK(PpW7pQoE3ckA z81C^9MM_;$sU(#P#rno1IHjn!bHg$hzChdF^=fy$V?ukf!uK?uRk-X&Oq*Sfg zu2;^*#&_Jxs%G@PB&ZE8uPT`h@CH6tUYuxhcW+NAtRMv_@B znQ0u$EAeh45;%oiFUJlmuoLT2#NW)=5(6k{E~_o82IxK9|Y z{5T$9dI=#(&e6NKGmPHjN%Br`emUHgk&mBJ%CS5|kL>T0($Ua#KlNLF$z`5?iUoQ= zYBbaK)HsKK5Z>cHeu$G##eXAS4CJ;Z$Dep$$n`J=_wL9!KXrPM29E%xOf&VZH3ke;VP+`C!>NONgQc|3s6p1>mgwG@!#IUQH}GK9@ajJa z(R8*N*uR;_p)16|XDMR1<@_|n0w5XhBHJy0XilyQB1OgK*Nqu#k-`zZhcb{|I!;`4 z+h|-|j`2`oso=v++DvaPXhgoujv*qxPF*FK-m>91m;}@8D9WIyJf{=ulpliTIFJji z%pt0?2iA@+ypt?d!9$`bF zp_fyQM39NWq7QuCnf5`Ofqxn?38vcmZ3&Q<2U&1wCF|B_$d(DnN@2SrmPwj^mPs>v z(q%RvZtvIx&s2D)t)S-SAt-tzz$-%>jC)Z+-+7DQx%;+BtmLGSEb%i>KeSyDLnV!D z;xNbh$npTZD+~AM?EvZLfins>&;yuNFp-s9_#Us_vk|!FS@oF4a$}SRR0G9Slewpt znj2cFD<6Si8?c?&G&^!mmT3Bk2jYN8?fRI~?s1O1e7uxy1xHTRK{`qY;ig*&474Xr zLmY!@PL(l7(#9e|Vv_VpKk+~iBVSeU11JA!nlbFZsm)N_r;;aa_uz;ubzwK*jFov+ zH=J1dgNM7}W?8!T=JgSnff!F*ZrEckldvpJ;dixZ=QtuIJMQsXUV?DkM_aCf8PfNI z1r%=9XIV)oM{8(ZN-N$$IF7T)6Kav8`u<2!>Ox&`4dk>4`Y5=+QA<^^v`Vj!Sr4LW zp7Y>&Io(7gB=;0ewIDL5#cXetno+KX7Q-9V46ugxT6Wh zTgmzg=D$P%fVy)g4-yh9IVh)KiRm$AJI^)X=bJ@%ulYl`oXN4kss~#x$UY;TjXDmp z=kgKKVw&#LgM3K6|11kG#VjUu#+W@eHhH~}?Pqte;vmR59^|{?-KYF0+nj^&p0H;S z?^PU{Cvu=iIeisGu-(7b+ubfT$CfzN)%l_@XC2;%6_G@nvu|^{=Il7Yj4rd4#SE%9Aa$p?hpB+?Xx-+ydsdXG3DjmZ&b@`7t2En`Y0Ckx=%akDgom zfU7Z4w1=vHO{5O@wIoMyg?}O*a3iPQ;d{>Wm|Vq4{O}g5RSes0F_Gob(2^e&fVV+Ze!pbo^)CVR z(GcTo5)j|Ws^cEhYzXOw3e8W5`HEM#{q-ietLhq}T0msvJa6010oJ7MZ@pVgc#ifW4i zJ)*prr-sIAmZU{qER&^yyH5*J zNOYqr(AaKYQ*PFA;G4L|nkF3tqk4+ovmmw)kzaObigzg4$;X)*5#;=IuxB3&e=x%) z?Y!J(mK)<-xD`r3s?VE6Y(;(gQB`UMuYz_-_R*!hlBSh$pGy!pJI@6_@0=vCa2(d>#TI(oRk`}MNk?~`%uEZ&4Ng`8PB1M zcFI+vo*2$SdNkow(F36ACRILfd^&}aioGh4so^7PC${T|TWAFT)q!k>Jtvk$Fetp_ zfva43JO?60vgK^a{t(SDx=J4X65NA~$JLYhfU)5u7lwPAn)O3Y3GX3JAoH{le}BjY zK)Pq@E?-Lhoc)NxypMIEqFr1L$?|-Z?a0^RzuR|bIZ$qvItj#Rq=qtGD}H-M6p997 z_PGBPco?eVe8+FBvKv^bsGt`H)Vhaqt;1dmSy|S?46^qVyHY53B{VmBFAa(J@~7h9 zdfH=DTwvsS`eX1OjY}%j+Fl3g7XnIj!<~GUR%u|Hv`0n=KcTs-&0z%jY0LB$q8SPJ zJ|x_{F4R9RoWHGXD~&lV>79D|QEj`we@RV|9Dk|5#JD~8`o_NNDagzJ?)H8i%nj6w z^=-nG$m=S!NX%Uw+v(0Gt(VAKxlWoL*yCA64Dv(l@rx8#T8Nf;;kpQ_HkYUS12EE_ zO01IK6ts5)A#rBU$}%8--`e|p#%$#7KPT*Y|GM1iyoR*v^?Jn9ixZl$q;Uj!fEyR` z+2r-6N>sAg<@;#rD)+q{G32a}(MH*rW~PM+@nFm#phJww3OKCT2Un8g_MP3~`hbD> zhbYv66?+GaWGn>VVB8b{4yk^d9x{^eJw6d(Odl=;v<;(@8F_NDBKPd#F=TapbL?6p>Py;_y3Ea5h`w#xu$4u zbkC|(E=`h3_!vv@efT5&Ec7!nzS&W=j53~fX!D@fa9GLvDRX+djBr7!oXRCJSd zz&Z=fScQb*A7x#EngWmxWv>?>Jo&$}G%ysf>EQMwP3G8NEzxz@IhR^S{Y_+XD;0P{3={o>vK=mx4)D=5l^f3Y&_pKLALNxehcUx!e0|Vk zuY6l{<_(Qgr(H{_MOr(Mzsrd}kc zx5m*dVMYt4v;vs&-9vzj+b7sh{jvm(0&&O1$rLIthp36z2u+BT{bN8Z!_Xd}3!4KH zOua{RtP}Vr;O+p(1cVpVW7-i^dLcdX*D#LjppYPQf4}Mh(61cnVVGfKV>n13NT8jx zwwk)8MZ=|On7TL#u|FWCzY=7~QS}uNS|}q7EfozKkZ4O-x3D{sLO>g-pgPb`!>mp^d<`vNeFMm)e;; zvb12=2pYlarW!Vuz>G?O7k=l5lslOzb^zJ#WW47CmqYARtJFpJxn+nehZ46UqXM4w zIm<7+6Qfh@*CjEDp-LkKYi0m4GXrp|;A+HC#=xWg4c1hUV^kV7`r&ER2?6+!?7{3! z9wmZ$dz|u|8g;5A4%M_#mYiD5+6Vdx_W(MFSiQvydjL98^AFgw_!E_d(%(KTa3ieP1uERlmeMQr$E1A7?r zG7O-}T4(@vJ%`uK52eRhrGV7vZL&_0tSt@h@rr2SRsPDNl#`4v-NE;Lt^9{h8Pip# zQ>kiP4pAIv!1VvY))etmOq^#OQ| zWmo8;$2ni!1(=jtlahW-9KFRJ=P5?mQcyjkq5C+gTIQNKLM*O$&l`;RPzaxK9!SF$ zpPST&!6vkae9wXX4Ncncwf-9!lE6f~ye3L(8W~~Q`qCfxcW|B4>zG{+NT*s*$U-Lr z>5CB^8k@bf1dm>^^~O;xiu3HWmAwvO4X`K!dJU4X>iviO)Sk79{tA)vEVi$l4po)C zEb%p?`U+97^<>8Mo_G#Xj)BJCY>-rE)~T&;qX1VMQIhrEGinU!_ZE%6sCybviah7_ z&F-!nyuv=lChj=t!6s1C1NM2su>Ko?IlgVEr^YDangp6x+>o`>N)=8|s}}^={*9ub z1%gEzF(n5y6afnB2gaQ;jY6 zmKDrVJmYSwN}g9lL|b;2J2@`GT9Lg657(*@$WgrF&ksm`oZa&YJjAdaw>c_CfF{}O zJghruxmOMYTcZ2OSf7@`M^b#35UTa>=UK!7C#&;QdK3e7a?x2Hou_kjAy>p=mk1E_+$}J!w<Q zqfE1&^YC~VvYF-w=ZkN)h%o*T{B{oG@Ml34Vb2p3;>LLPC+dnyV3{2p#PC3z4oHb7 ztT2rSvpk+*tM70on%!}uR1V?8ka~ez2#yajU?Gx2MC2WjQTG?^>dvvC#5xBUTh@lktkU#6v&tnnq;mKh=&^# ztPnE6aumyeH(()E3|yz!!9b+%psOKAA`s;fV>bovn(cr`+Y>{lLZlRE5Z@DXg*_{B zK(-$c5ew1~=tmmhnzMucCf+6@+|rIoLl+Uz48J4xME18-3`PR|dH;~~_h2(ugiR%@ zL+eAA9?0ytE#T}T3o5aB7+PD76@W3I9lvL02|T}M@+-CQY8n#p3RSv8&m#+HuYc1OE=QIK`mqW zjUNX|y7wm8m4dI%wlhc8b%XcA@96Iwx`hv~6}|9scVOQ#jy99n1LsntMv7SG+wY!q z2WDx)MUhBH)Cyn@CLv3J8KthfylD;t3ZtC#r%VpP9N4=JawlOVC=CtU+T+s}IQFo(!A#71!Au7R8A(rqV zVo$!V0fg&6K;`f|L_7be=8RyHJYu4#JVWtv6b# zv*G#r1{p-JH(>@MMA(05Le)SM^Jx3ch(O}@20?Ld)u6hzfUhmez_F+Vl>{{Rj}qCtEMP)zZFnMlEW!2)p`e+W<~cuS5mWPoTjSRfH$k7)5xncNL_ z{`pu3&kP?L1FO4v%nj*MjTaCXz3uRzUxINm1LFuE?iK4j$8dD0nU+Iu%Oo zyd}VekDAH|k;a@InDHmtBFa|3Iw)ay7_6#0BtRNlk#pn(fH=!T?$}9;A8+|b+~*YP zc=siq3{>qGV4%M}d92XOGldK(UfBf?q(<`(kAQ&N#e)c(r_Fp_xxYJgqDS0~x8V_i zwtc_cu$Vo++yG*3U(d&!-|w@7*QNc+N#i?njg1~agXCGoWeOzT4|VeDxB)ihKQjdI z#Zt!pm&ygsXR!L<_SO?W4^hji_iH~5{u5K${=YRf_uvoT{pt_i)L(iz(;qvY?+n4N zFN9*zn|klxJ2akOH12GZmz4g;y@4F#-ARvWL>iZfRlQw&xeAzO$Bkm(dV1`*zJUU8 z6#Z=<5c5AW0lks5>9r9Xz0u_(4oYf`S2D%bITq+tRkpZT`d-G(V~Ab?$g6uj#16Cf z`|^kHxms^~)OSJ)I-d7$t{jE2x5aovyRUJ@hLf+m-luOuZ~W8G}>0ko^ac z_Uf&Hcjin)vqgQSk?`2FcUIP<_EDp97w?5asiaxPetj$qpDs;jnKNoGZL%xf%ojd) z@kd_6Y!lMIXBD*1&zc~TYA;TbuA6w5% z)t)6jv=uo{%(5xdne~R*+53lsp8dBxR%+0oYpfusz4l){W3Kp$9aukGhp z-rxKwHBy4rSKHD|9k50Mt!!dsV>aayaq>4|lnF|xr<=jbU@~^_`If^KRZ80A-BaHh zWH&qit#+2gZ`z~!48PCro;%!?3;eRhx7Eq;(ng?-4dfzUwE*zgm6=PR=aAC*_!Hh>1uJbFQi&FT{kw|yy-N5 zh7TdmI2d^{Vp9;jYk=a_OdXxLlN=5K$Tu4U3FfKtJCz&JLb zgr8sWBVY9RemL0d>`hf%>U5iN+LOMA^BrA8>bt zB{fKk7ovM~Z%W&PY|OwRZs|O|E(bnoF+Br6J|4Nx*ew;f5tuCR6Mn!%TkF&+tfhax zuF99ZHVI_!e??kzPZP?R`vD$}Xp6fxoC7kM6*=wdUiRoK__*s_Fvr=qE5*v^<3$x_wwuJhU=wF{)F`TUa0)BHGKxJldHc( zbx(7dOhS~4TTJfp+1QzJvB4Dm2HShMJ7#Tb zc5h)iK6hkn+No}je$0$8&q`HPA$Rm#>i0E0g?Ro^hu?3Z%XrI={phTG@b6&bWZ+-m zZe11Lm-V~*zcv4`DyJtN;&tq-OwM=X5EXWi=Y5BQsCz#--!IN+{>TaP#<*i@e1Wg4 z!_4oOajM8eeNKK*O9@aURh@^2-lIao%fCtYxre+`rMF&X%$bq0!Fwb*5~QDYCXdEU z|E81^y1sung29`!dAl+Reuxl+dT<>IEa4XT>fV!co0^wXmbu;)WXBlJB3h`2+{lR( z7n?l2&VDs6$L}-F&J38j*CB#t zHkL>ItI4)ux2)~*!)zL+2YgpJ8t&zJ)OXLH<--bRA4Dj$vdu3c<&Q&rT@TN@COiK& z-c$Ed&S(sL_I21sK-%7bn+SZM)Z}@QoV`)iPG*#E8q_WmKD}{~yswKKbi1H1C`R45JiW#tBtaDr5rc^t1aX9E<=kK=k ztW7Up8JmfF+>)to0fX&eD}r9CZ1SXqvB*X!GP!(v7%BP!sbF-BdPp1HMkHUk z1JBbky8+LKjMk|xb=uIihA~IW*~a!e3x`xNGNxBh%L7RUa4-Cez7(a=FT1l`g^`iQ z-KtFTcHch-Dtl`lo*cIi{B#Q3PH&kUrMj#JX$Ta!6as5Gdltx68RzN6eJ1l%nx)1( z0gYJ{wnI=96F-_e)kjqlL+w5+uA_3}U4+84#b77WxXQp5nq2KM_LVj0&CuP*f^l)1 z#FmAuU+tL!+rvq_B@Fxt1ME#Ue;4SU*KR^<+i~+_{v1`0T|;G0m7Yv&syDZv4z@QJ z$`5cuuOntnS$!&Vuh5Zy0_t3?q#vD=uqQ=L?*|Gs&Hn~bs^2c6 zzymvLdhRKom)C`0 zrx2a#UrIs0hO6=HX}VHD`R8i)2P3Oms--I3-mk?wtP8`CwcW^k_|}Cw`AqPixvyNb z>pT1B1K<|T$Qmk8>*a@@Kud9R{g}uuAP#dOmTduFY(PCqb3Qmu4B7_A0;I&L~S#E519<1V?Fe{ZsuPKLPwYOdIhk%zO9$Fb(46ZM%Z#oa(>m_y%=m zq=>q`C`hW3Z}|Cy6Rkk&l?%dQ_P{Dvq8`F=hyT03^E?vbQq^>!?=hXL&l}$zh8gB6 zYo_Zv9?$EU_i(4^Ns^`hF_$lhBG312RH4tWiEcrkYA*ft8vF{<_OJ?Q9Z8J;{Qz1nzb z;&o~0V#v(>s7U~3V^c!TOp=xY$;fTkzrWzM`*hfYkX)95d=q>@zd!b82lal0t=)c8 zQ;>wAN#1YyygBPB{sE(Swb1|mw-`H0iOt+r&$}zJenjTf6YdqlJnp9&ElsNA9$<_v zp!_e*`t2?-Xb@(oaG4}zJ{m0=V&F}`nLEY~?o>eXdg?xV&Eea)GcT>uHsOBs)4n=` z0xq2|Z;MoW`a08z_v;ReK(DjbSEg?`!u9|Wr={PkPU*LBaOPWW(cW8Y`FCsoH@WdL z{{ph1B0Pmz5TdKQ>+h5xL(T^G0H!yh<6mqv3IqaQ|MNa9Je)4M|ELc}R{XgSWp#hM zLQ3MJthcdBIB8EN?Se_y9vi1lz0KM)YpVwFmM03iFf}$i1%Hj+xrmH5U+14lY04h& zH*4meDjj!Td~8^JZ;yHt>eVjS>war~nlyeNOE=Vff8$MG2euxwo3m1;V+gtM(*}k1 z0#~>I>G^#Zc0=WW{pT_1tLOW~APLfsS3!EzA>;db-@;bnlNofKiBSGM{^0p@M}Uzq zl_e*2@q`3RzCseAq9ac|ofwM(vmD;{dt@6@aVu<>ayrpt3$Z6ta97bfHMymxq=d+y zY1Z64f6pR5nurBMX{nemq?WSI1PGL}i~Dwu=&ryVPRPMQA{<+0h}_NHFolt zJ9`7s9MSnoY#;fmhl;uGk)8|>Bn)}Kb0|54WtTrlB?KkWF5U%FQJEA?y?hr+ryB5R z&s_W$;d>7^qzV@JR9HKBu!G-gVVFqB{NJsWK9~Q*|HTwADXvYFnh)w%lBSO<4#Q%)oGwA*hWVXq+dJ9UiR_5~M zqlY3tUhK~0@JjAWYxA;SYj6D#b!~*T57R|u^Pu`L4L(@=V+k*y@J92lj2B4t zAED--%F$pB4g;10l?br#QVOZ#kSp*CwBk*=B~{SH*pKTOSQr@-a;;6oVg~FJS}t>B z=QdwJp7qo+j<_-*{9!5*GUQ>r(WqK;-#&lyb`Yw&X<%i`DpUifoh;IEER|6bKCnnK z!^}o{Uwgr&6()>ht`)dUJm$v9*V1Ywg;I($@P~yoA_`Gq@qd5pzX5k@WZzVCn~eL= zft}fEFiI-uXf6gI6d8)s2)Q4JB>8wa8RUz3?J=Phs5`8t*jLcI4|gdwc2hU)E-~5Q zr-hwkhhf}Ao|}tKVsC6OoQyeZkMQ;kR?9sd_qO~q>TbjCq+d@dV zbYa+>6S z0@)cSvvTE|V^LVkhKmoQ4?Az%_#v$Z1|3B$0~mNfFRUE?T#sEw*sghiMZ%b5(l~-h z(`YyzR&^G);{pe+NmUzs6qpY~u=J1z4SrLSQ_*rbRhNlSTpG1?&^9~ArAK;Y*{)5x zGELVetj(Z5)p9&gE8vz?ji+7Px-B)<4*aUi^{11f-QM|gC|8HBZT5!PZ)upilvnUK zfuqIHj=<$C&!>w~UOSz(fUJ4L|HdKDK}6GwjWOa@XYR5O-Rkt1mTvZ^S#2QuVT*RI zy}Q%+&$Mit{<${(DYlPE~R1_Ze2_8 z?RRhAV1h_G?_U4y?Vo@6-v9p1-`>1^{%Y|3(^qu`#rxyrX9k0tV;(J+IUN0u*H7R6 z8D9RqnTb9+J$d&I12E;gcOZ)sJ5O>@#CPvbU%?M{Hih3uQ|Rm8y~`39K)rhxSs_f3 zh|p>)QwhJMSF4Na;?Acf3<7Z>*D}kpq3_tc(=>`0D5B2>j-JG;NmBFtvUFFT9;p z)5W~rfpoBS3mwZ1ov+`df!mV z`m%qyDDG&SRI|~v!j-6Xws;N2%hf)Et5cdc4=p|#&!B*F3fO8Vm$TXFXic}9w`{#a z_W%L9w|+Cp@wrX%kuA&iw9VmA%jrM94)JU_8cjbe(UWU*H!HO_mNVn>wQ35yY*@_A z`m5DJTAN2q9j)EX=iFI$w7EKYbLlR;M7A#}1|F^#|C43*fAKxjhPFR2Yqr=yY0s^NPZyaHhf zl0`LbmR|(LJQ>>$)A^6VA5LF=^N&BQR*pbldcH_uz&n0?*_%Di#&SHG&dP84bp+}3 z+-5%pe?NT{yn6W>kPGr@R$di>->%N|uNR?QLiFd8<<#_t)StNrO z$#^)r2)J5KV6)&&WoI))i}ix13xEuiC4ZJe|%$Y4ft|@OvA&o#W1(K;r@7;@o~e#fBy5IX;Pj)dWaVwEMEo3e+_Ck z5EEqMU_K7&VOMZmEPt6TAQg_~f@A`8oiC2<j-^_bQZ z6^55kt>Q#aSo72S<(r=tLtE9n2C4ggTG?ia#;^!3riZlxyCz1(jMkAC7 z&Lg8wTAuuIL!ukW^-Adug4`zkHHfut{>7HimDVd%UN4Q+NpOqa7z=7F(`&@LF$TDL zuBq>L!P6{T%#-Y5RdWE;Bw6;)li;5kyvTp{;03UHj?v3iQe;IonwIceK73yn8vOHr zJ_*1bjbbJLxhnhh+LDE};@>ng#JHkixgrN^V;X$QX-SY*WPv`$##<;Iv|I36M z|K&35{J+ofkQpXAML;K^l8>kOJ-@Z*!Gr$~GKKno0Hy)}A|FHk diff --git a/summary.csv b/summary.csv deleted file mode 100644 index 97fdfaf568..0000000000 --- a/summary.csv +++ /dev/null @@ -1,2 +0,0 @@ -Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.2863,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json deleted file mode 100644 index 0f823304de..0000000000 --- a/summary.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "0":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows", - "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.2863, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v2.3.9. ", - "compliance":1, - "errors":0, - "version":"v4.1", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - } -} \ No newline at end of file diff --git a/tmp-run.bat b/tmp-run.bat deleted file mode 100644 index 652af1eb8f..0000000000 --- a/tmp-run.bat +++ /dev/null @@ -1,36 +0,0 @@ -@echo off - -set PATH=C:\Users\runneradmin\CM\repos\local\cache\6e338fa30e7e4b4e\bin;C:\hostedtoolcache\windows\Python\3.12.6\x64\Scripts;%PATH% -set CM_GIT_BRANCH=main -set CM_GIT_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\a647227f091b481b\repo -set CM_GIT_REPO_CHECKOUT=main -set CM_GIT_REPO_CHECKOUT_FOLDER=repo -set CM_GIT_REPO_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\a647227f091b481b\repo -set CM_GIT_REPO_DEPTH=--depth 5 -set CM_GIT_REPO_NAME=mlperf_inference_test_submissions_v5.0 -set CM_GIT_REPO_PATCH=no -set CM_GIT_REPO_RECURSE_SUBMODULES= --recurse-submodules -set CM_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 -set CM_MLPERF_INFERENCE_SUBMISSION_DIR=C:\Users\runneradmin\CM\repos\local\cache\f6b16c5abc894157\mlperf-inference-submission -set CM_MLPERF_INFERENCE_SUBMISSION_VERSION=4_1-dev -set CM_MLPERF_RESULTS_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 -set CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE=Results from R50 GH action on windows-latest -set CM_PYTHON_BIN=python.exe -set CM_PYTHON_BIN_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64 -set CM_PYTHON_BIN_WITH_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64\python.exe -set CM_PYTHON_CACHE_TAGS=version-3.12.6,non-virtual -set CM_PYTHON_MAJOR_VERSION=3 -set CM_PYTHON_MINOR_VERSION=12 -set CM_PYTHON_PATCH_VERSION=6 -set CM_PYTHON_VERSION=3.12.6 -set CM_QUIET=yes -set CM_TMP_CURRENT_PATH=D:\a\cm4mlops\cm4mlops -set CM_TMP_CURRENT_SCRIPT_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github -set CM_TMP_CURRENT_SCRIPT_REPO_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops -set CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops -set CM_TMP_CURRENT_SCRIPT_WORK_PATH=D:\a\cm4mlops\cm4mlops -set CM_TMP_PIP_VERSION_STRING= -set CM_WINDOWS=yes - - -call "C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github\run.bat" diff --git a/tmp-state.json b/tmp-state.json deleted file mode 100644 index e03109d44a..0000000000 --- a/tmp-state.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "docker": {} -} diff --git a/version_info.json b/version_info.json deleted file mode 100644 index cff706099b..0000000000 --- a/version_info.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_fp32,_onnx": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "fp32,onnx", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_cpu,_offline,_onnxruntime,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_resnet50,_cpu,_offline,_onnxruntime,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "resnet50,cpu,offline,onnxruntime,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] From 664416c8b6bdb1718d39b25dfd13a46b4443ed43 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 4 Oct 2024 18:59:52 +0000 Subject: [PATCH 017/658] Results from R50 GH action on windows-latest --- cm-run-script-versions.json | 602 +++++++++++++++++++++++++++++ hardware/gh_windows.json | 26 ++ mlperf-inference-submission.tar.gz | Bin 0 -> 19297 bytes summary.csv | 2 + summary.json | 32 ++ tmp-run.bat | 36 ++ tmp-state.json | 3 + version_info.json | 602 +++++++++++++++++++++++++++++ 8 files changed, 1303 insertions(+) create mode 100644 cm-run-script-versions.json create mode 100644 hardware/gh_windows.json create mode 100644 mlperf-inference-submission.tar.gz create mode 100644 summary.csv create mode 100644 summary.json create mode 100644 tmp-run.bat create mode 100644 tmp-state.json create mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json new file mode 100644 index 0000000000..08ddbd1975 --- /dev/null +++ b/cm-run-script-versions.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "onnx,fp32", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_resnet50,_onnxruntime,_cpu,_offline,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "resnet50,onnxruntime,cpu,offline,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json new file mode 100644 index 0000000000..b7cf960dbb --- /dev/null +++ b/hardware/gh_windows.json @@ -0,0 +1,26 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "host_memory_configuration": "undefined", + "host_networking": "Gig Ethernet", + "host_network_card_count": "1", + "host_networking_topology": "N/A", + "host_processor_interconnect": "", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "status": "available", + "submitter": "cTuning", + "sw_notes": "Automated by MLCommons CM", + "system_type": "edge", + "system_type_detail": "edge server" +} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..72da9f1ccf825115f5a40ae85eaa180c41d87ada GIT binary patch literal 19297 zcmcG#b8u!)(D)nM+L%vl+uoQPZfqMH+fQuUwrwXH+cr14`M$rZd#i4}Rrjy^*O}^> zIdx8-?&&$xpC*cg0YT*jF#`d=?DBA0m#Swx^Q|F!CqPMeH%pe8D@9aO4BF^C;{yg35j&)7qvzVVh@zIod_CV@^nTUJjSp?aTo$fPVP8!9PUp*iT_3Inf8~A^^l_z4 zg(Nh5_KcqzPw;>MVOGwLd&2NP;jw~&0owP*8ONN{UwYYmNC5?H^pwrYyK`e}$CN-{ z??ry(_f_F9VJ55sOJqTNx3`(p_^gi}pc(XM$`2Ehk>|m$Ft_&};K;y!!+qb#E+EH; z4tVTJj=O5@z?UT}F!dwBlB+ooxpHPtKycb7&d&tfctM)k`WC_Vr;1_dws$0lxr&Mw)M`6N}DO1%I={b8w)|<)y?#8=SHdd~L%s&0L#F$I(GZk47VS-$kI(0(NG1Uk(05o*5GO#TaQ#nRt_gPd+~Q-|EM%NP;aT zYx?Jyoi638Ug@j#5J3$9UOW4bm_y-@O8|^#pPO~-XLOA(YKk78iTv@>LRf_u9_e<( zn3~qF;&l}pxWz|t+lWc1aB?YCM^ih~8yW)_<_Z~)x2K`AM@g#oHI(Hs!%UIrh5YJ9 zof^yolp=7hr3Z+YE5Q^3&`^h0ZQd{a;}N=xP;Wvwi=(4`j~N44pUYs10OLH5!Bo;;;cQfluI* z{-5s|(bdHlEcHs>$QN?q*B8RO9wZ6_q=DNfEy37-HtQw-sSpX)N%-dM_GyJ9LGX3G z|4N4d?Tei`d0}V)>Uz{;jE{Y%|9TCfPUt;$+;BsrEszC7rvew(NB%rl{Lk=?8*T~i zkTi@yT#p5pMcz$cdd{4Qyu;1l2|4I-dT+dISuQ#zKW&)5j)%VrW4L@;u`^jSvQcKbWHYy>G~%R>n2s z|CZsw$jb{kPBW<@Twg--mN5)i+xfgay}1S2^$>NwJw3{J@k70iC(!t(^xz{-_-DNd)Qa7ccuXRaXOpDAv%^ z8%W9z-4T3IauBVN(IUeBVAQh+^bpREwQ)594-ZF+^%f5u=wln(}srk1X<)lf>u*&j|QG&L@r;fA?Gf z^K5I`!j&y>C#d!{Q;W+Yj7^=gF(!W+Me;=zr~;twi)NcdA}2~#iPI44JSZg=YxJYpf)tWAII{W2D4aH?=OZWTB*P=INW-oQ3fZsj|NG@ zy)?#ot1E=+&Y?u8iU<^>g%$9P1&7De&HpUro06rQXX228!7@yEJprS9OJ`4&`=nbg zi&r9*&0Gg8YAYcd#CCr-H(CQg%W#S^XfCQ{cAN`%uCHixic9>p0e#np+1SVMGkljqJm+=f}5kLt8I$wQj7f9A0^|!9t|w=(e*V*03bhBRVitt z6A^3-qmQ&O&vEKR;31ARgET4zjU5aNg%1|`?=;5ynW{A?Y!YggdQAe2Y$6dfh8yc> zevrryg4%+q>&cCfjb@U|S`lJ7n6^g6>gLiDe{#b_u#?iU7{Q@PPj2jFAosio>&po{i^pLQ&*79Ns{a~F9CPQWM>O=SnU)=NPe0PEY=coN zm(y8ctCuyQf>BzMn)EzEgp1lC-Q9UD!K$V5Ysss_NldrgNgt(;f+tNd)fk-@wAEh@ zC;`cf;2Pyl$u-U8;}%K-nGxD**ziMRFE&i73)!?siIufy!sHf8Dv7TPV8+225Z(p_ zc_FW+rL$>DeQW@m+*gV#?zK=mh#!MkNeHC>xu@u#MNNt_Csyc?&3QL>kfEDvf`)28 zXRI`-VXHPsIso{QCg0ltO))GBqFRYHa)`kye=!4{L|+jiNs&Skg=y|$2wAInJbd6_ zNpJpV60MhK!ciMNDh*1JcY1?!YAV+mbinastLH#R6c0?TE)L?8pg^ zPcdc=+Y;@J8em@za{wn-4?#Z}tE-P$FNNiYKsCXBNX!j6k%Xc@Bo5_@zk;$hdz1#| z9lp=CtKX-bHr!NCSH5dLW*rwOKAq4A!e2j1Hj7g2*P&@?SgQpkD+W<~M1!YJAzGD$ z$p_T1PLT2=zNM;z`w6n23HnqSCaaTi*tCTtBx5t&hca9o))emNP0Fb+Aw-BX6UhXr zf{P{Hnq9}C)i%!}hvt_ggJ;GAk)gH12Td(Apvs&eRz6zajiz2T zP+5A~og^v3hOYNh5+^Ly&j)VYSh9yOaYo z)vvYQfI9bIY~$j8=`=N=_ojx{d%AcVldsQ2ax`lJ6HCZ;n6>^q)xd>#P*3 zf9GqGqL`Rcj(Kry&PEWW->}cY2OAtRMg?s5tbqid+uO;W(b&(O7MHc6r4dmSL=HfO zULsCAS{msKPU%xY3cj0DRoFl`(Y%8Mb5Yme!>@F3YrH^epbt%!{Es~ zzLjCpU`Y%I=~$uHrBYJ_C5J{~BHQgAWvzRGnm6qM_QPn=`-zH4N&42Kk#S2tx-$Nz zS9-QrAyx!(Na}1LGBv!i$<6P=a4PX*TQ6?vt7sQ_ru=3RZ1_9SXZBTX%|S%^I(t_S z=FEZLhshlpIq^xb;)#-=OrJL%IPAi0LfDXP#w zW`tTm7CN(MZ66TvnXS%Sad|L_>^J>;6S&C^hD`uR5ntEEyvhmYF0^y=0a>hGY+B&W zZ{6e2>myV<`e9Q1^xO;+N{E7c)@&XX4+jjer@%wj7R&DdgG`5RZt4HG+;=*1E4**? zGz<;yFYUoRX4w%i($9x27hy-swhldy42{|Ci`=#DJOL`)Z&$>RjE3ewOMN>@hu_G( zd#A2kKVdup>Yr~*PI+!L7#cMGc{N^DoQniQve~hzc(SCT2p#nI(`S-ER+s=uo>!8; z#(vEV?#$0`*A0B~>A&9SPCRSZi~t^Duczzrr_}F;&eL_TEi|7}1CdW@CEx*O%Sykxep#h9UG4e6^JA%FSCagwJTl-Qju3hL%P`96 zn6tS#!R~u#8v5Zzp#dDK9|18p)c`hRN9_VH#=;&U-utmTA@N2UPsQ7z852+XVhbom z=8Txy`!r2A7|G-yu2q4ek~A2`V0MHle5})d6k`zB6HRj6x6o0q>t==2>1YDdjYgp` zsH8^r2vGpUKg_^J{d)MRcZm){%W(kyYuwH3xAh;l$Xk_aK>=qac z>9DU7vZaJeG8}H1KiG?Ozk8IZOptH{f)FBPu%bOe65hf<7be?hG^I2$YbDvw{JvVtJ5Ou8$B(k~j?jY{$rtf(<4NLlGKeAg)RZ>4r2Q zB^3lN%HJRA4we|{rwJM{#1*{tuT2zUGz~xaR4y2hYmz2E05pe#Iaef;(~62B(kJ+l zlQDR$qzDVBIqd<3Vt9X{B1LSD5Vs=&4wy(E2s&zT7mPSG%4{rxR^JWl4H#5&EQCzp zPo>J(*VxQ~KcyLgI*zd@xoTjD*yaAFtUH)@h<6T&+@)Z+<06U3bCDpbfuup$LU`OF zV3QTXfm*e6{zV|wifEitsCPIEpyYmdk|zEk%#-jYQX#TX<#m|MKcmlZ4hYcT{^lO! zlY++gvVqD;7C%`Y_!kvJ2Oy96u|)MUOM$6}#B9QjvVh|7H;)v~`@Kb>t;_FMGgO4)^ z&DX}27W+W}_CTWil9~MXfI>p1UZKUHuz{v|g9S-LrXVE5zEa4d2Y%LzKN3X}eci!z zDyIM#popNF5SR(2Q&eWed}zT0_rf&(QI+DD(b#ZU$*}RrzStZP@p)~A=Iq3<<7i$` zq{dN>;N-;DY%Jtgf5Dm-^kI@|aWIVQclzxq}G%WkUQ8TwxW0T%lDYnKccx zz)B^U&+i4L0jpw2H@mo-rOCAqsDx>xk@y&Jp>LemtFl66t^;-?E%L%Or@{Eg5N6We zKJxr>Jcdm91#{8pcnA_3W=9QQvD!a%J9@^=#O8|B_L~#MLo=>ll9m|UT+7#~60EB`>>r%xDYj_K}$_MZzUw^W-&0 zT79^Jm)j8D52g6D5xp<{b9KIY`Pt&gOt_DT)S(OFHt+!qaCrVypl>|6^6db=ZD(J; z@OixdPcOH(AkDJ_5^@C=34JsPFTi@}fKLAZirWxx{8ckb z(uDq*7Digx1)3KA;qLanYt3WAPL6FriV^U@-^qOWyXXe%J8o$1Bz3FTtkb0@v9R{L zXOK&#HljhNNP}VpI{4T_K+G~;>8AN-xnmAX?3oE8>brsRC$oL_V;w`GyufCxXO*4* z0n^WRhL2(W)3JR|3{)K#J=>*d(7}X}+vu@nK1r!+SD9)%+3u~yHQH5bk517$Z>~3_ zWy?S))seN^mqlQ-pfKFDwTF8FzMd-R7u+kfknu_pi<7UbBB6`*&H9E#U`Aa z-rRJ{B{!WEXR@}wy5qklkfRH*x17IP7Gu7keGl!Xzy3UsXJeLRpTe|v-+Y}ynk`s- zB0MKUJi zf}CQZedr2#rF@Ujehnz#0fE?#=~WHgb)p6a=>3tGI5LhpG#zI0@}TjMk+?FhbTg|q z;X_pW6Wu>%4pX)>*JP5Ify3=YjX4Y6bfA*)nl8N<9W^65ro|#59}JOhCu=2esfry3 zmu6A4Xl;stfjh087fv9HL!#DWUKNMh7cM<*x8~FyUod4Vk1haQcjC!cVJulo%=~d# ze-?&hVLZBa&mB*89zKGo94yI;zuMPTtqQfa7|7jECbHYCeE!5kaBGmc_}Zy^l}ao4 zh)R_)>9snICh3=s-B_Vxv*;-4XMbFiu+V8)mB5-p8`0RHYu-jCq!Y{2W;=5#6{{!~ zf?zj8ty?`ZtJk?!Z}0N(_Ryw6Q9y|FT;|)Y_R>C5To;BsMcUX6CY&4Rt-`@*_ae zHb=UW&FUIEXgw4*-;=Os4*DruvH5^t5QAVr&umGj!~f^+aVhN00X8eR0l z!Q?wK-B$zcC-`|xB01SvCm=>^S1R$8*N}F&S~cFmnmJCn>$XUnZR~k+`VFR)n~W~H zF_p^Nl)vMlBno#Je|7D8WLd0x>gBLZ8wFLo?eu6uLMri|cBkjoH0E*~Qh0HvWeSP7 zEXwJz0)IS8hn8H_ua&JC$!dhS-vX9Hl#U<75~_4}Q4095XttqX1ZExlVfe6)SsYA_ zmobJ1PXmXNagwI|K%@n)~a%QCp>k6&umAw64-Fem&s(zRzP_O{dcQKgZWd^m{& zSC89fWU19O4E>b#a(Ic01;aLDCW#og$$hRmYnUKv$e5{k9lCv(V0I*kGP^I;0YF9i zyULOiX83Xl0Q%(d=gVy4w`|<*l*OOv6@n;>evBnTlqg-$2S==eo_#!-pr}o*yq- zE?}T@J`8(UrTeze1F3|}ZYI4GS5uO3@vhnIpSnOP^kCHN9jlXeu|cLz^K0RZzQvGg zr4b3wfJs|li7x6T0qx!mkfzwdrtr2BbW)ND_pUA2zVS=HJK** zF5612`LG6SH@sB5%H(*2wgEdnP2Gw?7;S~%P<^*d3Fk*VqTVa6=zsp%rL0c-e@Owh z6$(?EVNImUCmDGTWJ{Uy2<4-!sE4lh>9rD_55_LwF(J-}lSbMbxh{?DDx10NMwZ&( zyAoA6I3jO62%MHvMz}k0>zex^Rly{3-l!h+?WWSH$y(ZPYMk~{C)PO}gnpZ=$kKzc zf86(nt8uc-Vifo_HyL7L0{I;9k0fe>^9d?OT3dIm%1&j0W;|-~3^5u%W&i?CqAOGE zYZ)9Xp|$6XsNss2qPRy;b<}MF%+3WX5%QZJ2LP^c;o8M`SV5+n~KfmV`n&&t##JIX4y?8Ee?UJnIT)= z@L(I}90m8dGyGT3ux2U-ca!)moZ0lG>3_i!@aEMWwbjT;Y|bDg3H|Mj*5X|&#}`J# zjkSW@ld`F(B1bO-lpaz&U|c_nk_84>&j9{5JQoi^uh`+%cxZjk(Kb@%8EH1UKjbJ@ zPSU9qwAs@RD4B+`Nt*FCi_yH!aOy4V6@#J@yipNwwIfj|5*?QAlAt)Ufw!S*d}H;_2bWVcL&EYwGi)m zUHOkiJF(LAZ|rNtD{@AMzRIG9yCnR~`7?ZMTbUtQ82T^?$;7!bb&3ml1-^nxVbjbW z8WPkY{VwW`fFADVwAbmL+{VPYBiAV5>HIg^4PK(3Xp;3NDja|s{O8aCX)}xyd0of< zsnQrA88uNx>5P`(63gazS%+dfn-K%5=Ru0f468Pu8_7tkDJoLJzV1zbCDBDT&r35i z&&_#4f1->s6GcTSd>tD?({sCc7D8jhqilOk*oOCblvPD(J`yJJ8I-4e*uR%a*w%k1AlEJP2E=GJ+d)3I?E zM>NkrKtH0-f(7i6@L06Bll1Z5wSo4`%9Y{@3-<$yax$4k#3yGpwkNs9jbG%j-spYL z50%D4uIs;(T!S{E$r>Z?9T#=$FHOA&J)Ts!lrbWcukxF+jK?@!8u@5pxiuT)j4ma< zoe?X1TGMk$(TVen0Nud-b3S1~mrTfv*ZU@Zj0?^7p|W5n9vjfEu)C;2yDA`Jokn># zvRT`bdpkEu_7XvtS!jd+43+5LayIFQS178cntKFxXR$bcG@oO_0spE4n#H=j6`+eo zIi&K-vv=7kXeK5n)w4|DcRitbbB$U0W3fphrq2U zsB69jI6l&PF&^CAZzXuL$}J9ebWBEjq8EJ4lYLu~HXPd2Bf^=7*i!GDxK=hv9NvSi zAtzpwCD7q}{-V)sTafLf_U!M+MKYQ1#zJgMNlo5)+iEj$8;Kot>C9y~bS&nkGtLn_ z@-KExNXb@QPfsT~j$~_In}MXzOFV3*Km-Fip8X&Dz_)c`sxf< zhiqvSd66w=8Or+y9NW>1P{8nM8rVSy9a>n+BLo5}MKitKMl5Fue3Mr-mgcscZpv}D z)~0+60}~8CM}O=pS9|#E!WfQZx;a3@J%GY?GGpQIZZn~ z%jrLd$XA=eLb!tJI)(E*Rl-BNFK(dBCPCFaqK8!|swpxiz)u^!Ikl8+mYXL|mxM8@ z7f7k>-p{V;{=RLx(&cLE#c(HPR$lWhly9mVK8`P6p+u%ljcC6#X_ zv18GBgzyi>yD45^*fvL&Fh0gZ)tfmo-W~2JBGon~-Inm-EG<0YllTKoDb<+~vgMVr zFWspfUzL*2ze4&g(ZW*rMJCm$9i;t_K47q8><)X|)`xRL&aKXal&C-2bp%j(fO7&@ zKg|o^V6%41hUAU~0P?R-H-qpluog4B?(ByVUGKwM3A0!Ag#$K27}}nwxOGqDI8*ENCvJ1{s8=l?cYMNe2<_sIr~haC^4yphzsg zL$Ub6dP92wSJUM!NI#-Gt!H!i%>kfaY~~LNq38DpgcE#rb_|d`Q|`JoCLc+h{W$4n zRg;?4j`Q})yE7*5NIPhLip^9NK}JF)3_3L+FC4j*?g3~kHeYdU)N(d$_mjyUbfUGR zj}K-Sw=dW`la!=zR-0I^)@%x>qu4I4&uVhbQ1c#4qOG-f7XN_=_gv8QIlL9xD0Q1a z4JKTrEJ>ctYcu=a<{HVIh-mKPn6@g}N&LcL0V1jclyPimdm~z9oi&ZAem9xA7>AG7 z92Zj7&u1`zJYGPeqNWMv=?<5bsIwH{*_{h@2;13)tzU6R$%zd<7yGl#a0O=Sla8iC zYzxKEJr+a!5vd?0CGocbj-jJ|D=U?Z&r?3xb729gHMHh?1!ge2v6ja&E^?dT1|i(v z3+bY6o8c#<);*P@MCN~K|9&j9Qa-E3Ac_VD=fVxbDpcD=Z6>s^n@Lt!&NI2^xR+s^ zZ!cElTruk!6S2c(dWKMk)I6zLo9?oj`Pn?&)_$g`N;b-4=Si3_!wxv_JQXXuR@6az zC~j~lAq*z_7D#ty=XH@}KuK{$d2}MzPiHW=^E|eGd;6brbn0}1bt`Ud4%Vg_4Y>Nr z4wLK3Ygv(H?fioLg_P{d@5RaqqH%=4#b_)3A84JKz5wN+{CdR*6P+{E!seq!u7PTiI5pU+jY7DV#SFXpJ|5{r-?E{NP`ov2KGQ_s8@Am zOW|l3P1y^OUdf0%)l|69im!E(K^bszHkNW_jVtc{H=K}E`9N`YrYT4%eV~o{75J)8 z3hg?#N;;F41h|ufT_OzF9OZn3=zlT@0r=as&p+aAmz5X~!R9TyEz*2@gVE7s!r#k- zlfv4PU&#jm+osE2m3T7wQAb(@Kk0avtth)wyq{?QijQ*wY=f&b#|$>u-go239dSAN zrx~B5m?%TSvms8nr<2iv;SBh@*;->fc4z+XbP0I>jzY34usrMI+jN_%_GVezr1c)7 zC6S~rGO~?uXcg4mtnm4RmyO7zPIyT{DfE6FhIF5DcV_#gI+Ja)klq4ke^GOT4-qH zJ4TfjmM;Q;?Y0?pR3hos?Ao;%|DMDWIgej^e6ucvYEP6je8i{ z=z9M^@o=AIxobJR`AD9Ibe&QSII(2_C^Kit()HX&$vG_6rrJ)Jz+@a}IGqdh0W zs)-K2ZAK}-W0?QVx)AS?;{h9Ve-}YhMf%u`UzbeteCo6@VB{o$wd1r4o$yC0zCGGw zbL5+J{k--ZA{AlECZgxUH^Sm6J(^LCa8I|L@~~*@86Q~^Nm`Q8NcZr>6>OZx+4P$2 zVSXCYd73XZ!`u}w)``PxmaXzgu6arjr={1ZASvbO%&F^m2|9-8vEW^pOcyII(>lWoPb5!Q0x%o{5CfEZ3U zXT`c!@&(}TUAOMARcI%%^F-Mz3uOoFO@$XYpMA$$cgcrwL|}KEU(41Lm6kN=R-?+} zybL}$TIbCW{AO57uGSUza*4s-aBU{GKC;Rwja@tYR>9GMF}kNgx*3u}d@QG~|JQg* zQl{HaPMVjspYmfQOs`-v-=VT8TX}VjY{d90+>&|gQd74BaAL|#4?Ss+_Pu5`BC%i@O3^mHgG(?dO(A(|iN-QFK)dO+b>OIk;;2ZRY> z?@c~0>i+|0Gihw3_^P3=uH2^axyt3IUb+zC|F-e@WW7IT?ENj5SaR0C{32{ z+-G2g`(*#VVz<$em^h`*a~c}mL1=JHJGe8J+7oeI_|wplbyzw#;Q$eYIe4b13XwM+ zO6{8;N7C4;K_JRD6vT;Px%exJeW_&JpItw4C^tqtI43-}b! zaMr10(0>n#Pp;kNh(4uaju*DYsE{dF%aODW7J*7HL4= z8A0D^S(J6IRcQh*@(qLU=>c62!6*g!)HJ`9T+oJlqA zj!7QPz{w;riMppa6IYqxjtlw>9utq&nnJ7^eOM#aPs4m7b#KiSH&PQ$L&X<^nvarO z&KV4WHV9&eG~-N+JUl*6u5ym4oo0#EFc=GFx&fw6J$!6$P%3b=BzQX9@;)-;lyr;)c)*9yfDt4JH1uPKoWm4X_7gM9Ro2qQRH4{k9pwlE z?dD-c-M2_Lt2-i9E#q)7SQuqV$|DxjBx62L*(IA*VD7GAxh8`1v;X&{Fr zj89Yk|Bq>ejO@_Tx)au|P{e@`SG!2|1f|nW}}O@=QpxxbSyGS2Y}F~a7;VHa5U9=ASow@lo?WltHzmFy+Y~x zUhtI_-~T_LMjuM*59P9zVXzC|M1L6AlagFn*N1aqZbZ{xC3(pt&Y5-5%h7ZQQFNb- zx4B%v9xiOF&3P#7N^TY>O0IEn_TFY|;@hLW?*s&a)`FXB!3-UCg2_GfE0{CnUsL63 z(K)Hl-QHRJn@)5DY^!h9^XvYpqX68BXT9Xqi8o`KHUqibA}Oj@TSvCsg=(=+8l(pk zVI|_!D^r%`4W)O5TZpq|M>#o6ENxYqg-D(g<2zT9ET>9lco#Xaa zbIZ2P_G2a`?&E)ujgXK_%iu(}T-)8lN-T%P1v?vkf=21`6sRpomj6R-m}}_r;oC0~_^W7Blv*p2-?M7-ganJ@BROi#|J}PY$_r*z;!!3Fc7jE)c6F~D3NcCFUiWeQ zJt+5u0d&pw_nF^L+(q^7>VNkIL30C-wJV(CA#jm;x!?n8fC^AXFh&no9O#;sF^?|)%A_7OeuZDCMg z>3W;w@FD!3W3`0h!82~=4X_N3YD;qQ-=QauN1^nQvtL77D5(E8nxnIyMq~~Yf(;O; z&h(DZ>4(Yy=$p?~#-KG8036>Co*faV-~%Achrnf-t-+^Edub2w4hH^I+}0bKQ^6|< zh%tgGB1WJXg4i6uz%tc9AzJp;LnMQ>ib%VwaQ=P>X|ZB@iFwwwSH&{N9_Unr+5+=) zqi3=fk5Vr1gz^Xt5@DxmE0yvCFd;J93jfD+{Bh?+h7kaOk2B$q5F_ruOYMZ;xKk4{ zkoMy3-*BMEQuDO{3o^m}#-cp4?|PeYX+|SPDme;a->^!NS^lm%7ZQpxd9e4WlQ0pf z&lQNa775AE)Q2)hjNnd&6!ibM_xIMFiL@FxsJ-fNPsE_x4?lxy5&NT427lY{<>z2` z=nu1fxnXx#!0%Y_BCt63P1ddTs2@zC9V8Axueq31qEyOgWqy@l>qvTkVC=)@5yC_? z-AU*O1;Z%8_8lzUPw2NAAQ$(&3*GCewoXHTdGBSZ=1bu8h4c`|Dq?Q2g^wiWC(v5n z<*6Fuo!}BiMsZ@{CJGop*y)Xnm#)u@7!0^K z0FHaB4%4{>dTmh(iA^n}B%~>+?H`oiNyZJL6UG5e5154%3*=vfVU7jNMhN8zr8y0*Hya#fpwfJ4v69jUwrSO^%=9} z!$)NIJmTDM$cn}N!BubU#rO_UfV6{7(6=DMe6T3MhyDKm5C&PQZmnZdLqkS4+A=qM zYgT$NibA5{0PiN0(ajXY-@>&E0m=CfJ6VW|viRHVRLGF&T*hw5(OHYVLiwQ}u`54_ z@khXMVF^#zLB<_CX28+#nc?e?UEs2cplL z6Qx|j&4GK!hp)L`4_*wPdf7AYC#G+VL9Q=E;&fiEPyfl^-EKV7*OfqKe4*`9{!pGX z3fHa?Y501$3FI(LPn$);b^9z^zrcPnWkT%V0aHK6_P+$Wy_N)iKL>l+_ypz$B(?5CM(kLeuLE%FGQ z%@vNBr|L5yL<(%twUfIfWmcg*+seL!@)dXz)=k&P8M>$?zrE3c0%+g4Tau>A_bpag z<7P#QcogI)OyUTt=D*gwUYd^c7Y{DwCamU24u6MElXEZUIf~ztL-MPh^Fo`^p2B~k z&>h;@Q~L$Fz~bk^EK%wT!To~yb=(c*+lg~6Ps3XV=o}yJVv8K$!qTy_GPb;MjbDVh zY$851D8JK@@hgvo-I{X>gkd*{$OtkkW1Lv*s>tlZgIh)7oHOerB8ZU(N@_0bK2=lE z9IrN2#+A)aijD8!x)Pk@X^M7yc(@kC&ounz)}8dvrXA<_sqwW0us&+A03;|KPFf<5Ucw%Oqyhh#H~8GsBLd`t+W!+RJ4F5OYVr-$0qctVMSpwIit z%<+OI&&MKk#zUhQ?ma9c!$r^F$uYcAM2V&{hAYVXJJU?=b7-E^*Tu0-kfXR^1Fok4tmIl6fhddz) z^%|&W8vLxGGde6D*ytN}C>acca}(=70U ztsX&dM8TC+44<0oGoNqqw$s}u|AeTLSerb2+Ruce7efVZT?<-3)kBQ3(3 z0#AaBWjr@lD?z(`qId_ZB3t|CMjcva2Hf?>AS(<#D`K?shhR?Q4U8g8482mTK|-)! zJjpk1T=MqMQZAWv0*~{FW=h$3ojq!fv-MT~>eQ4p`aK<%>qAvtR8p2CZshp!l*LMU zrzNB{ks=M$Sqt4%{5OTloOst}J@gllM^c2-)J7YrQlo4t5ouY{Bvx{CD6?glFAyWr zcCwgU`&;q2s6^qev*im>_>AL!cv?9V-#6XF2DJD=PjsT@cw)I%L1wkPo~_BD*^)?a z0t`yVvuIlK9c#)v`C6=~Jw^%))*^)u+|h8dMFO3p1`^oQ&JQXQYBqaM`8pU2{426A zT9!t9)D^Yi2q!}ttb|>yx>1d_#^l_QDNT~1BxvtE*OPW2TJvxSJKHYqYalP13<<%` z&Zlm3w=2c3`Npe91Ya<*S6hGQG%~;4H)P4)Tln&fd|<9RXA36HeoCE{X#DhOI|bu3 z$t}IAd)Z?s=jW+)`7yzlKQS#X z&WEAC#>G)4+hBhReX-O~8DW#t+fiwi?Pbl=Q2cN{f{3Kc=_*`k!RaC@HATLg!loRw zw$_-nK3k#fQCfGfeRpHb(enMf(qLlY#L}=y+Zy476Kau;q5PNP{%xV>-RwBj#b+gX zr;#D`B{s^FhuYD^(bn0(q|aVM9m%)Zn@`xPS9FQ{Ei2Xraav$nKyx=Sy7$99{bcW5 zFCj+|u|R>eVeDd|?dOO>&&SoJ@B1UhkRua*4ro&WC=3mQ=RdvsIpEXY>g$YScHzPc zxN4f5IE9D<>P$i0XG^_jGq#CYnUui0wt(mGmHF&V(Woy-AdnGi^B8BaF^0C=^z^FI z)vQTDU^)JtbVAwx7p7$dnxN$WMM8)$EzD1EGx|#K}dT5m-)SX&C&TJ zavOP;^W}Y@+~oAqzxve6og-7oC2*?01`{%+5s&w-8ER`WhYae*E|oLLwh@dB7jdH6 z?(bB*)9P9KTt~xIgM_Z@y4kEv^vRxnR>K=+m3;>JDv;RKo3E7at@fvvZGsoX{A#!k zrDKQx0=?&(Wn#THw3{H8o6LT<1eA{UUEt2tBmG0I+>_=C@|%R~mc=oR*T>eWJzKiZ zg;)i8)wlU|&i?t;J-yzk&Bt4F7YPmt_~?#(MVTDk1ZI&+qMwe;x2Zx!1ioe^lF z%Q^Zwh3sf4NL*$;F}GTjMYNyA&kuT}W?57eL6$b0k6ta`+K&40tJ3=>GLl%D>oc1>!kJQpdBr?|(hsspby5huLrbZO+<-FQR zQ9u1L)2111Ltc-DF9`EHs#|{#M>fv#As9^*sh%chy@9Q@K(-e?Y-08cqc@~?B51&9wdNr|Pqe1BPre_F#@-FmY{!l%&;BW94UbJW!k0w1Vfav=Zf|?w zpf>wBeyr|DV0{_%J9IZ9e?r10zIicItt~@vdpKdYm{B01pQBMXvP}CbdKXd8iBklR zeQKGahs>ELC!f_#@9$AD@OUYP2lgCb&1uh5^C)ww-;(;!(7RSe{Q-CLoAKlX^itFk z_h46eC1r0dxHxCQ>|*dU8}wSIbXt0QwSJmmeivZ2nP!_6dkG_dT;eOtexdU?6Ks9l z>_rgEYr+r6`KXr@FnvgSF;w}d3%%_T$7|(fQ884RE-$&cs+N`d-!w@dCGiWsv6)bt zV;LX%+=O|;c)_+M8f2Nqvj#>in5*`G(*+xv*EO!BbmVmj&GKum;3ZCUJy6~k?-Mgf z4#%cGCM$1n-cs_EHkqLc^+ZprKOFggi6g4x)ExS0??&U_#QC5TsT()h$Z2)0dvng4 z;PFSby8fD9ZXjq}S6OnX+5g>w*@4GLFskUSPFlf(dM@^@kM`TwA1eK}ik}|#mlAt= z;5&Ha{DsWV_2PsaVobA0ygwSwzldTY+Z-n3DGjoME}`U)4Sc8xF7PL3z^sn-d^|ci z;XGkR?KnmU0yAuD&+#X#-vouAo$9e?7HT5QZc=+w>Z!*>lN=Cg#Y`*Fl)@|{=90O+bc23NQ`VF2%JsxUO&n8v{2<*}^d-jx`egj`#e07y>3GVEK?$_v;>Po89B8;N$Hy3m8-kCTgXXR& z*Yvo1IGu52nJW1>#i#e>$U0UakMbie%P~B=tLGIui(MHm;!a<;%toM8^yH^(a9#q2 zB&_D+9!Mu4lt-pjWsyBdCfDGJ*m@z0D~ucR5YG282H~f?`U++rxh^uY))L(BHX=*=Y$Jby-y3l6TWCfz1nd?@9ORb6dq(2~I5@_w z?L7Pb^9DKe#*ew9OFRD9S%<|y8Zwo(IJdNf1H(p-4#A?yh8lxmF+w%bU2}Sg0MmwI z@(b_`xV2})g(BCekp3;;5oNKpMk6#$Jk*;koEBw922YAuA9D#~9iPP+ykzoO)##L~ zk0xtoZkRg^PgQH_2ZlD?ha?=4-iUTSNLy!x>32bkc=}a#UE#zOthD2msx5^5oFeF< zll*$?o}Lfc{5P4qEuxmAc6aP4jkJ`m>E6=nEDA+|@VMaBUAtVwqQ7>`@!kJA)k+}E zdfN*sF>9*a*Jd4yg!KO4);uu^-KTQEc|D?V-j8i1%GnrrZ=gh{SO64^Zl)6oUTNHB zCuKtvd7pt12)sfTHqEfEXvxK}GGWjY@SwZfA9~V3syXOx=gW80j&vA_O(Bj@B#{NH1EoIfmHtqYW}Gl4d&o5U^!5U01GdrkU9>z z0yDu zsx|lB%eU_ap}Ly}R<^7{HE`O=A|1z4871KZizGA5Y^3+K7hGCl!Z_wyfy=~WZk&8A ztyWSfr8onBSV$wH5ET~x_ox0_aHmH0Z8f*axDOrJnXLw+q=Js-VgN#sp*W3@`*BE; zkB5^%zL?h@6Iy|~!)l6s4ZZttn^I#pb<^$=lMQ}a*g19>#!cj@nI@`BSlVPr7x8uD z<|h&{sofHS(6Owfq&S{@uFHagf&%u_ucE@G9p+)!5?f($w4y7gXyBe&%OE!5<)0MfyVA_ z)wETodTHjip=!gvZIfX<`~)ub=-7PH>DDa=`Lc5`IFuVK^uBEqsLh0HlCoZmM~E)F zS}!eXU5NY?hs#dUn?fvmG;fM%t;@SEP2Is=LrA#fwJ!;8THe+f-H-)j^-?m{MRw_l zqGR);oh)S!d@V;-Q<>Pm$F{JZZB$?V*T9(Skq*L z!`ULqD$pZDnr8P1suXpZY&EL%=QiqE$+7H}pGn z{FkXN&0Y_>Y_NwPlZ&Ncox;nyo9GAMEUMtUX_EgLWM`br%9U@9MPV%)Ey4J?VUe|a&_p&W^ai7H4Rgj@(TVYaI_fO5xAV?`E)VLYp2r|kTq}k z-#Fwsh-iATF-E-BnY-*muXTD%OE>$|tTvGSuthu9-rZ^ZXIi#R|J)e=1=!vA4~!qW z_|JVjP5k%t_24f*z~JeJw}W^8_2wmpQI}G&47aW&_~zSpZ!tk6op*2k>)l^|_}>5i z?SH*}_wx1N`{%Fg3W^WM$fgW55*RUyPGU zF(2AVK58gkJKN}Jm`{@B`sb@3e;k7QHywRwsAPTFzg!e|G)}77Xj~yrI+s#|HUZHz{0Nq=^ndJD~Ci%#gWqaD@aH!?< zA76)fHXMznAD8IKHM*OX+8fK6ars&`1zt8R=4SoX>L9JnBc_hl?&fputUKCVoxHhp z7hWRUmlOjJ*N~fC%(1ax>_qcYZM>}E0FU0T2;#uXun#>|bw$>ji~oB5`nwgmSIwAP za4^eiNsxg5U%uM77sk0Q<)|EfT5>U3|MlXM_t5gIqQH4xpy${Kb-b1X#2o}sG}UA6 z`1`NjTa(P zN4-5F!I_c;8o@du@%Kw=gVX6~WV34c!3M8ESb}6xO`GKxK`~Fp_TzN^WAKO5*MI!S zA66?zpf5dNq%hzeKfUbDo@Qe?9!+QEANzF#>Ga%YKL&q4eI301?hPOp^Qw_Wx85%zqdB1uSnc1AB)7_`06phqu9U-R(*6pG#>6 z|Asfi=i|Ej;-Qe{`0X1UJO`(5{&tn)`Rf;v3|=MU;pigZYB_<;g142O%@8ft3!X0k zGEkNb5@P|)$l|JuW`q&^0T&ZL{m^_nNo2xvp7w&bb`}ItfG$@L{(HjM2^W6^4!k`L zK7i`UFaQeDVHNzLCgzX-z&h6W^ZKf$+VNUK0l)usF$pkPGRg7Bx7OBxFZahZJUm+r zbGsexPnQ{=HZ1(-KmVB~<@uAxcmcxlRdD>bpmqZ>K{gKNXhl2e>}!*_Fv$ zzNl9YJX=}A<&Q}|?=QW#eoe;Zg%8UF=zuR1JiD-~vW6J9M76A0^K5xGRP!#ynDr!o zwq}}_Z*u!#nAvAww^yxL1~;FrqMb`Tz}Z-jX+2Y6cnQ@iPV|g5KYduf`DrnlzZI);Zi{N6q2tFoF+;g#9u<>{FG(ZE_Y{cAV!!!3vkb zwi1!LfU*(8c0u|GHVC9BScy{|m4op_GC|E~gfhW-Wb{ePlRs@pbR)T5DcwSl+oZn+ zvDVGM+7i0bdWFiHrLj5*ZqXZKL5*d4gLpT_09Vg7_5Ch*o@I-9l3lE74uF~@%l>&1 z{8NJ$`OhA_09Ma2dbvu9tjI>w5`N2vAL>GbfBwJE0&qv8Sjm5`%6_x9WMQrNx6KSO zu4q`U$ido}?rRw{$o!~#zQFVO{J+Z>_s#2nnQ-I3T_eck>aXZd`$spG*G_ z0Uyl9fLLB`5uLF7Bee>c`_jFJ9bWivE)}BX?{(rrIh~ogJ0s!kD BR9FB2 literal 0 HcmV?d00001 diff --git a/summary.csv b/summary.csv new file mode 100644 index 0000000000..e1aec9e546 --- /dev/null +++ b/summary.csv @@ -0,0 +1,2 @@ +Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.7491,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json new file mode 100644 index 0000000000..d14c5d207f --- /dev/null +++ b/summary.json @@ -0,0 +1,32 @@ +{ + "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_windows", + "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":17.7491, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":1, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.19.2", + "operating_system":"Windows-2022Server-10.0.20348-SP0", + "notes":"Automated by MLCommons CM v2.3.9. ", + "compliance":1, + "errors":0, + "version":"v4.1", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + } +} \ No newline at end of file diff --git a/tmp-run.bat b/tmp-run.bat new file mode 100644 index 0000000000..4a67015352 --- /dev/null +++ b/tmp-run.bat @@ -0,0 +1,36 @@ +@echo off + +set PATH=C:\Users\runneradmin\CM\repos\local\cache\b512f8207ffe4e97\bin;C:\hostedtoolcache\windows\Python\3.12.6\x64\Scripts;%PATH% +set CM_GIT_BRANCH=main +set CM_GIT_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\20d7b1d3e976416b\repo +set CM_GIT_REPO_CHECKOUT=main +set CM_GIT_REPO_CHECKOUT_FOLDER=repo +set CM_GIT_REPO_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\20d7b1d3e976416b\repo +set CM_GIT_REPO_DEPTH=--depth 5 +set CM_GIT_REPO_NAME=mlperf_inference_test_submissions_v5.0 +set CM_GIT_REPO_PATCH=no +set CM_GIT_REPO_RECURSE_SUBMODULES= --recurse-submodules +set CM_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 +set CM_MLPERF_INFERENCE_SUBMISSION_DIR=C:\Users\runneradmin\CM\repos\local\cache\53f350da4cc54190\mlperf-inference-submission +set CM_MLPERF_INFERENCE_SUBMISSION_VERSION=4_1-dev +set CM_MLPERF_RESULTS_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 +set CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE=Results from R50 GH action on windows-latest +set CM_PYTHON_BIN=python.exe +set CM_PYTHON_BIN_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64 +set CM_PYTHON_BIN_WITH_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64\python.exe +set CM_PYTHON_CACHE_TAGS=version-3.12.6,non-virtual +set CM_PYTHON_MAJOR_VERSION=3 +set CM_PYTHON_MINOR_VERSION=12 +set CM_PYTHON_PATCH_VERSION=6 +set CM_PYTHON_VERSION=3.12.6 +set CM_QUIET=yes +set CM_TMP_CURRENT_PATH=D:\a\cm4mlops\cm4mlops +set CM_TMP_CURRENT_SCRIPT_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github +set CM_TMP_CURRENT_SCRIPT_REPO_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops +set CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops +set CM_TMP_CURRENT_SCRIPT_WORK_PATH=D:\a\cm4mlops\cm4mlops +set CM_TMP_PIP_VERSION_STRING= +set CM_WINDOWS=yes + + +call "C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github\run.bat" diff --git a/tmp-state.json b/tmp-state.json new file mode 100644 index 0000000000..e03109d44a --- /dev/null +++ b/tmp-state.json @@ -0,0 +1,3 @@ +{ + "docker": {} +} diff --git a/version_info.json b/version_info.json new file mode 100644 index 0000000000..08ddbd1975 --- /dev/null +++ b/version_info.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "onnx,fp32", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_resnet50,_onnxruntime,_cpu,_offline,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "resnet50,onnxruntime,cpu,offline,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] From f27fd8baca1514b87f7865fe4d7b36833d95bb32 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 00:44:30 +0530 Subject: [PATCH 018/658] Fix --- script/push-mlperf-inference-results-to-github/run.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index 7800479ae9..4cb9b2bd00 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -5,8 +5,7 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH ( echo "Error: CM_GIT_REPO_CHECKOUT_PATH is not set." exit /b 1 ) - -cd "%CM_GIT_REPO_CHECKOUT_PATH%" +cd %CM_GIT_REPO_CHECKOUT_PATH% git pull git add * From 84a94fb09452a325f7df3a3bb2a5ad208735ad46 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 00:44:52 +0530 Subject: [PATCH 019/658] Revert "Results from R50 GH action on windows-latest" This reverts commit 664416c8b6bdb1718d39b25dfd13a46b4443ed43. --- cm-run-script-versions.json | 602 ----------------------------- hardware/gh_windows.json | 26 -- mlperf-inference-submission.tar.gz | Bin 19297 -> 0 bytes summary.csv | 2 - summary.json | 32 -- tmp-run.bat | 36 -- tmp-state.json | 3 - version_info.json | 602 ----------------------------- 8 files changed, 1303 deletions(-) delete mode 100644 cm-run-script-versions.json delete mode 100644 hardware/gh_windows.json delete mode 100644 mlperf-inference-submission.tar.gz delete mode 100644 summary.csv delete mode 100644 summary.json delete mode 100644 tmp-run.bat delete mode 100644 tmp-state.json delete mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json deleted file mode 100644 index 08ddbd1975..0000000000 --- a/cm-run-script-versions.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "onnx,fp32", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_resnet50,_onnxruntime,_cpu,_offline,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "resnet50,onnxruntime,cpu,offline,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json deleted file mode 100644 index b7cf960dbb..0000000000 --- a/hardware/gh_windows.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "accelerator_frequency": "", - "accelerator_host_interconnect": "N/A", - "accelerator_interconnect": "N/A", - "accelerator_interconnect_topology": "", - "accelerator_memory_capacity": "N/A", - "accelerator_memory_configuration": "N/A", - "accelerator_model_name": "N/A", - "accelerator_on-chip_memories": "", - "accelerators_per_node": "0", - "cooling": "air", - "division": "open", - "host_memory_configuration": "undefined", - "host_networking": "Gig Ethernet", - "host_network_card_count": "1", - "host_networking_topology": "N/A", - "host_processor_interconnect": "", - "host_storage_type": "SSD", - "hw_notes": "", - "number_of_nodes": "1", - "status": "available", - "submitter": "cTuning", - "sw_notes": "Automated by MLCommons CM", - "system_type": "edge", - "system_type_detail": "edge server" -} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz deleted file mode 100644 index 72da9f1ccf825115f5a40ae85eaa180c41d87ada..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19297 zcmcG#b8u!)(D)nM+L%vl+uoQPZfqMH+fQuUwrwXH+cr14`M$rZd#i4}Rrjy^*O}^> zIdx8-?&&$xpC*cg0YT*jF#`d=?DBA0m#Swx^Q|F!CqPMeH%pe8D@9aO4BF^C;{yg35j&)7qvzVVh@zIod_CV@^nTUJjSp?aTo$fPVP8!9PUp*iT_3Inf8~A^^l_z4 zg(Nh5_KcqzPw;>MVOGwLd&2NP;jw~&0owP*8ONN{UwYYmNC5?H^pwrYyK`e}$CN-{ z??ry(_f_F9VJ55sOJqTNx3`(p_^gi}pc(XM$`2Ehk>|m$Ft_&};K;y!!+qb#E+EH; z4tVTJj=O5@z?UT}F!dwBlB+ooxpHPtKycb7&d&tfctM)k`WC_Vr;1_dws$0lxr&Mw)M`6N}DO1%I={b8w)|<)y?#8=SHdd~L%s&0L#F$I(GZk47VS-$kI(0(NG1Uk(05o*5GO#TaQ#nRt_gPd+~Q-|EM%NP;aT zYx?Jyoi638Ug@j#5J3$9UOW4bm_y-@O8|^#pPO~-XLOA(YKk78iTv@>LRf_u9_e<( zn3~qF;&l}pxWz|t+lWc1aB?YCM^ih~8yW)_<_Z~)x2K`AM@g#oHI(Hs!%UIrh5YJ9 zof^yolp=7hr3Z+YE5Q^3&`^h0ZQd{a;}N=xP;Wvwi=(4`j~N44pUYs10OLH5!Bo;;;cQfluI* z{-5s|(bdHlEcHs>$QN?q*B8RO9wZ6_q=DNfEy37-HtQw-sSpX)N%-dM_GyJ9LGX3G z|4N4d?Tei`d0}V)>Uz{;jE{Y%|9TCfPUt;$+;BsrEszC7rvew(NB%rl{Lk=?8*T~i zkTi@yT#p5pMcz$cdd{4Qyu;1l2|4I-dT+dISuQ#zKW&)5j)%VrW4L@;u`^jSvQcKbWHYy>G~%R>n2s z|CZsw$jb{kPBW<@Twg--mN5)i+xfgay}1S2^$>NwJw3{J@k70iC(!t(^xz{-_-DNd)Qa7ccuXRaXOpDAv%^ z8%W9z-4T3IauBVN(IUeBVAQh+^bpREwQ)594-ZF+^%f5u=wln(}srk1X<)lf>u*&j|QG&L@r;fA?Gf z^K5I`!j&y>C#d!{Q;W+Yj7^=gF(!W+Me;=zr~;twi)NcdA}2~#iPI44JSZg=YxJYpf)tWAII{W2D4aH?=OZWTB*P=INW-oQ3fZsj|NG@ zy)?#ot1E=+&Y?u8iU<^>g%$9P1&7De&HpUro06rQXX228!7@yEJprS9OJ`4&`=nbg zi&r9*&0Gg8YAYcd#CCr-H(CQg%W#S^XfCQ{cAN`%uCHixic9>p0e#np+1SVMGkljqJm+=f}5kLt8I$wQj7f9A0^|!9t|w=(e*V*03bhBRVitt z6A^3-qmQ&O&vEKR;31ARgET4zjU5aNg%1|`?=;5ynW{A?Y!YggdQAe2Y$6dfh8yc> zevrryg4%+q>&cCfjb@U|S`lJ7n6^g6>gLiDe{#b_u#?iU7{Q@PPj2jFAosio>&po{i^pLQ&*79Ns{a~F9CPQWM>O=SnU)=NPe0PEY=coN zm(y8ctCuyQf>BzMn)EzEgp1lC-Q9UD!K$V5Ysss_NldrgNgt(;f+tNd)fk-@wAEh@ zC;`cf;2Pyl$u-U8;}%K-nGxD**ziMRFE&i73)!?siIufy!sHf8Dv7TPV8+225Z(p_ zc_FW+rL$>DeQW@m+*gV#?zK=mh#!MkNeHC>xu@u#MNNt_Csyc?&3QL>kfEDvf`)28 zXRI`-VXHPsIso{QCg0ltO))GBqFRYHa)`kye=!4{L|+jiNs&Skg=y|$2wAInJbd6_ zNpJpV60MhK!ciMNDh*1JcY1?!YAV+mbinastLH#R6c0?TE)L?8pg^ zPcdc=+Y;@J8em@za{wn-4?#Z}tE-P$FNNiYKsCXBNX!j6k%Xc@Bo5_@zk;$hdz1#| z9lp=CtKX-bHr!NCSH5dLW*rwOKAq4A!e2j1Hj7g2*P&@?SgQpkD+W<~M1!YJAzGD$ z$p_T1PLT2=zNM;z`w6n23HnqSCaaTi*tCTtBx5t&hca9o))emNP0Fb+Aw-BX6UhXr zf{P{Hnq9}C)i%!}hvt_ggJ;GAk)gH12Td(Apvs&eRz6zajiz2T zP+5A~og^v3hOYNh5+^Ly&j)VYSh9yOaYo z)vvYQfI9bIY~$j8=`=N=_ojx{d%AcVldsQ2ax`lJ6HCZ;n6>^q)xd>#P*3 zf9GqGqL`Rcj(Kry&PEWW->}cY2OAtRMg?s5tbqid+uO;W(b&(O7MHc6r4dmSL=HfO zULsCAS{msKPU%xY3cj0DRoFl`(Y%8Mb5Yme!>@F3YrH^epbt%!{Es~ zzLjCpU`Y%I=~$uHrBYJ_C5J{~BHQgAWvzRGnm6qM_QPn=`-zH4N&42Kk#S2tx-$Nz zS9-QrAyx!(Na}1LGBv!i$<6P=a4PX*TQ6?vt7sQ_ru=3RZ1_9SXZBTX%|S%^I(t_S z=FEZLhshlpIq^xb;)#-=OrJL%IPAi0LfDXP#w zW`tTm7CN(MZ66TvnXS%Sad|L_>^J>;6S&C^hD`uR5ntEEyvhmYF0^y=0a>hGY+B&W zZ{6e2>myV<`e9Q1^xO;+N{E7c)@&XX4+jjer@%wj7R&DdgG`5RZt4HG+;=*1E4**? zGz<;yFYUoRX4w%i($9x27hy-swhldy42{|Ci`=#DJOL`)Z&$>RjE3ewOMN>@hu_G( zd#A2kKVdup>Yr~*PI+!L7#cMGc{N^DoQniQve~hzc(SCT2p#nI(`S-ER+s=uo>!8; z#(vEV?#$0`*A0B~>A&9SPCRSZi~t^Duczzrr_}F;&eL_TEi|7}1CdW@CEx*O%Sykxep#h9UG4e6^JA%FSCagwJTl-Qju3hL%P`96 zn6tS#!R~u#8v5Zzp#dDK9|18p)c`hRN9_VH#=;&U-utmTA@N2UPsQ7z852+XVhbom z=8Txy`!r2A7|G-yu2q4ek~A2`V0MHle5})d6k`zB6HRj6x6o0q>t==2>1YDdjYgp` zsH8^r2vGpUKg_^J{d)MRcZm){%W(kyYuwH3xAh;l$Xk_aK>=qac z>9DU7vZaJeG8}H1KiG?Ozk8IZOptH{f)FBPu%bOe65hf<7be?hG^I2$YbDvw{JvVtJ5Ou8$B(k~j?jY{$rtf(<4NLlGKeAg)RZ>4r2Q zB^3lN%HJRA4we|{rwJM{#1*{tuT2zUGz~xaR4y2hYmz2E05pe#Iaef;(~62B(kJ+l zlQDR$qzDVBIqd<3Vt9X{B1LSD5Vs=&4wy(E2s&zT7mPSG%4{rxR^JWl4H#5&EQCzp zPo>J(*VxQ~KcyLgI*zd@xoTjD*yaAFtUH)@h<6T&+@)Z+<06U3bCDpbfuup$LU`OF zV3QTXfm*e6{zV|wifEitsCPIEpyYmdk|zEk%#-jYQX#TX<#m|MKcmlZ4hYcT{^lO! zlY++gvVqD;7C%`Y_!kvJ2Oy96u|)MUOM$6}#B9QjvVh|7H;)v~`@Kb>t;_FMGgO4)^ z&DX}27W+W}_CTWil9~MXfI>p1UZKUHuz{v|g9S-LrXVE5zEa4d2Y%LzKN3X}eci!z zDyIM#popNF5SR(2Q&eWed}zT0_rf&(QI+DD(b#ZU$*}RrzStZP@p)~A=Iq3<<7i$` zq{dN>;N-;DY%Jtgf5Dm-^kI@|aWIVQclzxq}G%WkUQ8TwxW0T%lDYnKccx zz)B^U&+i4L0jpw2H@mo-rOCAqsDx>xk@y&Jp>LemtFl66t^;-?E%L%Or@{Eg5N6We zKJxr>Jcdm91#{8pcnA_3W=9QQvD!a%J9@^=#O8|B_L~#MLo=>ll9m|UT+7#~60EB`>>r%xDYj_K}$_MZzUw^W-&0 zT79^Jm)j8D52g6D5xp<{b9KIY`Pt&gOt_DT)S(OFHt+!qaCrVypl>|6^6db=ZD(J; z@OixdPcOH(AkDJ_5^@C=34JsPFTi@}fKLAZirWxx{8ckb z(uDq*7Digx1)3KA;qLanYt3WAPL6FriV^U@-^qOWyXXe%J8o$1Bz3FTtkb0@v9R{L zXOK&#HljhNNP}VpI{4T_K+G~;>8AN-xnmAX?3oE8>brsRC$oL_V;w`GyufCxXO*4* z0n^WRhL2(W)3JR|3{)K#J=>*d(7}X}+vu@nK1r!+SD9)%+3u~yHQH5bk517$Z>~3_ zWy?S))seN^mqlQ-pfKFDwTF8FzMd-R7u+kfknu_pi<7UbBB6`*&H9E#U`Aa z-rRJ{B{!WEXR@}wy5qklkfRH*x17IP7Gu7keGl!Xzy3UsXJeLRpTe|v-+Y}ynk`s- zB0MKUJi zf}CQZedr2#rF@Ujehnz#0fE?#=~WHgb)p6a=>3tGI5LhpG#zI0@}TjMk+?FhbTg|q z;X_pW6Wu>%4pX)>*JP5Ify3=YjX4Y6bfA*)nl8N<9W^65ro|#59}JOhCu=2esfry3 zmu6A4Xl;stfjh087fv9HL!#DWUKNMh7cM<*x8~FyUod4Vk1haQcjC!cVJulo%=~d# ze-?&hVLZBa&mB*89zKGo94yI;zuMPTtqQfa7|7jECbHYCeE!5kaBGmc_}Zy^l}ao4 zh)R_)>9snICh3=s-B_Vxv*;-4XMbFiu+V8)mB5-p8`0RHYu-jCq!Y{2W;=5#6{{!~ zf?zj8ty?`ZtJk?!Z}0N(_Ryw6Q9y|FT;|)Y_R>C5To;BsMcUX6CY&4Rt-`@*_ae zHb=UW&FUIEXgw4*-;=Os4*DruvH5^t5QAVr&umGj!~f^+aVhN00X8eR0l z!Q?wK-B$zcC-`|xB01SvCm=>^S1R$8*N}F&S~cFmnmJCn>$XUnZR~k+`VFR)n~W~H zF_p^Nl)vMlBno#Je|7D8WLd0x>gBLZ8wFLo?eu6uLMri|cBkjoH0E*~Qh0HvWeSP7 zEXwJz0)IS8hn8H_ua&JC$!dhS-vX9Hl#U<75~_4}Q4095XttqX1ZExlVfe6)SsYA_ zmobJ1PXmXNagwI|K%@n)~a%QCp>k6&umAw64-Fem&s(zRzP_O{dcQKgZWd^m{& zSC89fWU19O4E>b#a(Ic01;aLDCW#og$$hRmYnUKv$e5{k9lCv(V0I*kGP^I;0YF9i zyULOiX83Xl0Q%(d=gVy4w`|<*l*OOv6@n;>evBnTlqg-$2S==eo_#!-pr}o*yq- zE?}T@J`8(UrTeze1F3|}ZYI4GS5uO3@vhnIpSnOP^kCHN9jlXeu|cLz^K0RZzQvGg zr4b3wfJs|li7x6T0qx!mkfzwdrtr2BbW)ND_pUA2zVS=HJK** zF5612`LG6SH@sB5%H(*2wgEdnP2Gw?7;S~%P<^*d3Fk*VqTVa6=zsp%rL0c-e@Owh z6$(?EVNImUCmDGTWJ{Uy2<4-!sE4lh>9rD_55_LwF(J-}lSbMbxh{?DDx10NMwZ&( zyAoA6I3jO62%MHvMz}k0>zex^Rly{3-l!h+?WWSH$y(ZPYMk~{C)PO}gnpZ=$kKzc zf86(nt8uc-Vifo_HyL7L0{I;9k0fe>^9d?OT3dIm%1&j0W;|-~3^5u%W&i?CqAOGE zYZ)9Xp|$6XsNss2qPRy;b<}MF%+3WX5%QZJ2LP^c;o8M`SV5+n~KfmV`n&&t##JIX4y?8Ee?UJnIT)= z@L(I}90m8dGyGT3ux2U-ca!)moZ0lG>3_i!@aEMWwbjT;Y|bDg3H|Mj*5X|&#}`J# zjkSW@ld`F(B1bO-lpaz&U|c_nk_84>&j9{5JQoi^uh`+%cxZjk(Kb@%8EH1UKjbJ@ zPSU9qwAs@RD4B+`Nt*FCi_yH!aOy4V6@#J@yipNwwIfj|5*?QAlAt)Ufw!S*d}H;_2bWVcL&EYwGi)m zUHOkiJF(LAZ|rNtD{@AMzRIG9yCnR~`7?ZMTbUtQ82T^?$;7!bb&3ml1-^nxVbjbW z8WPkY{VwW`fFADVwAbmL+{VPYBiAV5>HIg^4PK(3Xp;3NDja|s{O8aCX)}xyd0of< zsnQrA88uNx>5P`(63gazS%+dfn-K%5=Ru0f468Pu8_7tkDJoLJzV1zbCDBDT&r35i z&&_#4f1->s6GcTSd>tD?({sCc7D8jhqilOk*oOCblvPD(J`yJJ8I-4e*uR%a*w%k1AlEJP2E=GJ+d)3I?E zM>NkrKtH0-f(7i6@L06Bll1Z5wSo4`%9Y{@3-<$yax$4k#3yGpwkNs9jbG%j-spYL z50%D4uIs;(T!S{E$r>Z?9T#=$FHOA&J)Ts!lrbWcukxF+jK?@!8u@5pxiuT)j4ma< zoe?X1TGMk$(TVen0Nud-b3S1~mrTfv*ZU@Zj0?^7p|W5n9vjfEu)C;2yDA`Jokn># zvRT`bdpkEu_7XvtS!jd+43+5LayIFQS178cntKFxXR$bcG@oO_0spE4n#H=j6`+eo zIi&K-vv=7kXeK5n)w4|DcRitbbB$U0W3fphrq2U zsB69jI6l&PF&^CAZzXuL$}J9ebWBEjq8EJ4lYLu~HXPd2Bf^=7*i!GDxK=hv9NvSi zAtzpwCD7q}{-V)sTafLf_U!M+MKYQ1#zJgMNlo5)+iEj$8;Kot>C9y~bS&nkGtLn_ z@-KExNXb@QPfsT~j$~_In}MXzOFV3*Km-Fip8X&Dz_)c`sxf< zhiqvSd66w=8Or+y9NW>1P{8nM8rVSy9a>n+BLo5}MKitKMl5Fue3Mr-mgcscZpv}D z)~0+60}~8CM}O=pS9|#E!WfQZx;a3@J%GY?GGpQIZZn~ z%jrLd$XA=eLb!tJI)(E*Rl-BNFK(dBCPCFaqK8!|swpxiz)u^!Ikl8+mYXL|mxM8@ z7f7k>-p{V;{=RLx(&cLE#c(HPR$lWhly9mVK8`P6p+u%ljcC6#X_ zv18GBgzyi>yD45^*fvL&Fh0gZ)tfmo-W~2JBGon~-Inm-EG<0YllTKoDb<+~vgMVr zFWspfUzL*2ze4&g(ZW*rMJCm$9i;t_K47q8><)X|)`xRL&aKXal&C-2bp%j(fO7&@ zKg|o^V6%41hUAU~0P?R-H-qpluog4B?(ByVUGKwM3A0!Ag#$K27}}nwxOGqDI8*ENCvJ1{s8=l?cYMNe2<_sIr~haC^4yphzsg zL$Ub6dP92wSJUM!NI#-Gt!H!i%>kfaY~~LNq38DpgcE#rb_|d`Q|`JoCLc+h{W$4n zRg;?4j`Q})yE7*5NIPhLip^9NK}JF)3_3L+FC4j*?g3~kHeYdU)N(d$_mjyUbfUGR zj}K-Sw=dW`la!=zR-0I^)@%x>qu4I4&uVhbQ1c#4qOG-f7XN_=_gv8QIlL9xD0Q1a z4JKTrEJ>ctYcu=a<{HVIh-mKPn6@g}N&LcL0V1jclyPimdm~z9oi&ZAem9xA7>AG7 z92Zj7&u1`zJYGPeqNWMv=?<5bsIwH{*_{h@2;13)tzU6R$%zd<7yGl#a0O=Sla8iC zYzxKEJr+a!5vd?0CGocbj-jJ|D=U?Z&r?3xb729gHMHh?1!ge2v6ja&E^?dT1|i(v z3+bY6o8c#<);*P@MCN~K|9&j9Qa-E3Ac_VD=fVxbDpcD=Z6>s^n@Lt!&NI2^xR+s^ zZ!cElTruk!6S2c(dWKMk)I6zLo9?oj`Pn?&)_$g`N;b-4=Si3_!wxv_JQXXuR@6az zC~j~lAq*z_7D#ty=XH@}KuK{$d2}MzPiHW=^E|eGd;6brbn0}1bt`Ud4%Vg_4Y>Nr z4wLK3Ygv(H?fioLg_P{d@5RaqqH%=4#b_)3A84JKz5wN+{CdR*6P+{E!seq!u7PTiI5pU+jY7DV#SFXpJ|5{r-?E{NP`ov2KGQ_s8@Am zOW|l3P1y^OUdf0%)l|69im!E(K^bszHkNW_jVtc{H=K}E`9N`YrYT4%eV~o{75J)8 z3hg?#N;;F41h|ufT_OzF9OZn3=zlT@0r=as&p+aAmz5X~!R9TyEz*2@gVE7s!r#k- zlfv4PU&#jm+osE2m3T7wQAb(@Kk0avtth)wyq{?QijQ*wY=f&b#|$>u-go239dSAN zrx~B5m?%TSvms8nr<2iv;SBh@*;->fc4z+XbP0I>jzY34usrMI+jN_%_GVezr1c)7 zC6S~rGO~?uXcg4mtnm4RmyO7zPIyT{DfE6FhIF5DcV_#gI+Ja)klq4ke^GOT4-qH zJ4TfjmM;Q;?Y0?pR3hos?Ao;%|DMDWIgej^e6ucvYEP6je8i{ z=z9M^@o=AIxobJR`AD9Ibe&QSII(2_C^Kit()HX&$vG_6rrJ)Jz+@a}IGqdh0W zs)-K2ZAK}-W0?QVx)AS?;{h9Ve-}YhMf%u`UzbeteCo6@VB{o$wd1r4o$yC0zCGGw zbL5+J{k--ZA{AlECZgxUH^Sm6J(^LCa8I|L@~~*@86Q~^Nm`Q8NcZr>6>OZx+4P$2 zVSXCYd73XZ!`u}w)``PxmaXzgu6arjr={1ZASvbO%&F^m2|9-8vEW^pOcyII(>lWoPb5!Q0x%o{5CfEZ3U zXT`c!@&(}TUAOMARcI%%^F-Mz3uOoFO@$XYpMA$$cgcrwL|}KEU(41Lm6kN=R-?+} zybL}$TIbCW{AO57uGSUza*4s-aBU{GKC;Rwja@tYR>9GMF}kNgx*3u}d@QG~|JQg* zQl{HaPMVjspYmfQOs`-v-=VT8TX}VjY{d90+>&|gQd74BaAL|#4?Ss+_Pu5`BC%i@O3^mHgG(?dO(A(|iN-QFK)dO+b>OIk;;2ZRY> z?@c~0>i+|0Gihw3_^P3=uH2^axyt3IUb+zC|F-e@WW7IT?ENj5SaR0C{32{ z+-G2g`(*#VVz<$em^h`*a~c}mL1=JHJGe8J+7oeI_|wplbyzw#;Q$eYIe4b13XwM+ zO6{8;N7C4;K_JRD6vT;Px%exJeW_&JpItw4C^tqtI43-}b! zaMr10(0>n#Pp;kNh(4uaju*DYsE{dF%aODW7J*7HL4= z8A0D^S(J6IRcQh*@(qLU=>c62!6*g!)HJ`9T+oJlqA zj!7QPz{w;riMppa6IYqxjtlw>9utq&nnJ7^eOM#aPs4m7b#KiSH&PQ$L&X<^nvarO z&KV4WHV9&eG~-N+JUl*6u5ym4oo0#EFc=GFx&fw6J$!6$P%3b=BzQX9@;)-;lyr;)c)*9yfDt4JH1uPKoWm4X_7gM9Ro2qQRH4{k9pwlE z?dD-c-M2_Lt2-i9E#q)7SQuqV$|DxjBx62L*(IA*VD7GAxh8`1v;X&{Fr zj89Yk|Bq>ejO@_Tx)au|P{e@`SG!2|1f|nW}}O@=QpxxbSyGS2Y}F~a7;VHa5U9=ASow@lo?WltHzmFy+Y~x zUhtI_-~T_LMjuM*59P9zVXzC|M1L6AlagFn*N1aqZbZ{xC3(pt&Y5-5%h7ZQQFNb- zx4B%v9xiOF&3P#7N^TY>O0IEn_TFY|;@hLW?*s&a)`FXB!3-UCg2_GfE0{CnUsL63 z(K)Hl-QHRJn@)5DY^!h9^XvYpqX68BXT9Xqi8o`KHUqibA}Oj@TSvCsg=(=+8l(pk zVI|_!D^r%`4W)O5TZpq|M>#o6ENxYqg-D(g<2zT9ET>9lco#Xaa zbIZ2P_G2a`?&E)ujgXK_%iu(}T-)8lN-T%P1v?vkf=21`6sRpomj6R-m}}_r;oC0~_^W7Blv*p2-?M7-ganJ@BROi#|J}PY$_r*z;!!3Fc7jE)c6F~D3NcCFUiWeQ zJt+5u0d&pw_nF^L+(q^7>VNkIL30C-wJV(CA#jm;x!?n8fC^AXFh&no9O#;sF^?|)%A_7OeuZDCMg z>3W;w@FD!3W3`0h!82~=4X_N3YD;qQ-=QauN1^nQvtL77D5(E8nxnIyMq~~Yf(;O; z&h(DZ>4(Yy=$p?~#-KG8036>Co*faV-~%Achrnf-t-+^Edub2w4hH^I+}0bKQ^6|< zh%tgGB1WJXg4i6uz%tc9AzJp;LnMQ>ib%VwaQ=P>X|ZB@iFwwwSH&{N9_Unr+5+=) zqi3=fk5Vr1gz^Xt5@DxmE0yvCFd;J93jfD+{Bh?+h7kaOk2B$q5F_ruOYMZ;xKk4{ zkoMy3-*BMEQuDO{3o^m}#-cp4?|PeYX+|SPDme;a->^!NS^lm%7ZQpxd9e4WlQ0pf z&lQNa775AE)Q2)hjNnd&6!ibM_xIMFiL@FxsJ-fNPsE_x4?lxy5&NT427lY{<>z2` z=nu1fxnXx#!0%Y_BCt63P1ddTs2@zC9V8Axueq31qEyOgWqy@l>qvTkVC=)@5yC_? z-AU*O1;Z%8_8lzUPw2NAAQ$(&3*GCewoXHTdGBSZ=1bu8h4c`|Dq?Q2g^wiWC(v5n z<*6Fuo!}BiMsZ@{CJGop*y)Xnm#)u@7!0^K z0FHaB4%4{>dTmh(iA^n}B%~>+?H`oiNyZJL6UG5e5154%3*=vfVU7jNMhN8zr8y0*Hya#fpwfJ4v69jUwrSO^%=9} z!$)NIJmTDM$cn}N!BubU#rO_UfV6{7(6=DMe6T3MhyDKm5C&PQZmnZdLqkS4+A=qM zYgT$NibA5{0PiN0(ajXY-@>&E0m=CfJ6VW|viRHVRLGF&T*hw5(OHYVLiwQ}u`54_ z@khXMVF^#zLB<_CX28+#nc?e?UEs2cplL z6Qx|j&4GK!hp)L`4_*wPdf7AYC#G+VL9Q=E;&fiEPyfl^-EKV7*OfqKe4*`9{!pGX z3fHa?Y501$3FI(LPn$);b^9z^zrcPnWkT%V0aHK6_P+$Wy_N)iKL>l+_ypz$B(?5CM(kLeuLE%FGQ z%@vNBr|L5yL<(%twUfIfWmcg*+seL!@)dXz)=k&P8M>$?zrE3c0%+g4Tau>A_bpag z<7P#QcogI)OyUTt=D*gwUYd^c7Y{DwCamU24u6MElXEZUIf~ztL-MPh^Fo`^p2B~k z&>h;@Q~L$Fz~bk^EK%wT!To~yb=(c*+lg~6Ps3XV=o}yJVv8K$!qTy_GPb;MjbDVh zY$851D8JK@@hgvo-I{X>gkd*{$OtkkW1Lv*s>tlZgIh)7oHOerB8ZU(N@_0bK2=lE z9IrN2#+A)aijD8!x)Pk@X^M7yc(@kC&ounz)}8dvrXA<_sqwW0us&+A03;|KPFf<5Ucw%Oqyhh#H~8GsBLd`t+W!+RJ4F5OYVr-$0qctVMSpwIit z%<+OI&&MKk#zUhQ?ma9c!$r^F$uYcAM2V&{hAYVXJJU?=b7-E^*Tu0-kfXR^1Fok4tmIl6fhddz) z^%|&W8vLxGGde6D*ytN}C>acca}(=70U ztsX&dM8TC+44<0oGoNqqw$s}u|AeTLSerb2+Ruce7efVZT?<-3)kBQ3(3 z0#AaBWjr@lD?z(`qId_ZB3t|CMjcva2Hf?>AS(<#D`K?shhR?Q4U8g8482mTK|-)! zJjpk1T=MqMQZAWv0*~{FW=h$3ojq!fv-MT~>eQ4p`aK<%>qAvtR8p2CZshp!l*LMU zrzNB{ks=M$Sqt4%{5OTloOst}J@gllM^c2-)J7YrQlo4t5ouY{Bvx{CD6?glFAyWr zcCwgU`&;q2s6^qev*im>_>AL!cv?9V-#6XF2DJD=PjsT@cw)I%L1wkPo~_BD*^)?a z0t`yVvuIlK9c#)v`C6=~Jw^%))*^)u+|h8dMFO3p1`^oQ&JQXQYBqaM`8pU2{426A zT9!t9)D^Yi2q!}ttb|>yx>1d_#^l_QDNT~1BxvtE*OPW2TJvxSJKHYqYalP13<<%` z&Zlm3w=2c3`Npe91Ya<*S6hGQG%~;4H)P4)Tln&fd|<9RXA36HeoCE{X#DhOI|bu3 z$t}IAd)Z?s=jW+)`7yzlKQS#X z&WEAC#>G)4+hBhReX-O~8DW#t+fiwi?Pbl=Q2cN{f{3Kc=_*`k!RaC@HATLg!loRw zw$_-nK3k#fQCfGfeRpHb(enMf(qLlY#L}=y+Zy476Kau;q5PNP{%xV>-RwBj#b+gX zr;#D`B{s^FhuYD^(bn0(q|aVM9m%)Zn@`xPS9FQ{Ei2Xraav$nKyx=Sy7$99{bcW5 zFCj+|u|R>eVeDd|?dOO>&&SoJ@B1UhkRua*4ro&WC=3mQ=RdvsIpEXY>g$YScHzPc zxN4f5IE9D<>P$i0XG^_jGq#CYnUui0wt(mGmHF&V(Woy-AdnGi^B8BaF^0C=^z^FI z)vQTDU^)JtbVAwx7p7$dnxN$WMM8)$EzD1EGx|#K}dT5m-)SX&C&TJ zavOP;^W}Y@+~oAqzxve6og-7oC2*?01`{%+5s&w-8ER`WhYae*E|oLLwh@dB7jdH6 z?(bB*)9P9KTt~xIgM_Z@y4kEv^vRxnR>K=+m3;>JDv;RKo3E7at@fvvZGsoX{A#!k zrDKQx0=?&(Wn#THw3{H8o6LT<1eA{UUEt2tBmG0I+>_=C@|%R~mc=oR*T>eWJzKiZ zg;)i8)wlU|&i?t;J-yzk&Bt4F7YPmt_~?#(MVTDk1ZI&+qMwe;x2Zx!1ioe^lF z%Q^Zwh3sf4NL*$;F}GTjMYNyA&kuT}W?57eL6$b0k6ta`+K&40tJ3=>GLl%D>oc1>!kJQpdBr?|(hsspby5huLrbZO+<-FQR zQ9u1L)2111Ltc-DF9`EHs#|{#M>fv#As9^*sh%chy@9Q@K(-e?Y-08cqc@~?B51&9wdNr|Pqe1BPre_F#@-FmY{!l%&;BW94UbJW!k0w1Vfav=Zf|?w zpf>wBeyr|DV0{_%J9IZ9e?r10zIicItt~@vdpKdYm{B01pQBMXvP}CbdKXd8iBklR zeQKGahs>ELC!f_#@9$AD@OUYP2lgCb&1uh5^C)ww-;(;!(7RSe{Q-CLoAKlX^itFk z_h46eC1r0dxHxCQ>|*dU8}wSIbXt0QwSJmmeivZ2nP!_6dkG_dT;eOtexdU?6Ks9l z>_rgEYr+r6`KXr@FnvgSF;w}d3%%_T$7|(fQ884RE-$&cs+N`d-!w@dCGiWsv6)bt zV;LX%+=O|;c)_+M8f2Nqvj#>in5*`G(*+xv*EO!BbmVmj&GKum;3ZCUJy6~k?-Mgf z4#%cGCM$1n-cs_EHkqLc^+ZprKOFggi6g4x)ExS0??&U_#QC5TsT()h$Z2)0dvng4 z;PFSby8fD9ZXjq}S6OnX+5g>w*@4GLFskUSPFlf(dM@^@kM`TwA1eK}ik}|#mlAt= z;5&Ha{DsWV_2PsaVobA0ygwSwzldTY+Z-n3DGjoME}`U)4Sc8xF7PL3z^sn-d^|ci z;XGkR?KnmU0yAuD&+#X#-vouAo$9e?7HT5QZc=+w>Z!*>lN=Cg#Y`*Fl)@|{=90O+bc23NQ`VF2%JsxUO&n8v{2<*}^d-jx`egj`#e07y>3GVEK?$_v;>Po89B8;N$Hy3m8-kCTgXXR& z*Yvo1IGu52nJW1>#i#e>$U0UakMbie%P~B=tLGIui(MHm;!a<;%toM8^yH^(a9#q2 zB&_D+9!Mu4lt-pjWsyBdCfDGJ*m@z0D~ucR5YG282H~f?`U++rxh^uY))L(BHX=*=Y$Jby-y3l6TWCfz1nd?@9ORb6dq(2~I5@_w z?L7Pb^9DKe#*ew9OFRD9S%<|y8Zwo(IJdNf1H(p-4#A?yh8lxmF+w%bU2}Sg0MmwI z@(b_`xV2})g(BCekp3;;5oNKpMk6#$Jk*;koEBw922YAuA9D#~9iPP+ykzoO)##L~ zk0xtoZkRg^PgQH_2ZlD?ha?=4-iUTSNLy!x>32bkc=}a#UE#zOthD2msx5^5oFeF< zll*$?o}Lfc{5P4qEuxmAc6aP4jkJ`m>E6=nEDA+|@VMaBUAtVwqQ7>`@!kJA)k+}E zdfN*sF>9*a*Jd4yg!KO4);uu^-KTQEc|D?V-j8i1%GnrrZ=gh{SO64^Zl)6oUTNHB zCuKtvd7pt12)sfTHqEfEXvxK}GGWjY@SwZfA9~V3syXOx=gW80j&vA_O(Bj@B#{NH1EoIfmHtqYW}Gl4d&o5U^!5U01GdrkU9>z z0yDu zsx|lB%eU_ap}Ly}R<^7{HE`O=A|1z4871KZizGA5Y^3+K7hGCl!Z_wyfy=~WZk&8A ztyWSfr8onBSV$wH5ET~x_ox0_aHmH0Z8f*axDOrJnXLw+q=Js-VgN#sp*W3@`*BE; zkB5^%zL?h@6Iy|~!)l6s4ZZttn^I#pb<^$=lMQ}a*g19>#!cj@nI@`BSlVPr7x8uD z<|h&{sofHS(6Owfq&S{@uFHagf&%u_ucE@G9p+)!5?f($w4y7gXyBe&%OE!5<)0MfyVA_ z)wETodTHjip=!gvZIfX<`~)ub=-7PH>DDa=`Lc5`IFuVK^uBEqsLh0HlCoZmM~E)F zS}!eXU5NY?hs#dUn?fvmG;fM%t;@SEP2Is=LrA#fwJ!;8THe+f-H-)j^-?m{MRw_l zqGR);oh)S!d@V;-Q<>Pm$F{JZZB$?V*T9(Skq*L z!`ULqD$pZDnr8P1suXpZY&EL%=QiqE$+7H}pGn z{FkXN&0Y_>Y_NwPlZ&Ncox;nyo9GAMEUMtUX_EgLWM`br%9U@9MPV%)Ey4J?VUe|a&_p&W^ai7H4Rgj@(TVYaI_fO5xAV?`E)VLYp2r|kTq}k z-#Fwsh-iATF-E-BnY-*muXTD%OE>$|tTvGSuthu9-rZ^ZXIi#R|J)e=1=!vA4~!qW z_|JVjP5k%t_24f*z~JeJw}W^8_2wmpQI}G&47aW&_~zSpZ!tk6op*2k>)l^|_}>5i z?SH*}_wx1N`{%Fg3W^WM$fgW55*RUyPGU zF(2AVK58gkJKN}Jm`{@B`sb@3e;k7QHywRwsAPTFzg!e|G)}77Xj~yrI+s#|HUZHz{0Nq=^ndJD~Ci%#gWqaD@aH!?< zA76)fHXMznAD8IKHM*OX+8fK6ars&`1zt8R=4SoX>L9JnBc_hl?&fputUKCVoxHhp z7hWRUmlOjJ*N~fC%(1ax>_qcYZM>}E0FU0T2;#uXun#>|bw$>ji~oB5`nwgmSIwAP za4^eiNsxg5U%uM77sk0Q<)|EfT5>U3|MlXM_t5gIqQH4xpy${Kb-b1X#2o}sG}UA6 z`1`NjTa(P zN4-5F!I_c;8o@du@%Kw=gVX6~WV34c!3M8ESb}6xO`GKxK`~Fp_TzN^WAKO5*MI!S zA66?zpf5dNq%hzeKfUbDo@Qe?9!+QEANzF#>Ga%YKL&q4eI301?hPOp^Qw_Wx85%zqdB1uSnc1AB)7_`06phqu9U-R(*6pG#>6 z|Asfi=i|Ej;-Qe{`0X1UJO`(5{&tn)`Rf;v3|=MU;pigZYB_<;g142O%@8ft3!X0k zGEkNb5@P|)$l|JuW`q&^0T&ZL{m^_nNo2xvp7w&bb`}ItfG$@L{(HjM2^W6^4!k`L zK7i`UFaQeDVHNzLCgzX-z&h6W^ZKf$+VNUK0l)usF$pkPGRg7Bx7OBxFZahZJUm+r zbGsexPnQ{=HZ1(-KmVB~<@uAxcmcxlRdD>bpmqZ>K{gKNXhl2e>}!*_Fv$ zzNl9YJX=}A<&Q}|?=QW#eoe;Zg%8UF=zuR1JiD-~vW6J9M76A0^K5xGRP!#ynDr!o zwq}}_Z*u!#nAvAww^yxL1~;FrqMb`Tz}Z-jX+2Y6cnQ@iPV|g5KYduf`DrnlzZI);Zi{N6q2tFoF+;g#9u<>{FG(ZE_Y{cAV!!!3vkb zwi1!LfU*(8c0u|GHVC9BScy{|m4op_GC|E~gfhW-Wb{ePlRs@pbR)T5DcwSl+oZn+ zvDVGM+7i0bdWFiHrLj5*ZqXZKL5*d4gLpT_09Vg7_5Ch*o@I-9l3lE74uF~@%l>&1 z{8NJ$`OhA_09Ma2dbvu9tjI>w5`N2vAL>GbfBwJE0&qv8Sjm5`%6_x9WMQrNx6KSO zu4q`U$ido}?rRw{$o!~#zQFVO{J+Z>_s#2nnQ-I3T_eck>aXZd`$spG*G_ z0Uyl9fLLB`5uLF7Bee>c`_jFJ9bWivE)}BX?{(rrIh~ogJ0s!kD BR9FB2 diff --git a/summary.csv b/summary.csv deleted file mode 100644 index e1aec9e546..0000000000 --- a/summary.csv +++ /dev/null @@ -1,2 +0,0 @@ -Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.7491,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json deleted file mode 100644 index d14c5d207f..0000000000 --- a/summary.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "0":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows", - "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.7491, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v2.3.9. ", - "compliance":1, - "errors":0, - "version":"v4.1", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - } -} \ No newline at end of file diff --git a/tmp-run.bat b/tmp-run.bat deleted file mode 100644 index 4a67015352..0000000000 --- a/tmp-run.bat +++ /dev/null @@ -1,36 +0,0 @@ -@echo off - -set PATH=C:\Users\runneradmin\CM\repos\local\cache\b512f8207ffe4e97\bin;C:\hostedtoolcache\windows\Python\3.12.6\x64\Scripts;%PATH% -set CM_GIT_BRANCH=main -set CM_GIT_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\20d7b1d3e976416b\repo -set CM_GIT_REPO_CHECKOUT=main -set CM_GIT_REPO_CHECKOUT_FOLDER=repo -set CM_GIT_REPO_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\20d7b1d3e976416b\repo -set CM_GIT_REPO_DEPTH=--depth 5 -set CM_GIT_REPO_NAME=mlperf_inference_test_submissions_v5.0 -set CM_GIT_REPO_PATCH=no -set CM_GIT_REPO_RECURSE_SUBMODULES= --recurse-submodules -set CM_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 -set CM_MLPERF_INFERENCE_SUBMISSION_DIR=C:\Users\runneradmin\CM\repos\local\cache\53f350da4cc54190\mlperf-inference-submission -set CM_MLPERF_INFERENCE_SUBMISSION_VERSION=4_1-dev -set CM_MLPERF_RESULTS_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 -set CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE=Results from R50 GH action on windows-latest -set CM_PYTHON_BIN=python.exe -set CM_PYTHON_BIN_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64 -set CM_PYTHON_BIN_WITH_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64\python.exe -set CM_PYTHON_CACHE_TAGS=version-3.12.6,non-virtual -set CM_PYTHON_MAJOR_VERSION=3 -set CM_PYTHON_MINOR_VERSION=12 -set CM_PYTHON_PATCH_VERSION=6 -set CM_PYTHON_VERSION=3.12.6 -set CM_QUIET=yes -set CM_TMP_CURRENT_PATH=D:\a\cm4mlops\cm4mlops -set CM_TMP_CURRENT_SCRIPT_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github -set CM_TMP_CURRENT_SCRIPT_REPO_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops -set CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops -set CM_TMP_CURRENT_SCRIPT_WORK_PATH=D:\a\cm4mlops\cm4mlops -set CM_TMP_PIP_VERSION_STRING= -set CM_WINDOWS=yes - - -call "C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github\run.bat" diff --git a/tmp-state.json b/tmp-state.json deleted file mode 100644 index e03109d44a..0000000000 --- a/tmp-state.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "docker": {} -} diff --git a/version_info.json b/version_info.json deleted file mode 100644 index 08ddbd1975..0000000000 --- a/version_info.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "onnx,fp32", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_cpu,_offline,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_resnet50,_onnxruntime,_cpu,_offline,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "resnet50,onnxruntime,cpu,offline,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] From b86b2dee6fad232bef62993608c4b8df2fd2b7f8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 4 Oct 2024 19:34:38 +0000 Subject: [PATCH 020/658] Results from R50 GH action on windows-latest --- cm-run-script-versions.json | 602 +++++++++++++++++++++++++++++ hardware/gh_windows.json | 26 ++ mlperf-inference-submission.tar.gz | Bin 0 -> 19309 bytes summary.csv | 2 + summary.json | 32 ++ tmp-run.bat | 36 ++ tmp-state.json | 3 + version_info.json | 602 +++++++++++++++++++++++++++++ 8 files changed, 1303 insertions(+) create mode 100644 cm-run-script-versions.json create mode 100644 hardware/gh_windows.json create mode 100644 mlperf-inference-submission.tar.gz create mode 100644 summary.csv create mode 100644 summary.json create mode 100644 tmp-run.bat create mode 100644 tmp-state.json create mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json new file mode 100644 index 0000000000..b42a41b1f2 --- /dev/null +++ b/cm-run-script-versions.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_fp32,_onnx": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "fp32,onnx", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_resnet50,_onnxruntime,_offline,_cpu,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "resnet50,onnxruntime,offline,cpu,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json new file mode 100644 index 0000000000..b7cf960dbb --- /dev/null +++ b/hardware/gh_windows.json @@ -0,0 +1,26 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "host_memory_configuration": "undefined", + "host_networking": "Gig Ethernet", + "host_network_card_count": "1", + "host_networking_topology": "N/A", + "host_processor_interconnect": "", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "status": "available", + "submitter": "cTuning", + "sw_notes": "Automated by MLCommons CM", + "system_type": "edge", + "system_type_detail": "edge server" +} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3599b5aebe18c3e6a1abeda5222157cce83c5118 GIT binary patch literal 19309 zcmZ6yb95$6^Z0$mb~d)Tv9H*6Hp#}eZQHgtwrx8bJJ~oJ+j(<8&+k6Z`M!TmpPH$e z?mpF3HQk>gjD!Zz2?I<4VCUZMj<{oWB_G*I(c~4AKHjHFIt%=;|t<4m6lNo@G& z89y-^;RYa8-D7;fonPrPmJxur9FBdTvCaDI0eWD8i#z_5v-;IvnAkKEOYrfM=Xvqm z5c(0QM<=$3@Yef&&L7y5{r-tJL-5AHFg_l`kNH{nPk*kjp3c0k^f|{$2*>x@w8O>N zf8xngCdS?l72(93@9_R@Xq}05{7I&V4Y}&11pgBlX!+t>EmZOeb(tFswe@MMSP7*& zH-GmjnX($6O}XgAM|Jp60-@mLKi#Vs+sU7svZ%*dc82%FTc;6x((U`!NPhBB2D#|O zjxRgol^lES#@&NuG)6ZTvKZVmtJ<$ghIxg%(@pMtz9yGYcVCh&7t?UNTS@J1HsJz0 z0aEMD=znwN-6|6&TS97=ab04>sr!+Jw2d@Du2C8?ksHYscdgI;S@-cSwHffMoj!5V z%FN%83B(ys&T1=VTC|Xj$<0@tH;H?O7;Yg(QbRhyIPrrO2$%EY$OVzVrDRR-@3&6p z@>Q>l)q3!t1~ML7JMXcZybl*t@U9(l+-l#wzq*45nDXiKZTGByXy!sjTMQhMmy%B& zR5C%Ex<+N=SBdJt!=F zAtWVBhq|U;*E0k5{OQNdHcTb(DPe|km%rJIn~+igGNKpZsfKj*Kd*?ojC1Y90sC4S z?t=r{@Si`jF9I2^&w2-n$-%I?k<>!Gb60!t4?j1aa-*EEHn*R)_c~nLZDby{33=Z} z3PX|Fk2qe=NRJG!NY1c@|K z(R4%N_m((Jq`2sOsl$pn{hb2-*Z?0Xq)#AId?h6L#EpA+6+fi4RJ4al9QAy^EROV! zTok!WMF9hfb$J#-SsD6_Nlwa|<_2u$-d-44| zdFr^-eY10NC}RHhi+eeMMG?9$f|XU#NEsK_#wv$`Mm5~zS@H7tS~rK^`{C5x7d|<# zNis=lUi~!6R6nCwYGU#bNS&5EjNy*;SA`7n_SbBmP~ybMDp5K@oj0YpLax3$>d#`0 z;bhKJn^DDitWE3OU(G4&tf7Q(J?0!xKpo6ysqM(XY%& zUn_IaI|_)=%YwWp3`myT6W|IHWyE)I56n#r;zQ~Lg=&#AC&e*250)zjt!_-+lW5A& z^#5#7XpluvsnJY`PA1p87bQnHO#Qqu#7$!sO;OJ-6%t$8d0E$zE{+WPrkJrRB*Gp6 z8@bvSuCdZ554S!^_2c%$%;*RKJIWH5HYZe{n4U;;AHG*XbbNqQGH5DUQLWfZnv1PS zYu_Gm0h5MZZAB&wD1~LH4{Cs-F^nE-g`v+R4Ptq^tm*$z%NmZVr%p@;D6z&Nr;$v8 zw>FR+Bvq|69fYMv&N9WRQwtyaGbj-_S`vIvmlUk8RVTAXp<5L&0XY&+&#Z?X8Kv5W zED1gvtQaps9m}Q%z1<|c#9|Qao-<4|4sA1BB<1g|VIn)6@2-JDvNr&f{tIi0+3j_Y+xte!x_Ll_lu0!-Is zmBExmbSFE~Py0pm))oV@gf8*}T@nN}+h0obv^+4fYDFdSXlhx4AyH&SaE{zb31z!iAv7uS{Ua&n znGD-hGt39bFg23=e34|L(gCIF^f43RCRtm#NHjmy5MpJ%%VN>ao({t5IVRyIj+8a! zoij-T{?biY%QBka2%K~JZnJU({)Vt4g%`!biL+p=W`>x6_DU^FnVqw%NGEE`G)7D` zY7AOGErk~3OonU|N~9BnV==;150;q#jRP@WRYFWquGNA8QPVQg8}UcTN3hPJo=xS9 zK|{o#OE%Y{ohZPGNtwjpQr`zk!p)LK0znaWVOukQQ9xE0_llS?vi(@CWyThZuMarN z7-F9T1rPr_NJ9k@b}$#uAp z+;PQB@V8tJhFHYPlQRn0-1Su_omDRd*$SgZt(g|cU~W#Bot6N7dI?X$-W48!A2rK# zDB;WviAH=`HdMn%vUHiHkgXrOc#u7rP5!V79*zp9+;%Xh#ci#q8H5ytkpp6(n|Rp= zG9$e{S$%1+K2PS&bJ?|EkYYjCQ~uaUls~l3+`tmA;+XVXY8m$S_udCZHs$2h^fUjy z<^&3fZp$fMP}9s!DVA+wY{==^5A=T{BUxaS!(fh>OSdXnz zuT#K9yP;yv7?NQkCUGCtyS`%o*lL@?1pJ^?U|!NnNB&^^6%6mbiBIrdq2Z&88#6I#O`T*s|2B7&8buVGa z=#A8f$Wh2kw-0qr8qpYxG&!ZVT7vesu9{0=cYDTWE-1imXERb8)|Kg9DfWF4w0jy zA0um85@up$GrgP3WS3&`(~?1hqtm;e-fqir>0?GP7n#h{Ir_1V0f`MIWzKU*AUn0z zmYm!}S|Ash%ECFBv3IXqu3lU4x869a)B0Bv^e@K8Ymk1!55NZE}GQSy_B! zV8SLFutP=`A64Lw4d5o4RnxQx5hk}wT<`v{T7Rki@)hCkr@CP+q;_pa&g!7VZH>!$ zcRa@$QnwbSy3ND7uW7)&I-Mtco=M0(=if1`kI|fkO3EF~;T?J`ZjAJt7>_HVaDpg4 z4-#}}TC#+_~5R8L0Q@DS%&6r4Xl3;ELKsLL>k_NHd{8KM4?L03+UN#6Ez5 zwQ~?$K@T3DHBKI2uRxOsAhy%h>o^So#P@<~?Fpv3$hd05g$L4yC-vn8xt@^H zX5tZg31G&+^Qz)91;}9&h0+ClKo3TljG&<*z+{3DJ#j>#ki&?u8v}MtcOatdiD6P8 zlMB>|Z;81=9~C&D+V+Wv1uDRC812BOOnPbeargJA92Fo~lY*%T3rav#0!f0f1aP^8 zKqo5%12t>u{C@$e6;L@OP;Rj2LCF1Z#f|+#m?q(jB|>B%%j>=|NkyMx@8hGwF6QkQ zkborgvVzEp7e83+`~ND23P2k3V}>7Pk^p-GMNJ4KBFYpT1J|RCOBA4hr5>kNK!CCL zO;m`3E$<5%qT(Wgf5I+8VXIkkCL$VU_GlgE6fzd7&+CR>4T2Zt>Vu!fL3gBs^8vj- z0Hla4*)<{qm{Gx%`UIp<%;PW-jrVYob;DP{5TJQs!QL#4kfb0;uxrEh2L$u-HsjO{ z`OF~3kt32sXoxC$n+KK&K=BG#!3q>GpcLaEo1o}1m;H=J$=uZ~8095+R2X`HOyJRMJ=2 zVR2032uXgOk#W@aH5htD08tQQd}|2Qh?|&wfF1D~USpg9CNjbUqfnSXnZFu1ECed) z0;Cxu<8q!$A%URKf)o+-&91bHMa&v_E=qe&Wk+CX=$~Lnq$(7j!L>pHff(8>Y?Hn% z2xL%4_)>x7uW1D)#XP{@3ltJV(&TvsA>u>;LsINm2?UUdDE*g-9^e%~9$;0V8&~x* zAWT9VsqFkB1S%pAu{(f`uvA$@w4hZHil27gk!jvt-N7b*#1MZd+48Iy8-R0n@~J!C zFA5Po>c+Z5^yx@zA_5|wnt#`32yEabRpx*N8h$?gZJryweNidUOHlEfGjp;>?ZlA_ z*&E_4&4;rPg-wto*fFD4C&m_7e-TG?pow^eZI5S&$^OhLkfoYJn1%xK3{=SRku~%= zcpv8B$yT6tt6IB^#8-1BwNmpW==ytU=NLC~)kf}hKpu1ppoN}YU#v zg9K3oNChof0ps>n3%sa!&ca&difzDW&=S7-B0TglJ0}LBa2o{4D%Fz$sP&+X2syhV16)mea?AYo#fF%nzF9cJ5a1AiE!4E6i3c4igcip|IfdClw$MM{(6LZL%jsqxsar9%5@r-J6h z2nh&oLfoi;DK6F=X-K%(EZ%M7yS=sHwqdfJxa`1yUFvzUx8hnwe0CkHg3VekqgPKu zt3QaD^ecls_ErQ=@)Hf)@3sd9z~K(uhYr9+#w#Jj-o<-v{H(s~=lW6W=lNL9Okrq^ z&YyhZK3$xGK#ph$phv5(r0vHGyMQuk7WlpX!_~yiKE{)=yAIp}$CR}nQqdZ)3j8<6 zfU=edF6+Sjgy2bGC_->wZBUl8*6rK?;HaslxJLR(8ACWzIHG6O;ob)u($e!CZ-BIq z2I!D%kiJ7f1)RDiR!WQt;80ac=AZ^E)6u-!K}x*I>~&unDT(8Lc9T7A@&P0aw3_^U ziQb(m3~gE`)N*FCSwe{YoEvyr+qaA%Ap7x39$PKjy|u6wyh7{VDSne__ds9W;+8!r zC`-e`5ZIH#-XqCn#HOIY0P1mfbsB`ZKY8$YpOOV07n7wwN2ZIE2jz=ZLvHiz>^SmL z?>9xiPFugkaRgcA>F~qa!KDQ9JX49dse_lrY&-e+SpV+wx^6a7GZ&*g6(4?tln*s*t?wd!7+_Fyql>O6^z)0=Yt1TO z8v%JaJYApO5Hn4SVDsXb#USG%ZtQ zMTNw{5@`PvYfF!3NmT;jRgmUuN=JI6;Kt4k0BgGyp9zYrFZ#iJX!-lS}rA)n?`-Os)&vLfIjwX`-BzyC|O6 za^^@Prt%1uL*g63xt$TzW2F+dnKIHLbS!l6x)jz`^U_He(q7SY$+^Rb)^phm>#EV= ztxF0g=z`2?+xVgwy(_rHlJYNHuBpf+ZZ5~R0tue=;1nLvL5Q+M@pCSG7}SW-xZQKE zUlYkI$;5m4YHbO~9SKCaEru1F+%Wvf7BeTKal{y5E$dPjVR7)JE$NCII^=bNA*D@L zm&jTu5nLyoKrVa~)C>T4a&wb%>*f!$uxK082s=>;=!E(C>o$CKk--Od&mN+rbyHN@ zsWl$R-3R>us#V8Li{2p15cHF%mB!fi=aI&2axi%lpry<&_z2Wqt}1Tm<0{5;Jd5>EeZ zgP26iA22F-W*@w%BNIG0V8x*`&L*hY&ecRo$s{<~;&$DbOI~1ay5Lsj!w&;LT&yqlyCDn*Wsjs{pSjKdsZz_Yw<+O=3v6>7gQE4 zNb?z3T@N~f9iZqa!j@~nYCNE1=7bdMRjCxDOo9vTo2{06nhX|2qMq5+C<#Vd=*^Fd z@$2D39WL$LvGGfGNLUM}c$%}=N2L>|jNyJg;R9SF9 z51$W_fjpRdewuFR$i(kXnR`yJ;75rDQCIR3VSNoUQBJfaGhyYvh_oQ9+?x@jBL|r> zXZF^mhu=k{G>6MRq~bgRwWFIApF88>Son;4o+%t=%w;!Hn-m>2@wu}LF6ygAyA174 z!c#{pJRFyG?y#OkTqpP)|5b_bviw{d`*S zq8XF5#QFHbu>HH>XQ37rI7zxwjs3EP`!JFM#(p#}1ntx?tPs<4UNe@owOO{)6$$yc z)$^HBj-fSExHQ~6jD%KprpJX-NG!wON3?ro$?9ZlvTkkzVYXd`+WIzWnj&OH7uVzI zxFyAcD@XFY$?XM3VFFn%SLFD(_`Xlz{Y_FLce$@7_*#z{kaNI3Au*vvrE{qo+=>*| zKXmJxkcW^JzsQ6yu71j^Ez>adFDdU5ui%%UXumH()$+&{`s7ibPHwn0L5HuODm!u* zRE43xn3!#Zv6Y|&N_kQgY#@FosYIFK)!6haB-ZA-UBTcmt8z6d$89u`IkzC%C}krR zHq#B=6{~WW0n^gwvD~O_Vk|0~X{4;MObIeCYiiPU23wAreh?i7tL1qm4##ppvfWr= zE4-b&E~BhiADXj|ql_8YIBrsCG`z>c=<(}V5mrE{+BwZR}37^?Oc5&2xC`Nc7 zK6{ZCAyIja3uzOylB$-6-=u0TP(&b-?RuSUqJcubb!j$_bk@a}Y3|<|E}m?<|UWlEI2Q z5y5ndhF6g}!$abg`&7*r_?gl~@aB`+d9*`AvU+O?@NJG};(Tip7iVn}=034y5%)Em zcu5~n=5X!B#b|pD#b{anDdAe;l<;=m9Glf>2S>Ry(zWFlM$|(LvaA_i#x{E! zG0`+nJZaR^;%mlSbCNn6r4!|ur;TW5M_CVv87+Wg;AU2};tro5zJA1H_2F!s9Y?q? zbgVwfwT)aB<_t6DA!KLxl0pn=%pdK+nteO|2AHiduX$MZ8jD#Xz3diZ%nA}q0xY4AeW@xjtg`%tQhA!JNa4X%=hl}_QMO!&2!yHMQi;ymCZM*x)2EB!7PidUYo66t2 zWEZljwhnt^r_*@~9fRHX$Y*Qhf)rj}P3``+eb9Yt4Hil95abKz*L1Pu{OUc-~ zIm=Nfb(9XiZ*!KZsEWkf96s26(iwXvffnwq%tDt&NqcfM`kUg!{~vh@hM7eLYg?Ud zHB3MA&BCcfBJuENu=tDQ**ZRC2ICEMBJS&sIlScG+ZMu?@nGQLKEO>IdEK{t)$aVB(ki77d4^ZG-1 zBk8^kUcPo#YT-qKHOHRv`KtV`gd&}0d*qT!0{QyF+)X9fzR?tWbQA8u*r_HNx9N$7 z@YH6ht4z@DiKD!nYcwGH{H{-$Vo#!Vt|2Bs+92+jFZj#0j9#JZjB`c_T?e)ThQ*u2 zufAz;N{(O=AVmjtos0*W=16HP7W^gj<}p5gv0kP{C#a~BunPBh%|Vokw~iB0`z$KseAa-HVVs~V-otAxKRWTj=IcfP10EV zFZ~6>>$V`9$)B@2_Y0)bHj_LB#J2Lf!pn{g+JX+kTS^MK>u5+swCyL%lcogKVn*Na zb}f!irbz4SjF!!DpU8f)&R43vM{%`v|$V3%xP zw#A8AK6f{y(_O+KD~DRZW|YL=?T36Xvc5sb%`k$yW4pq{kwyId=7O{PX~p9C{xN9a ztRX3UdURnXG;em6eNnhrKrp4*pW#j}ZTyRLQcHdD;qbZD!$#|52nFszj1+$z$4+@J zR!{#9ca~vC191~K(^?7|P|mZXiHI*#RtaXdtU30lz%|;@nZ5$R&)*$#gD@QXvLO{Y zM74HcTTW{S>!X1vtIuMC%MDx_t=BeXqZrud>!X*D$>`Z5W|uqQ;jZ(unYNC=a98R2 zd!DE4*>hQn2=?EBPE7r#emaz%*!CQltDDjjPe`)8A zJ5Bq=>>L^qm4a&eN{8NjJ%aWcbt4DsYY<))0-i<55*5Z-Gaiieh9e=rLWAZ zxZ=co-@m&VZjzyYZCW3J?T_}r<%T=rG7ins5PDUcaf&5UwB??t0TP7a-rH~$%#yzD zFQ9U>KFUZyJ6OSJQ&|EB;5kmGj%kDs>-r)@Xzr^)E5jy4Qbj}lh+3+Nr^ zcAbDK%;+HJkG;X1Gt2QdC17v5{eW6S@Ng8~aGBA!5<=U~g!~QMgLeoBC$dEK0^_P* z$V5GZse+Xi0_`A3q-52w6iOonddQ`C%+DjXhSu1uWPUfjvXa9v`sF&o1t8e_6VgT9 zCeKSit$iX(iNyQVu4pS}S_-sm6cI7mJC6wlRkw(Ui^-2wH@|bs^ea`V?v~$Ddc@SoYgACc9JMme4F!7Q<>gyU>>ED<4exZ~_u zA??e;qVPb?JL3E)2c&4X3-Yue|Jzvx^`}neE7IY5RBIlONgcW@R!0cW>{ECzwh(ZU z)!BN)7*_%{HmmIK&~;*?iSlh_L1ntD#}T~e9o=|Am`=pHe^Fe=&icO;xy|zM(i~T^hM@B6Nb&Jx7xd8GIlI+KMnmB%s(zcTEd|n~ z-t^?xEbbnP-%`MX8sQM};oMf;%)g^O96eZZZ?6`jJ)~uq zV;#O6CDokvUXC~W*DYL+D8FmFkjk>6+lHL#_{#f3C(S;?yRr1myDgYlI2lCFxvtGW zJ_(g~YEEDgBgS7&oR#x5UI+E2n05d-@kq|wl$o>t=cHitgn}Nfi&u{L%lxdnO$jvH ziEZ1demlz4kfGR$O{O2IcuT5zSStN(csPNUjJYV5uI43b`+6pJp#eg*9mN*=w`dl@ zfY6dVu2RL(Z%|2+Ehh{1yC{ayRf?#mpl%dAuI|h`j15QGP~7X3ESsaNux{dbat~{< z*SlOYD7Q@Q<#Wj|*N-Fwysr%r>EyCclH;XnL$Q^u*|<8!g>e(_AP}XM9L#hn*YOB1 z6bZoWcKgcjFjOXZk6u`1Hn39Az|Qn(v=3xkhCLUuvaEy{WNs&SCI1=dmF`o0!mZqi z7=-T6F={Sw3SGT11ows|WtyL!`)OzViuA)BycL$I;G1-ZhKMlk^XD}=jG$lF(p(!c ztS|1Uu!Hx3*2ks&ldVOS8ILWMYkLQ#W$Uw?%yg~oQ?+lWqT}=4apTj&!QtlgV1n=i z{mtR7u!|G)0RLY)123ExbjT++3Ik(%%m?eu^Xgsst*+QatA9qHFtu^fQ6tgFYCgN|;0J24sH>IuaWb3$a3ru20uAo5H07tV6 zl8`piWGS1)WHo3x%V&yME+I9oxiLMjA45pu zl;SkXisg>X{;CtkJ)w z-7t#H%Nr-HrV|4+%&p=>>*PcY7+fjCaN{iffgQ z+!IDuPU&RbP%xP+CwviC4b`X;4;8a;;%`RL1EjTZ;szyA#!5$nBw?Q#*3`4^730W}{YWpe3nzB$?rt?n}sz_!-qaCr2 zktmAPDQSE+s7{(9tx`#{+)5x^?@gkvA3PQL##|r5^6gNQS(~gf?Dr%FNxh@W?tYKc7>OkX0#wgkp$@TpTRlC(4&U#TuI>Y!n8cdRK zKszH%0@XnsB>Zki^VpX%%fK%&7yo9zrz0Y&rLfx3C~5&RCdWWqK5%IzUAs-~J(1}zs z0<#=pEFi}d!IEIE;P-ha5U3OlY1FVSR}i2;#=xPJ*jU5NNPct9;69t(OnskahC$F`O^?Bz zsQN8~OVKu30RHBwXD|saFl1p*_YaB)6)9T5Bu{7QY3=~|- zIBRvXu_A)Fv`Hi$okLC$#BsbRfq#e{#Kx2oTHqqH9tmq&Mun9+HbG!b3&kFi0K%a@B zNU1IwV#PPx9~$PkBu~cW?yn#hf{AWPehRTuR$cVE7{;5($v>dR!#BUWC3+SwLpUvBr<7F~{NrT-tjzQpuf;GhfH~Q2rN+_7HWACjFqh4i` zDCoybW~w;Ul9Nkf@yxjuca0TaX|XCZsI5Iy*;2j&6eY-;8lxU<@vS}c*qJxL@Di}O z5g@HWf@*9+Ip)Q=sSrh&q0X*|K)#O|+IMMUo%b70&_(!**chJon%4qqi#_oOPR+5d$orIqj}mbSryamN5o1yT*<-(Wg(EOep$YTq}U}k?6=X^8Xg;Y4G(Y6 z&Z;zEUv>pT1xIve7i}o36l4}Ez0xJJlnx(eBJy1DpfE=WSCixycxasM3K`k#_OJ@p z?UJ<+{;V^|XQ$X#G-8cRHz-)8_o@8{+L-?TLmMFRhf7HV5T_VA(-tV=yzfhpOM;n2 z9_BmRL@b%|^JMrjA&W!rAmW=SoK407*#@f` zm{LH3dlq~RGq{XTqle8@9e8w^@gvf6Aj)G6W!*qd$^-{Tg2C~Db1zFS6M?W4yIPLk zM1sL|hl?-k>i#=W7z!TR&W2Z5RF7sCN4X3Rh6MA?+q6{Rwf99$qL-Wdos-YZZ*mHx zzg163N2Y$Q8UCDC>A&EH*?+-}Uym69{}c8Ly!@uB0=fB{_l=Q#g#-#NBz>Si^{|9a~{{wF9{XcNy zbTMFYaFRlR`^6*mEcALyjwq!`3-t@#So;^<@cR$Cu{)+n#OlUJveny%)d|5eLNY(E zh@?X`H_*{?O`-<5hnM9wY|ZBO9WzIBET3cOZw^-LeJfpt{Z^pEm4fTA43d0mzh|Jj znTOjP{uW?b27c8K6yijE11#X`yV$e{64K9oULcg3kWXobAZls0@bcj4Z-0@%?R<%E z4{0e^*rhPUPy;3c83Y7wF{#yna5qoh$UMDLq!~jac$zo?=_u75SE>~v?JzOuuai`X zk^C~ZK2|VWSmEeabl5FWW;ZwxoooPGA>W>0np+kT0ex`jxxE?8RzQIGd;`}1Fb;+* zJteHtI}!-v2NH0Wi`_9l1uh+<{y0uEpicfI7kgf4YA+QXh*Yuy+yWAGLH$m6p_>t>lCWCNN^B?D^=`V5mqIub`rgbvTx zM_b7nfk=cb$ZQ5NFw+Z%wJU^5hd?7*Cv+(6fOwg2gXl0ODCDOG>_s17pS6PQ6zUe_ z>ukYcpbZJBM?Mg~6R^MfK*Jz`oE3|S1Ke87=3&rD>rwmBCIxXg?}^&FbwCDQx@|-J zOLgqf1KgHD2h;|s*vd`gTqUSf0uv%5z$HNEul`^dS=5Mihqm1M?FcjH!4KHO{|0pX zf#*QSrvBL_0qtZ$v}FNF?{|FV{4d&}i?-z&lDN21i$x9<1i~N4T5!17jtD-eE{2qf zDPAfgfw6!|oEeu-MVSVo8E6n=+$W#c?dA+5hzk2ldadTN@fIqbde!eJg0l~Ci8{hT z)QwtqxjVG#mdKQYAC9>%P$od8^BDEWz6UWmYa>y~MA!&w3@s`}@-0qPZ++Ji2?S9w z14NMujv;7h59(S(4=9I#)HUSU?B~%Li_U;C#PfbtF5j6bwLnF>a)k+wTHc>Yah;~* z^J47R&Xr3EO$ef`XWH@ip!Ap*>-Kj5n%$pke5W(2?qhC-oO|5PEn zb%AZoE5I=-`4#xp$25b&h`NhA09ui)5j7?9;KCq1v4{r+5(H>LgF(YlEWlW?(kvA^ z6k~~qEy$6^W}vm@mo^0#gs+7HLzkfW4bByjh(?B{+@2T`qHZ7#TSF-~ z|EwU!Dh>)8&E@11n6VBDh$6-E4Y9~vGb#a9lM1f^0RS#wBg!DbOh5R;pOYlFNwOTw zaPRNMqW)kiS9T)61LRoRK}V=-fFKYCIreT(?0vxbJIQpn=8+JKJ*g{gnJeBIGd(C| zPmM6+y)G@d!{@KGAIN7-c?W>JFk4!Lij#pU;;~aMM$SP5yvqg_f3G{~qWQ%NqWgxzBu0n|;L1kH-yqKYf-8kUQ^sA*l% zS5_||y^{1vEhxklv5#ECi9P{~*!;i)@soq$Q1Vlv3 zX0w48&yzE9N4}|%*I2P<9lIbq>ocE-!eTR@OTg3V&kwvKUw)6LcUE-^2W%}57j`7Q zm7oUnd&!_!f~6A+(pvh(zzeS05tO0-7P;ixwk8AlbZWrvh$BdPKK4=(-Z7;ZuZa|p z_y%qv?mqi|+zJBh|dVJ8ju}z&*`5pBHa7=V1-lr0& zpCeWFbn@oPW11c{h(hS)UeR#p;I_&3n%sE#b!Chb0!0)5CwN9PTTKhqm5E;HIA6 z4}8x*`JcFRc#NIc`E+tXqR(?4K2?|cyPG~uX<&n$o){pGda|##+Yfo4CTAF82_X9v znKSV_z#ng@lr$33CDdu#^eCsYzjb|E_p~JW4Hkd0OH|8|>J1ri<5P>CvZcfsBNxzP zs4aLUeM?!2G{fXMzm~fJu~I!9PxpT@KHjRb*a{zm!u%l;>&d3s^m)mKe+TW zt`_W$o;D~HRlvO@W?cWb{>8SMK91$RO-Ww_d`8~xYr`9goh;jGk zWHx416c$ZvtpS3exu(ke%@1IQ+SrjsP1g5Afthc{xa5_-mZOASKI6|jYlq+~6pA$X;5>*HWbb6Y^O)7B8UswB4k`-mYjOTV+X z)M?+g&kwk$6G~1L->8P0F3t&-MAHQFa0u~%08SqPNCb>q#m~-ilY_Cn5kfw%cs-ZnPYFHkCPj{~A?CZd6oA1NzccHd|;vdQ^tqtuh z4l~wwE*DjMP04|{vTf|^6li}#wnaN^so=-R_?rnU+(8pk&996!NLVJCqTnEy>pa+= zD*NX&DyI$d6KA1y4QoYEisKLIrUw*oF!VpuRgT{06J0MqG=(%Sw*+UxX}cWt7CKx@ zYIdM_GW##@wKu=Sm!3A;U4~pp#tC(4>*|MDdy%-~CsI5(xDS$JtK){4nN>g?acA7j zWn5QYOMdIU%S3z2Up97+jaJE6n9-Xa{cI5}IAMd_-n(qr9HQnBLzt&Hb)mhC=vZk1bM&n$61m%`~vm>I^oto)W6d&gRzOy2}m3qCxF@R!K<* zwOA%>!XC@tRuGc7TM7}fJ=x#wys(orf@LBD##V;o^+hR^qu;~DfvKpiN%IC2Zj@2k zj2BRd5i$K8C1}*(Na_2cjV#=8?zF(79sHd>=9;0B90(mt!IJ9ic9+B;9tkB5$(1sP zlJTliFEv%2ob{%p&b>Lt^C2P!;9N+_Y#!C{?pVg8^UdPevd@lF&SvUdud4jR&%cKJ z)TQ(gNG1cTYH^zxH4y1)4)|LGk{aZNa}k|eHsqWjHfAAEw%=dhECoHQ(q8>uJRE_? z9F}uDvGsqik$fP;*sGT*ttUU;tSINb)^exqeMQ-`k7CIibtjIQQj~OT+IwZOsmeLj zz3ed;3Gi3iBaJg{7x!@;J~K4NW`^FTo=4<|Uu)cpuUQ;=GzQ$F#Cc+47g( z+FL25n-JF$z9tpVuX=-6DOq6S!HVaqHMpWjb%KO1fbxcRNb z?KClEI_1T>sHh!`9c&8v#{+h2EAc+8UfrZl{X#1pA9#q@v9mHcUyXv5*+C!o>`}iR zeP+D9*_ZN1Oj0z&9?{_Qe_R}7em+f5M;z#J@_|{1gCVPE-QD#Z780HhCEn&903}PV zJxiH0BV>a9kmUSTyt6gFH)0x}R*LWZZ1K7a+MLVVm5%pCf+W<1Zks)9WXDMv=$6`vM zuo{{mecYyxw3@r@()EObXD@CF8V^3nPa71^&_xYs zZF2h$U?o5A9u}#LTM#d9cQ)vwUvaQ&oAy2XGSBJnYGofZR*+uBT(-=QXuRIHPV87S ze2zsb(5l{hPdaC3s=cyX+*{mSbPg~faHLxa)tBk7dl9*w&$2SOF#bg=*eLl-7K2w ztFOl0e>;TYY9fqNX+AhU`#l|dJ)M&SxTPWeCi%@WeYgOvTCTMnP3?YkY>(W0e<&!!S1?`81rO({m zTJg}Iy%91;n9BoR-i8lIbJn!gsn;a*Bm^OK2TJ8mBh%g>=j$NaN-rOC*MEe7%y8!_)qvMY$632GjxehRld`D#gF{<+ z^t|*eJ1roDbTG!1r?T(2Cr@tdSge^N@NQM=?Ly}!Ec)Bx zM^pvZlV_@oaIMaRYzFK^=Zp*4C4X8>mv2nv+U||n&12?D=;N+68=R(iTDuJ_Z$!_E z$=)vQy9UV}DLWlrRIF{>?QX0oQtD?1okPhTHTh8EpJ62WpsIJU{n7>5`U`=-S?Dbx z8wdnKzbaX2o4rtb56hkLv+QKUEpnZB{#;rW<<8o#OkLwQ&>AAZr4yuKW*PHymPt>EVRGLfG_9^^tU+mL;* z0lF1`d*e7V{M^?M4DokaX-m`o^!3%6jQ;7u4fB`Gir64TvDc+I z>n-5ZAV@%<A+a_Ojj+dWb{*C)1fXjPPbs(6!cM6R1V_xd}^OA*eBdrU51VcPfYkD2Z_M+OC5 z%NQi@D1H}28nuF#bH%c34b1i>{SOa7*nT+ zL^j#{bc?oo`9Iz6u9l%T9WQemZu>%a$#Q+DVvk(N^@#9BEDc!PW?twnZud4Y#tc*t zGS)E(YmCmWmC6Z(yFu3W{wWdPFOOk_Z(0KR2p)NlL_)&C>SRokVI~Mt3gY%a4*JTl z8vxsOh(#OCcrEty%euvT`r$7hSf-7HU$yFMX4iZFP0492nUjiC<5q@$ab>Jq`NxYmq(^oWs}~Cr_|sIReK|e%8wgx6V^<<$i#AQ8y_9Xy?%-vzCL+$#hky)oHAs- z^Z(Fte*Y>#_QCArZ4oRK>}!NBztzWPeM3;F0(*LSB|I0sB6IsXQhIoN7e~qjKJB?H z3B^pj^$^NC@|>n(uf#mtt%Z{HTta>NKi6hmH&%*a_Sq?!bCsS3=;(*&WoZ?$KfM$B z5(>Bzh7okLMDc59r+my(u8@d3iZ{ zK)mP>L6{|rgw$99gA9&l*fU&~Bmyf9c|bFbj%4$l(|zgB@utlVsj9AxSG)WZjnRFwU&pB;}iDMU2rUI+0 zl`qTH)+j=m;z;6!Mp;*TR5Gqr66MfwkeTr@XZCz_5fB!4*CTT=5T(eO4;^Ab#e!j< z|6c&-2^jVn!DrC@A;@f#ZS@wEVy(>O%SR7Ie!SS7%i)#Wm)7QGzt-ORBkI}+YsrUY zH6Nyn%H~1!VH$j}_Qw)lK;ez%T^TQs>OVrwKb51w92^EL2P%O9o0d|@C=R&-uOL#q zNw=g5x)}R$Jp&81F(Hp4qgl*=eIk;}9ND?e7m#N?wTv}aCWJpsX(2-%#v6^QHTUiF zH*W`_x|;@8wyZ)maN5aq6vt8-CE)|pk{M<;()-#AF0C+O9P>zl%fw@DoO~^iXmnmAs$D8hV0XFQa=Ad?yTdG~kdi}b+sAY=vcNbtylMxPQizKT+ zj{xqvyZB)O&RbO#Rjqc_Y+BWhB9NT@2o@Jr4*P65Z z`8d&@3TSh<)tU?_-%N&TxB>o_YFpE_Rot4ewdz0ie`tn)JM=}Xl-_RWcj)*pQ(c<9 z9(37Y4?iRqOT#*amvuMM558Sg!OLlq{|RJgoXpCVZ;nM_EgLRAj6UqVapQ-y8W?mG zwG3e30llzt_;WpW8DYET0Tu~kl1bwTB2A;=cv#h0+>Q$zxF%I?@KIns48hVv9yIt( zNlrz};Z$8FLUC!-)B=-+o3J*6{#48HK&^mVQZ=4-ZR@twSUd2m zF4vzQY|8-vo{pLpuVOvpkd? zFE+-ATb;SfK6I)5L#I zUk(2J0}P&icr$qWx7W`xjJlMHWw>=M!MESNeS-2o4>tz`~215 z`=_t!3X1o~$e%q*8o{Ig@%ri8Kf}wvH#5;krzh{;VF0Fl_YP!nV&_Q?iumr` z=_~lb&ZhADXbOG(yLVXv1E_cJicE0EElY|dl!B>u@6xN)MRjrK(-H=OxR7g^W!cbo z?_SrF>F?e(s}|qA`>5r+cW>%R{PO6&&WYct{?CT9K{2=XXS-nnuF-!UO6ku3$}ax< zd7ei9uNZf=tmY_#?tfSe>!pn$mwnS0m&f-UW#`Ep+?L}BhMz{@yUd#fzv5~I>QM%| znl~#WML)j!c{&2WIUh}%r8G>fAHWN5C)IQ@uXi9FvHsfqz(?6dHfrh#QAMxvfKiws zIrTfW_M(73GF=onwpgx@1^U$0Yp83rLiI@Ym0ZhuHmnx&QC;k1jaQ_cFUHBFm=EnF zA2pP&oo#e9%qPim{qxn2KMukDn~vT$RImq8r{uG?TzKkxO}ae0xugDbF=%C%U2us!Z^339F@b5OD^j5UoS3s4=ukc3Y_N!dX9}y$7?x2+(7_EQ$5y>zyI3J zrQYl3uGSt;KVXbNYX#)o*A0*Ml$lyv0z7{Rp%UQ3f#7QE!h( zaHeE|MzD@Z{QZ*J;B-0~*{mAAx4|nAmLOSF(`NZaP|TCD{V<*X82sV%)i?k6!)oOS z^rh#E6b8KG$Ctg?<7_O)qv@>tre8;pPS0)jWAOLWSHY{7uK~FrpJwG%5%}%uO#gZj z$|bbl)RDU}2IF#DCBsqK|6@Hc|6TBBu)M(x>>UQ+>w1D8-UQ2awheDd;H?MK<9Gt%X>s5}Yubyccyhz5w(M7=3asrzLZz?;RAzG{#JY4`}pez|A z#sZpw#Z?*203-MVE+&5bzWH{N$b{!S?FDb_EC_UfE>{o!d&1ZW7vBH}-kb*SLG@%9 z00rr=3jR&lbbn?uPs0 zWyZ%13;+4gf2K)!{^%iIfUtZO9RD?_-9Su`jf43(sE1v_ak2bmwt!SPmJ5;z&~?5z zwv)4eUkOKWe4L?ROc&J?#(8EFUKkacI5&pFXPu}#&GJHWtxU#JqqBHb6*RV4f*t(7 zM-Tto@iD^v1b$tuLBBd?eQ>3l@cDc?UuCI(uPQr#a)od6=KR9|*GE0MGMUR4^~!-K zD{Hv?G0ErsrT5ma$+*1mVVM9O@MVH07j{+F5aX7pmNjdhEYF5&-lZ6`p5#x~O!M+h zZr=|x`vmOvsuj!N=95*lbBPBy8|yKxCn^jtp<2a>p0MVp_scgwErzzLc@0wc{j{>p z5{+RITuc|ihop&nE|v>6{*ImoXyBTSnA>bPvwLaw(jb6MSu(Wz27MN+a2aeX5vdC( z8!>Dbqz_<&K#GEuIMq=(7*8Y<)QmN!R)S4oi-*=SnAZ~5?jU1;#n|M?^UcT~qp{&Q9K>$N2dYsJ55W{7b`!*WFq z*2bbf4E7_){GfZj!1L++zsnf+_3M9`aO1z6%g+D%91odcqEiHP5-RyfCs}rytn#PZ zoa`q_u3d-_b_V1|&mLjQ3_6r6HQ%ES(1$3Bw(b9M=6^U38I1qbgU|4MD*Zo@=f?cM x8s1;~gxBu>a_m1B{NVS09mw Date: Sat, 5 Oct 2024 01:25:14 +0530 Subject: [PATCH 021/658] Revert "Results from R50 GH action on windows-latest" This reverts commit b86b2dee6fad232bef62993608c4b8df2fd2b7f8. --- cm-run-script-versions.json | 602 ----------------------------- hardware/gh_windows.json | 26 -- mlperf-inference-submission.tar.gz | Bin 19309 -> 0 bytes summary.csv | 2 - summary.json | 32 -- tmp-run.bat | 36 -- tmp-state.json | 3 - version_info.json | 602 ----------------------------- 8 files changed, 1303 deletions(-) delete mode 100644 cm-run-script-versions.json delete mode 100644 hardware/gh_windows.json delete mode 100644 mlperf-inference-submission.tar.gz delete mode 100644 summary.csv delete mode 100644 summary.json delete mode 100644 tmp-run.bat delete mode 100644 tmp-state.json delete mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json deleted file mode 100644 index b42a41b1f2..0000000000 --- a/cm-run-script-versions.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_fp32,_onnx": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "fp32,onnx", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( resnet50,_onnxruntime,_offline,_cpu,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_resnet50,_onnxruntime,_offline,_cpu,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "resnet50,onnxruntime,offline,cpu,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json deleted file mode 100644 index b7cf960dbb..0000000000 --- a/hardware/gh_windows.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "accelerator_frequency": "", - "accelerator_host_interconnect": "N/A", - "accelerator_interconnect": "N/A", - "accelerator_interconnect_topology": "", - "accelerator_memory_capacity": "N/A", - "accelerator_memory_configuration": "N/A", - "accelerator_model_name": "N/A", - "accelerator_on-chip_memories": "", - "accelerators_per_node": "0", - "cooling": "air", - "division": "open", - "host_memory_configuration": "undefined", - "host_networking": "Gig Ethernet", - "host_network_card_count": "1", - "host_networking_topology": "N/A", - "host_processor_interconnect": "", - "host_storage_type": "SSD", - "hw_notes": "", - "number_of_nodes": "1", - "status": "available", - "submitter": "cTuning", - "sw_notes": "Automated by MLCommons CM", - "system_type": "edge", - "system_type_detail": "edge server" -} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz deleted file mode 100644 index 3599b5aebe18c3e6a1abeda5222157cce83c5118..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19309 zcmZ6yb95$6^Z0$mb~d)Tv9H*6Hp#}eZQHgtwrx8bJJ~oJ+j(<8&+k6Z`M!TmpPH$e z?mpF3HQk>gjD!Zz2?I<4VCUZMj<{oWB_G*I(c~4AKHjHFIt%=;|t<4m6lNo@G& z89y-^;RYa8-D7;fonPrPmJxur9FBdTvCaDI0eWD8i#z_5v-;IvnAkKEOYrfM=Xvqm z5c(0QM<=$3@Yef&&L7y5{r-tJL-5AHFg_l`kNH{nPk*kjp3c0k^f|{$2*>x@w8O>N zf8xngCdS?l72(93@9_R@Xq}05{7I&V4Y}&11pgBlX!+t>EmZOeb(tFswe@MMSP7*& zH-GmjnX($6O}XgAM|Jp60-@mLKi#Vs+sU7svZ%*dc82%FTc;6x((U`!NPhBB2D#|O zjxRgol^lES#@&NuG)6ZTvKZVmtJ<$ghIxg%(@pMtz9yGYcVCh&7t?UNTS@J1HsJz0 z0aEMD=znwN-6|6&TS97=ab04>sr!+Jw2d@Du2C8?ksHYscdgI;S@-cSwHffMoj!5V z%FN%83B(ys&T1=VTC|Xj$<0@tH;H?O7;Yg(QbRhyIPrrO2$%EY$OVzVrDRR-@3&6p z@>Q>l)q3!t1~ML7JMXcZybl*t@U9(l+-l#wzq*45nDXiKZTGByXy!sjTMQhMmy%B& zR5C%Ex<+N=SBdJt!=F zAtWVBhq|U;*E0k5{OQNdHcTb(DPe|km%rJIn~+igGNKpZsfKj*Kd*?ojC1Y90sC4S z?t=r{@Si`jF9I2^&w2-n$-%I?k<>!Gb60!t4?j1aa-*EEHn*R)_c~nLZDby{33=Z} z3PX|Fk2qe=NRJG!NY1c@|K z(R4%N_m((Jq`2sOsl$pn{hb2-*Z?0Xq)#AId?h6L#EpA+6+fi4RJ4al9QAy^EROV! zTok!WMF9hfb$J#-SsD6_Nlwa|<_2u$-d-44| zdFr^-eY10NC}RHhi+eeMMG?9$f|XU#NEsK_#wv$`Mm5~zS@H7tS~rK^`{C5x7d|<# zNis=lUi~!6R6nCwYGU#bNS&5EjNy*;SA`7n_SbBmP~ybMDp5K@oj0YpLax3$>d#`0 z;bhKJn^DDitWE3OU(G4&tf7Q(J?0!xKpo6ysqM(XY%& zUn_IaI|_)=%YwWp3`myT6W|IHWyE)I56n#r;zQ~Lg=&#AC&e*250)zjt!_-+lW5A& z^#5#7XpluvsnJY`PA1p87bQnHO#Qqu#7$!sO;OJ-6%t$8d0E$zE{+WPrkJrRB*Gp6 z8@bvSuCdZ554S!^_2c%$%;*RKJIWH5HYZe{n4U;;AHG*XbbNqQGH5DUQLWfZnv1PS zYu_Gm0h5MZZAB&wD1~LH4{Cs-F^nE-g`v+R4Ptq^tm*$z%NmZVr%p@;D6z&Nr;$v8 zw>FR+Bvq|69fYMv&N9WRQwtyaGbj-_S`vIvmlUk8RVTAXp<5L&0XY&+&#Z?X8Kv5W zED1gvtQaps9m}Q%z1<|c#9|Qao-<4|4sA1BB<1g|VIn)6@2-JDvNr&f{tIi0+3j_Y+xte!x_Ll_lu0!-Is zmBExmbSFE~Py0pm))oV@gf8*}T@nN}+h0obv^+4fYDFdSXlhx4AyH&SaE{zb31z!iAv7uS{Ua&n znGD-hGt39bFg23=e34|L(gCIF^f43RCRtm#NHjmy5MpJ%%VN>ao({t5IVRyIj+8a! zoij-T{?biY%QBka2%K~JZnJU({)Vt4g%`!biL+p=W`>x6_DU^FnVqw%NGEE`G)7D` zY7AOGErk~3OonU|N~9BnV==;150;q#jRP@WRYFWquGNA8QPVQg8}UcTN3hPJo=xS9 zK|{o#OE%Y{ohZPGNtwjpQr`zk!p)LK0znaWVOukQQ9xE0_llS?vi(@CWyThZuMarN z7-F9T1rPr_NJ9k@b}$#uAp z+;PQB@V8tJhFHYPlQRn0-1Su_omDRd*$SgZt(g|cU~W#Bot6N7dI?X$-W48!A2rK# zDB;WviAH=`HdMn%vUHiHkgXrOc#u7rP5!V79*zp9+;%Xh#ci#q8H5ytkpp6(n|Rp= zG9$e{S$%1+K2PS&bJ?|EkYYjCQ~uaUls~l3+`tmA;+XVXY8m$S_udCZHs$2h^fUjy z<^&3fZp$fMP}9s!DVA+wY{==^5A=T{BUxaS!(fh>OSdXnz zuT#K9yP;yv7?NQkCUGCtyS`%o*lL@?1pJ^?U|!NnNB&^^6%6mbiBIrdq2Z&88#6I#O`T*s|2B7&8buVGa z=#A8f$Wh2kw-0qr8qpYxG&!ZVT7vesu9{0=cYDTWE-1imXERb8)|Kg9DfWF4w0jy zA0um85@up$GrgP3WS3&`(~?1hqtm;e-fqir>0?GP7n#h{Ir_1V0f`MIWzKU*AUn0z zmYm!}S|Ash%ECFBv3IXqu3lU4x869a)B0Bv^e@K8Ymk1!55NZE}GQSy_B! zV8SLFutP=`A64Lw4d5o4RnxQx5hk}wT<`v{T7Rki@)hCkr@CP+q;_pa&g!7VZH>!$ zcRa@$QnwbSy3ND7uW7)&I-Mtco=M0(=if1`kI|fkO3EF~;T?J`ZjAJt7>_HVaDpg4 z4-#}}TC#+_~5R8L0Q@DS%&6r4Xl3;ELKsLL>k_NHd{8KM4?L03+UN#6Ez5 zwQ~?$K@T3DHBKI2uRxOsAhy%h>o^So#P@<~?Fpv3$hd05g$L4yC-vn8xt@^H zX5tZg31G&+^Qz)91;}9&h0+ClKo3TljG&<*z+{3DJ#j>#ki&?u8v}MtcOatdiD6P8 zlMB>|Z;81=9~C&D+V+Wv1uDRC812BOOnPbeargJA92Fo~lY*%T3rav#0!f0f1aP^8 zKqo5%12t>u{C@$e6;L@OP;Rj2LCF1Z#f|+#m?q(jB|>B%%j>=|NkyMx@8hGwF6QkQ zkborgvVzEp7e83+`~ND23P2k3V}>7Pk^p-GMNJ4KBFYpT1J|RCOBA4hr5>kNK!CCL zO;m`3E$<5%qT(Wgf5I+8VXIkkCL$VU_GlgE6fzd7&+CR>4T2Zt>Vu!fL3gBs^8vj- z0Hla4*)<{qm{Gx%`UIp<%;PW-jrVYob;DP{5TJQs!QL#4kfb0;uxrEh2L$u-HsjO{ z`OF~3kt32sXoxC$n+KK&K=BG#!3q>GpcLaEo1o}1m;H=J$=uZ~8095+R2X`HOyJRMJ=2 zVR2032uXgOk#W@aH5htD08tQQd}|2Qh?|&wfF1D~USpg9CNjbUqfnSXnZFu1ECed) z0;Cxu<8q!$A%URKf)o+-&91bHMa&v_E=qe&Wk+CX=$~Lnq$(7j!L>pHff(8>Y?Hn% z2xL%4_)>x7uW1D)#XP{@3ltJV(&TvsA>u>;LsINm2?UUdDE*g-9^e%~9$;0V8&~x* zAWT9VsqFkB1S%pAu{(f`uvA$@w4hZHil27gk!jvt-N7b*#1MZd+48Iy8-R0n@~J!C zFA5Po>c+Z5^yx@zA_5|wnt#`32yEabRpx*N8h$?gZJryweNidUOHlEfGjp;>?ZlA_ z*&E_4&4;rPg-wto*fFD4C&m_7e-TG?pow^eZI5S&$^OhLkfoYJn1%xK3{=SRku~%= zcpv8B$yT6tt6IB^#8-1BwNmpW==ytU=NLC~)kf}hKpu1ppoN}YU#v zg9K3oNChof0ps>n3%sa!&ca&difzDW&=S7-B0TglJ0}LBa2o{4D%Fz$sP&+X2syhV16)mea?AYo#fF%nzF9cJ5a1AiE!4E6i3c4igcip|IfdClw$MM{(6LZL%jsqxsar9%5@r-J6h z2nh&oLfoi;DK6F=X-K%(EZ%M7yS=sHwqdfJxa`1yUFvzUx8hnwe0CkHg3VekqgPKu zt3QaD^ecls_ErQ=@)Hf)@3sd9z~K(uhYr9+#w#Jj-o<-v{H(s~=lW6W=lNL9Okrq^ z&YyhZK3$xGK#ph$phv5(r0vHGyMQuk7WlpX!_~yiKE{)=yAIp}$CR}nQqdZ)3j8<6 zfU=edF6+Sjgy2bGC_->wZBUl8*6rK?;HaslxJLR(8ACWzIHG6O;ob)u($e!CZ-BIq z2I!D%kiJ7f1)RDiR!WQt;80ac=AZ^E)6u-!K}x*I>~&unDT(8Lc9T7A@&P0aw3_^U ziQb(m3~gE`)N*FCSwe{YoEvyr+qaA%Ap7x39$PKjy|u6wyh7{VDSne__ds9W;+8!r zC`-e`5ZIH#-XqCn#HOIY0P1mfbsB`ZKY8$YpOOV07n7wwN2ZIE2jz=ZLvHiz>^SmL z?>9xiPFugkaRgcA>F~qa!KDQ9JX49dse_lrY&-e+SpV+wx^6a7GZ&*g6(4?tln*s*t?wd!7+_Fyql>O6^z)0=Yt1TO z8v%JaJYApO5Hn4SVDsXb#USG%ZtQ zMTNw{5@`PvYfF!3NmT;jRgmUuN=JI6;Kt4k0BgGyp9zYrFZ#iJX!-lS}rA)n?`-Os)&vLfIjwX`-BzyC|O6 za^^@Prt%1uL*g63xt$TzW2F+dnKIHLbS!l6x)jz`^U_He(q7SY$+^Rb)^phm>#EV= ztxF0g=z`2?+xVgwy(_rHlJYNHuBpf+ZZ5~R0tue=;1nLvL5Q+M@pCSG7}SW-xZQKE zUlYkI$;5m4YHbO~9SKCaEru1F+%Wvf7BeTKal{y5E$dPjVR7)JE$NCII^=bNA*D@L zm&jTu5nLyoKrVa~)C>T4a&wb%>*f!$uxK082s=>;=!E(C>o$CKk--Od&mN+rbyHN@ zsWl$R-3R>us#V8Li{2p15cHF%mB!fi=aI&2axi%lpry<&_z2Wqt}1Tm<0{5;Jd5>EeZ zgP26iA22F-W*@w%BNIG0V8x*`&L*hY&ecRo$s{<~;&$DbOI~1ay5Lsj!w&;LT&yqlyCDn*Wsjs{pSjKdsZz_Yw<+O=3v6>7gQE4 zNb?z3T@N~f9iZqa!j@~nYCNE1=7bdMRjCxDOo9vTo2{06nhX|2qMq5+C<#Vd=*^Fd z@$2D39WL$LvGGfGNLUM}c$%}=N2L>|jNyJg;R9SF9 z51$W_fjpRdewuFR$i(kXnR`yJ;75rDQCIR3VSNoUQBJfaGhyYvh_oQ9+?x@jBL|r> zXZF^mhu=k{G>6MRq~bgRwWFIApF88>Son;4o+%t=%w;!Hn-m>2@wu}LF6ygAyA174 z!c#{pJRFyG?y#OkTqpP)|5b_bviw{d`*S zq8XF5#QFHbu>HH>XQ37rI7zxwjs3EP`!JFM#(p#}1ntx?tPs<4UNe@owOO{)6$$yc z)$^HBj-fSExHQ~6jD%KprpJX-NG!wON3?ro$?9ZlvTkkzVYXd`+WIzWnj&OH7uVzI zxFyAcD@XFY$?XM3VFFn%SLFD(_`Xlz{Y_FLce$@7_*#z{kaNI3Au*vvrE{qo+=>*| zKXmJxkcW^JzsQ6yu71j^Ez>adFDdU5ui%%UXumH()$+&{`s7ibPHwn0L5HuODm!u* zRE43xn3!#Zv6Y|&N_kQgY#@FosYIFK)!6haB-ZA-UBTcmt8z6d$89u`IkzC%C}krR zHq#B=6{~WW0n^gwvD~O_Vk|0~X{4;MObIeCYiiPU23wAreh?i7tL1qm4##ppvfWr= zE4-b&E~BhiADXj|ql_8YIBrsCG`z>c=<(}V5mrE{+BwZR}37^?Oc5&2xC`Nc7 zK6{ZCAyIja3uzOylB$-6-=u0TP(&b-?RuSUqJcubb!j$_bk@a}Y3|<|E}m?<|UWlEI2Q z5y5ndhF6g}!$abg`&7*r_?gl~@aB`+d9*`AvU+O?@NJG};(Tip7iVn}=034y5%)Em zcu5~n=5X!B#b|pD#b{anDdAe;l<;=m9Glf>2S>Ry(zWFlM$|(LvaA_i#x{E! zG0`+nJZaR^;%mlSbCNn6r4!|ur;TW5M_CVv87+Wg;AU2};tro5zJA1H_2F!s9Y?q? zbgVwfwT)aB<_t6DA!KLxl0pn=%pdK+nteO|2AHiduX$MZ8jD#Xz3diZ%nA}q0xY4AeW@xjtg`%tQhA!JNa4X%=hl}_QMO!&2!yHMQi;ymCZM*x)2EB!7PidUYo66t2 zWEZljwhnt^r_*@~9fRHX$Y*Qhf)rj}P3``+eb9Yt4Hil95abKz*L1Pu{OUc-~ zIm=Nfb(9XiZ*!KZsEWkf96s26(iwXvffnwq%tDt&NqcfM`kUg!{~vh@hM7eLYg?Ud zHB3MA&BCcfBJuENu=tDQ**ZRC2ICEMBJS&sIlScG+ZMu?@nGQLKEO>IdEK{t)$aVB(ki77d4^ZG-1 zBk8^kUcPo#YT-qKHOHRv`KtV`gd&}0d*qT!0{QyF+)X9fzR?tWbQA8u*r_HNx9N$7 z@YH6ht4z@DiKD!nYcwGH{H{-$Vo#!Vt|2Bs+92+jFZj#0j9#JZjB`c_T?e)ThQ*u2 zufAz;N{(O=AVmjtos0*W=16HP7W^gj<}p5gv0kP{C#a~BunPBh%|Vokw~iB0`z$KseAa-HVVs~V-otAxKRWTj=IcfP10EV zFZ~6>>$V`9$)B@2_Y0)bHj_LB#J2Lf!pn{g+JX+kTS^MK>u5+swCyL%lcogKVn*Na zb}f!irbz4SjF!!DpU8f)&R43vM{%`v|$V3%xP zw#A8AK6f{y(_O+KD~DRZW|YL=?T36Xvc5sb%`k$yW4pq{kwyId=7O{PX~p9C{xN9a ztRX3UdURnXG;em6eNnhrKrp4*pW#j}ZTyRLQcHdD;qbZD!$#|52nFszj1+$z$4+@J zR!{#9ca~vC191~K(^?7|P|mZXiHI*#RtaXdtU30lz%|;@nZ5$R&)*$#gD@QXvLO{Y zM74HcTTW{S>!X1vtIuMC%MDx_t=BeXqZrud>!X*D$>`Z5W|uqQ;jZ(unYNC=a98R2 zd!DE4*>hQn2=?EBPE7r#emaz%*!CQltDDjjPe`)8A zJ5Bq=>>L^qm4a&eN{8NjJ%aWcbt4DsYY<))0-i<55*5Z-Gaiieh9e=rLWAZ zxZ=co-@m&VZjzyYZCW3J?T_}r<%T=rG7ins5PDUcaf&5UwB??t0TP7a-rH~$%#yzD zFQ9U>KFUZyJ6OSJQ&|EB;5kmGj%kDs>-r)@Xzr^)E5jy4Qbj}lh+3+Nr^ zcAbDK%;+HJkG;X1Gt2QdC17v5{eW6S@Ng8~aGBA!5<=U~g!~QMgLeoBC$dEK0^_P* z$V5GZse+Xi0_`A3q-52w6iOonddQ`C%+DjXhSu1uWPUfjvXa9v`sF&o1t8e_6VgT9 zCeKSit$iX(iNyQVu4pS}S_-sm6cI7mJC6wlRkw(Ui^-2wH@|bs^ea`V?v~$Ddc@SoYgACc9JMme4F!7Q<>gyU>>ED<4exZ~_u zA??e;qVPb?JL3E)2c&4X3-Yue|Jzvx^`}neE7IY5RBIlONgcW@R!0cW>{ECzwh(ZU z)!BN)7*_%{HmmIK&~;*?iSlh_L1ntD#}T~e9o=|Am`=pHe^Fe=&icO;xy|zM(i~T^hM@B6Nb&Jx7xd8GIlI+KMnmB%s(zcTEd|n~ z-t^?xEbbnP-%`MX8sQM};oMf;%)g^O96eZZZ?6`jJ)~uq zV;#O6CDokvUXC~W*DYL+D8FmFkjk>6+lHL#_{#f3C(S;?yRr1myDgYlI2lCFxvtGW zJ_(g~YEEDgBgS7&oR#x5UI+E2n05d-@kq|wl$o>t=cHitgn}Nfi&u{L%lxdnO$jvH ziEZ1demlz4kfGR$O{O2IcuT5zSStN(csPNUjJYV5uI43b`+6pJp#eg*9mN*=w`dl@ zfY6dVu2RL(Z%|2+Ehh{1yC{ayRf?#mpl%dAuI|h`j15QGP~7X3ESsaNux{dbat~{< z*SlOYD7Q@Q<#Wj|*N-Fwysr%r>EyCclH;XnL$Q^u*|<8!g>e(_AP}XM9L#hn*YOB1 z6bZoWcKgcjFjOXZk6u`1Hn39Az|Qn(v=3xkhCLUuvaEy{WNs&SCI1=dmF`o0!mZqi z7=-T6F={Sw3SGT11ows|WtyL!`)OzViuA)BycL$I;G1-ZhKMlk^XD}=jG$lF(p(!c ztS|1Uu!Hx3*2ks&ldVOS8ILWMYkLQ#W$Uw?%yg~oQ?+lWqT}=4apTj&!QtlgV1n=i z{mtR7u!|G)0RLY)123ExbjT++3Ik(%%m?eu^Xgsst*+QatA9qHFtu^fQ6tgFYCgN|;0J24sH>IuaWb3$a3ru20uAo5H07tV6 zl8`piWGS1)WHo3x%V&yME+I9oxiLMjA45pu zl;SkXisg>X{;CtkJ)w z-7t#H%Nr-HrV|4+%&p=>>*PcY7+fjCaN{iffgQ z+!IDuPU&RbP%xP+CwviC4b`X;4;8a;;%`RL1EjTZ;szyA#!5$nBw?Q#*3`4^730W}{YWpe3nzB$?rt?n}sz_!-qaCr2 zktmAPDQSE+s7{(9tx`#{+)5x^?@gkvA3PQL##|r5^6gNQS(~gf?Dr%FNxh@W?tYKc7>OkX0#wgkp$@TpTRlC(4&U#TuI>Y!n8cdRK zKszH%0@XnsB>Zki^VpX%%fK%&7yo9zrz0Y&rLfx3C~5&RCdWWqK5%IzUAs-~J(1}zs z0<#=pEFi}d!IEIE;P-ha5U3OlY1FVSR}i2;#=xPJ*jU5NNPct9;69t(OnskahC$F`O^?Bz zsQN8~OVKu30RHBwXD|saFl1p*_YaB)6)9T5Bu{7QY3=~|- zIBRvXu_A)Fv`Hi$okLC$#BsbRfq#e{#Kx2oTHqqH9tmq&Mun9+HbG!b3&kFi0K%a@B zNU1IwV#PPx9~$PkBu~cW?yn#hf{AWPehRTuR$cVE7{;5($v>dR!#BUWC3+SwLpUvBr<7F~{NrT-tjzQpuf;GhfH~Q2rN+_7HWACjFqh4i` zDCoybW~w;Ul9Nkf@yxjuca0TaX|XCZsI5Iy*;2j&6eY-;8lxU<@vS}c*qJxL@Di}O z5g@HWf@*9+Ip)Q=sSrh&q0X*|K)#O|+IMMUo%b70&_(!**chJon%4qqi#_oOPR+5d$orIqj}mbSryamN5o1yT*<-(Wg(EOep$YTq}U}k?6=X^8Xg;Y4G(Y6 z&Z;zEUv>pT1xIve7i}o36l4}Ez0xJJlnx(eBJy1DpfE=WSCixycxasM3K`k#_OJ@p z?UJ<+{;V^|XQ$X#G-8cRHz-)8_o@8{+L-?TLmMFRhf7HV5T_VA(-tV=yzfhpOM;n2 z9_BmRL@b%|^JMrjA&W!rAmW=SoK407*#@f` zm{LH3dlq~RGq{XTqle8@9e8w^@gvf6Aj)G6W!*qd$^-{Tg2C~Db1zFS6M?W4yIPLk zM1sL|hl?-k>i#=W7z!TR&W2Z5RF7sCN4X3Rh6MA?+q6{Rwf99$qL-Wdos-YZZ*mHx zzg163N2Y$Q8UCDC>A&EH*?+-}Uym69{}c8Ly!@uB0=fB{_l=Q#g#-#NBz>Si^{|9a~{{wF9{XcNy zbTMFYaFRlR`^6*mEcALyjwq!`3-t@#So;^<@cR$Cu{)+n#OlUJveny%)d|5eLNY(E zh@?X`H_*{?O`-<5hnM9wY|ZBO9WzIBET3cOZw^-LeJfpt{Z^pEm4fTA43d0mzh|Jj znTOjP{uW?b27c8K6yijE11#X`yV$e{64K9oULcg3kWXobAZls0@bcj4Z-0@%?R<%E z4{0e^*rhPUPy;3c83Y7wF{#yna5qoh$UMDLq!~jac$zo?=_u75SE>~v?JzOuuai`X zk^C~ZK2|VWSmEeabl5FWW;ZwxoooPGA>W>0np+kT0ex`jxxE?8RzQIGd;`}1Fb;+* zJteHtI}!-v2NH0Wi`_9l1uh+<{y0uEpicfI7kgf4YA+QXh*Yuy+yWAGLH$m6p_>t>lCWCNN^B?D^=`V5mqIub`rgbvTx zM_b7nfk=cb$ZQ5NFw+Z%wJU^5hd?7*Cv+(6fOwg2gXl0ODCDOG>_s17pS6PQ6zUe_ z>ukYcpbZJBM?Mg~6R^MfK*Jz`oE3|S1Ke87=3&rD>rwmBCIxXg?}^&FbwCDQx@|-J zOLgqf1KgHD2h;|s*vd`gTqUSf0uv%5z$HNEul`^dS=5Mihqm1M?FcjH!4KHO{|0pX zf#*QSrvBL_0qtZ$v}FNF?{|FV{4d&}i?-z&lDN21i$x9<1i~N4T5!17jtD-eE{2qf zDPAfgfw6!|oEeu-MVSVo8E6n=+$W#c?dA+5hzk2ldadTN@fIqbde!eJg0l~Ci8{hT z)QwtqxjVG#mdKQYAC9>%P$od8^BDEWz6UWmYa>y~MA!&w3@s`}@-0qPZ++Ji2?S9w z14NMujv;7h59(S(4=9I#)HUSU?B~%Li_U;C#PfbtF5j6bwLnF>a)k+wTHc>Yah;~* z^J47R&Xr3EO$ef`XWH@ip!Ap*>-Kj5n%$pke5W(2?qhC-oO|5PEn zb%AZoE5I=-`4#xp$25b&h`NhA09ui)5j7?9;KCq1v4{r+5(H>LgF(YlEWlW?(kvA^ z6k~~qEy$6^W}vm@mo^0#gs+7HLzkfW4bByjh(?B{+@2T`qHZ7#TSF-~ z|EwU!Dh>)8&E@11n6VBDh$6-E4Y9~vGb#a9lM1f^0RS#wBg!DbOh5R;pOYlFNwOTw zaPRNMqW)kiS9T)61LRoRK}V=-fFKYCIreT(?0vxbJIQpn=8+JKJ*g{gnJeBIGd(C| zPmM6+y)G@d!{@KGAIN7-c?W>JFk4!Lij#pU;;~aMM$SP5yvqg_f3G{~qWQ%NqWgxzBu0n|;L1kH-yqKYf-8kUQ^sA*l% zS5_||y^{1vEhxklv5#ECi9P{~*!;i)@soq$Q1Vlv3 zX0w48&yzE9N4}|%*I2P<9lIbq>ocE-!eTR@OTg3V&kwvKUw)6LcUE-^2W%}57j`7Q zm7oUnd&!_!f~6A+(pvh(zzeS05tO0-7P;ixwk8AlbZWrvh$BdPKK4=(-Z7;ZuZa|p z_y%qv?mqi|+zJBh|dVJ8ju}z&*`5pBHa7=V1-lr0& zpCeWFbn@oPW11c{h(hS)UeR#p;I_&3n%sE#b!Chb0!0)5CwN9PTTKhqm5E;HIA6 z4}8x*`JcFRc#NIc`E+tXqR(?4K2?|cyPG~uX<&n$o){pGda|##+Yfo4CTAF82_X9v znKSV_z#ng@lr$33CDdu#^eCsYzjb|E_p~JW4Hkd0OH|8|>J1ri<5P>CvZcfsBNxzP zs4aLUeM?!2G{fXMzm~fJu~I!9PxpT@KHjRb*a{zm!u%l;>&d3s^m)mKe+TW zt`_W$o;D~HRlvO@W?cWb{>8SMK91$RO-Ww_d`8~xYr`9goh;jGk zWHx416c$ZvtpS3exu(ke%@1IQ+SrjsP1g5Afthc{xa5_-mZOASKI6|jYlq+~6pA$X;5>*HWbb6Y^O)7B8UswB4k`-mYjOTV+X z)M?+g&kwk$6G~1L->8P0F3t&-MAHQFa0u~%08SqPNCb>q#m~-ilY_Cn5kfw%cs-ZnPYFHkCPj{~A?CZd6oA1NzccHd|;vdQ^tqtuh z4l~wwE*DjMP04|{vTf|^6li}#wnaN^so=-R_?rnU+(8pk&996!NLVJCqTnEy>pa+= zD*NX&DyI$d6KA1y4QoYEisKLIrUw*oF!VpuRgT{06J0MqG=(%Sw*+UxX}cWt7CKx@ zYIdM_GW##@wKu=Sm!3A;U4~pp#tC(4>*|MDdy%-~CsI5(xDS$JtK){4nN>g?acA7j zWn5QYOMdIU%S3z2Up97+jaJE6n9-Xa{cI5}IAMd_-n(qr9HQnBLzt&Hb)mhC=vZk1bM&n$61m%`~vm>I^oto)W6d&gRzOy2}m3qCxF@R!K<* zwOA%>!XC@tRuGc7TM7}fJ=x#wys(orf@LBD##V;o^+hR^qu;~DfvKpiN%IC2Zj@2k zj2BRd5i$K8C1}*(Na_2cjV#=8?zF(79sHd>=9;0B90(mt!IJ9ic9+B;9tkB5$(1sP zlJTliFEv%2ob{%p&b>Lt^C2P!;9N+_Y#!C{?pVg8^UdPevd@lF&SvUdud4jR&%cKJ z)TQ(gNG1cTYH^zxH4y1)4)|LGk{aZNa}k|eHsqWjHfAAEw%=dhECoHQ(q8>uJRE_? z9F}uDvGsqik$fP;*sGT*ttUU;tSINb)^exqeMQ-`k7CIibtjIQQj~OT+IwZOsmeLj zz3ed;3Gi3iBaJg{7x!@;J~K4NW`^FTo=4<|Uu)cpuUQ;=GzQ$F#Cc+47g( z+FL25n-JF$z9tpVuX=-6DOq6S!HVaqHMpWjb%KO1fbxcRNb z?KClEI_1T>sHh!`9c&8v#{+h2EAc+8UfrZl{X#1pA9#q@v9mHcUyXv5*+C!o>`}iR zeP+D9*_ZN1Oj0z&9?{_Qe_R}7em+f5M;z#J@_|{1gCVPE-QD#Z780HhCEn&903}PV zJxiH0BV>a9kmUSTyt6gFH)0x}R*LWZZ1K7a+MLVVm5%pCf+W<1Zks)9WXDMv=$6`vM zuo{{mecYyxw3@r@()EObXD@CF8V^3nPa71^&_xYs zZF2h$U?o5A9u}#LTM#d9cQ)vwUvaQ&oAy2XGSBJnYGofZR*+uBT(-=QXuRIHPV87S ze2zsb(5l{hPdaC3s=cyX+*{mSbPg~faHLxa)tBk7dl9*w&$2SOF#bg=*eLl-7K2w ztFOl0e>;TYY9fqNX+AhU`#l|dJ)M&SxTPWeCi%@WeYgOvTCTMnP3?YkY>(W0e<&!!S1?`81rO({m zTJg}Iy%91;n9BoR-i8lIbJn!gsn;a*Bm^OK2TJ8mBh%g>=j$NaN-rOC*MEe7%y8!_)qvMY$632GjxehRld`D#gF{<+ z^t|*eJ1roDbTG!1r?T(2Cr@tdSge^N@NQM=?Ly}!Ec)Bx zM^pvZlV_@oaIMaRYzFK^=Zp*4C4X8>mv2nv+U||n&12?D=;N+68=R(iTDuJ_Z$!_E z$=)vQy9UV}DLWlrRIF{>?QX0oQtD?1okPhTHTh8EpJ62WpsIJU{n7>5`U`=-S?Dbx z8wdnKzbaX2o4rtb56hkLv+QKUEpnZB{#;rW<<8o#OkLwQ&>AAZr4yuKW*PHymPt>EVRGLfG_9^^tU+mL;* z0lF1`d*e7V{M^?M4DokaX-m`o^!3%6jQ;7u4fB`Gir64TvDc+I z>n-5ZAV@%<A+a_Ojj+dWb{*C)1fXjPPbs(6!cM6R1V_xd}^OA*eBdrU51VcPfYkD2Z_M+OC5 z%NQi@D1H}28nuF#bH%c34b1i>{SOa7*nT+ zL^j#{bc?oo`9Iz6u9l%T9WQemZu>%a$#Q+DVvk(N^@#9BEDc!PW?twnZud4Y#tc*t zGS)E(YmCmWmC6Z(yFu3W{wWdPFOOk_Z(0KR2p)NlL_)&C>SRokVI~Mt3gY%a4*JTl z8vxsOh(#OCcrEty%euvT`r$7hSf-7HU$yFMX4iZFP0492nUjiC<5q@$ab>Jq`NxYmq(^oWs}~Cr_|sIReK|e%8wgx6V^<<$i#AQ8y_9Xy?%-vzCL+$#hky)oHAs- z^Z(Fte*Y>#_QCArZ4oRK>}!NBztzWPeM3;F0(*LSB|I0sB6IsXQhIoN7e~qjKJB?H z3B^pj^$^NC@|>n(uf#mtt%Z{HTta>NKi6hmH&%*a_Sq?!bCsS3=;(*&WoZ?$KfM$B z5(>Bzh7okLMDc59r+my(u8@d3iZ{ zK)mP>L6{|rgw$99gA9&l*fU&~Bmyf9c|bFbj%4$l(|zgB@utlVsj9AxSG)WZjnRFwU&pB;}iDMU2rUI+0 zl`qTH)+j=m;z;6!Mp;*TR5Gqr66MfwkeTr@XZCz_5fB!4*CTT=5T(eO4;^Ab#e!j< z|6c&-2^jVn!DrC@A;@f#ZS@wEVy(>O%SR7Ie!SS7%i)#Wm)7QGzt-ORBkI}+YsrUY zH6Nyn%H~1!VH$j}_Qw)lK;ez%T^TQs>OVrwKb51w92^EL2P%O9o0d|@C=R&-uOL#q zNw=g5x)}R$Jp&81F(Hp4qgl*=eIk;}9ND?e7m#N?wTv}aCWJpsX(2-%#v6^QHTUiF zH*W`_x|;@8wyZ)maN5aq6vt8-CE)|pk{M<;()-#AF0C+O9P>zl%fw@DoO~^iXmnmAs$D8hV0XFQa=Ad?yTdG~kdi}b+sAY=vcNbtylMxPQizKT+ zj{xqvyZB)O&RbO#Rjqc_Y+BWhB9NT@2o@Jr4*P65Z z`8d&@3TSh<)tU?_-%N&TxB>o_YFpE_Rot4ewdz0ie`tn)JM=}Xl-_RWcj)*pQ(c<9 z9(37Y4?iRqOT#*amvuMM558Sg!OLlq{|RJgoXpCVZ;nM_EgLRAj6UqVapQ-y8W?mG zwG3e30llzt_;WpW8DYET0Tu~kl1bwTB2A;=cv#h0+>Q$zxF%I?@KIns48hVv9yIt( zNlrz};Z$8FLUC!-)B=-+o3J*6{#48HK&^mVQZ=4-ZR@twSUd2m zF4vzQY|8-vo{pLpuVOvpkd? zFE+-ATb;SfK6I)5L#I zUk(2J0}P&icr$qWx7W`xjJlMHWw>=M!MESNeS-2o4>tz`~215 z`=_t!3X1o~$e%q*8o{Ig@%ri8Kf}wvH#5;krzh{;VF0Fl_YP!nV&_Q?iumr` z=_~lb&ZhADXbOG(yLVXv1E_cJicE0EElY|dl!B>u@6xN)MRjrK(-H=OxR7g^W!cbo z?_SrF>F?e(s}|qA`>5r+cW>%R{PO6&&WYct{?CT9K{2=XXS-nnuF-!UO6ku3$}ax< zd7ei9uNZf=tmY_#?tfSe>!pn$mwnS0m&f-UW#`Ep+?L}BhMz{@yUd#fzv5~I>QM%| znl~#WML)j!c{&2WIUh}%r8G>fAHWN5C)IQ@uXi9FvHsfqz(?6dHfrh#QAMxvfKiws zIrTfW_M(73GF=onwpgx@1^U$0Yp83rLiI@Ym0ZhuHmnx&QC;k1jaQ_cFUHBFm=EnF zA2pP&oo#e9%qPim{qxn2KMukDn~vT$RImq8r{uG?TzKkxO}ae0xugDbF=%C%U2us!Z^339F@b5OD^j5UoS3s4=ukc3Y_N!dX9}y$7?x2+(7_EQ$5y>zyI3J zrQYl3uGSt;KVXbNYX#)o*A0*Ml$lyv0z7{Rp%UQ3f#7QE!h( zaHeE|MzD@Z{QZ*J;B-0~*{mAAx4|nAmLOSF(`NZaP|TCD{V<*X82sV%)i?k6!)oOS z^rh#E6b8KG$Ctg?<7_O)qv@>tre8;pPS0)jWAOLWSHY{7uK~FrpJwG%5%}%uO#gZj z$|bbl)RDU}2IF#DCBsqK|6@Hc|6TBBu)M(x>>UQ+>w1D8-UQ2awheDd;H?MK<9Gt%X>s5}Yubyccyhz5w(M7=3asrzLZz?;RAzG{#JY4`}pez|A z#sZpw#Z?*203-MVE+&5bzWH{N$b{!S?FDb_EC_UfE>{o!d&1ZW7vBH}-kb*SLG@%9 z00rr=3jR&lbbn?uPs0 zWyZ%13;+4gf2K)!{^%iIfUtZO9RD?_-9Su`jf43(sE1v_ak2bmwt!SPmJ5;z&~?5z zwv)4eUkOKWe4L?ROc&J?#(8EFUKkacI5&pFXPu}#&GJHWtxU#JqqBHb6*RV4f*t(7 zM-Tto@iD^v1b$tuLBBd?eQ>3l@cDc?UuCI(uPQr#a)od6=KR9|*GE0MGMUR4^~!-K zD{Hv?G0ErsrT5ma$+*1mVVM9O@MVH07j{+F5aX7pmNjdhEYF5&-lZ6`p5#x~O!M+h zZr=|x`vmOvsuj!N=95*lbBPBy8|yKxCn^jtp<2a>p0MVp_scgwErzzLc@0wc{j{>p z5{+RITuc|ihop&nE|v>6{*ImoXyBTSnA>bPvwLaw(jb6MSu(Wz27MN+a2aeX5vdC( z8!>Dbqz_<&K#GEuIMq=(7*8Y<)QmN!R)S4oi-*=SnAZ~5?jU1;#n|M?^UcT~qp{&Q9K>$N2dYsJ55W{7b`!*WFq z*2bbf4E7_){GfZj!1L++zsnf+_3M9`aO1z6%g+D%91odcqEiHP5-RyfCs}rytn#PZ zoa`q_u3d-_b_V1|&mLjQ3_6r6HQ%ES(1$3Bw(b9M=6^U38I1qbgU|4MD*Zo@=f?cM x8s1;~gxBu>a_m1B{NVS09mw Date: Fri, 4 Oct 2024 21:00:41 +0100 Subject: [PATCH 022/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 9d3ad69466..c1e249f2ff 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -42,6 +42,7 @@ jobs: - name: Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' run: | + git config --system core.longpaths true cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_windows --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet - name: Test MLPerf Inference ResNet50 (Linux/macOS) if: matrix.os != 'windows-latest' From 2b3467d00a0b6cdd51ccce5df0e4faa86cf9879f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 01:30:54 +0530 Subject: [PATCH 023/658] Added an error check for push-mlperf-inference-results to avoid push to unintended repos --- script/push-mlperf-inference-results-to-github/run.bat | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index 4cb9b2bd00..2646d1a399 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -5,7 +5,13 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH ( echo "Error: CM_GIT_REPO_CHECKOUT_PATH is not set." exit /b 1 ) -cd %CM_GIT_REPO_CHECKOUT_PATH% + +REM Change directory +cd "%CM_GIT_REPO_CHECKOUT_PATH%" || ( + echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" + exit /b 1 +) + git pull git add * From 7c44964d8101f6ed9039c18ed842b281851a6724 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 4 Oct 2024 20:07:18 +0000 Subject: [PATCH 024/658] Results from R50 GH action on windows-latest --- cm-run-script-versions.json | 602 +++++++++++++++++++++++++++++ hardware/gh_windows.json | 26 ++ mlperf-inference-submission.tar.gz | Bin 0 -> 19293 bytes summary.csv | 2 + summary.json | 32 ++ tmp-run.bat | 36 ++ tmp-state.json | 3 + version_info.json | 602 +++++++++++++++++++++++++++++ 8 files changed, 1303 insertions(+) create mode 100644 cm-run-script-versions.json create mode 100644 hardware/gh_windows.json create mode 100644 mlperf-inference-submission.tar.gz create mode 100644 summary.csv create mode 100644 summary.json create mode 100644 tmp-run.bat create mode 100644 tmp-state.json create mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json new file mode 100644 index 0000000000..a15a67dbda --- /dev/null +++ b/cm-run-script-versions.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "onnx,fp32", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_cpu,_resnet50,_offline,_onnxruntime,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "cpu,resnet50,offline,onnxruntime,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json new file mode 100644 index 0000000000..b7cf960dbb --- /dev/null +++ b/hardware/gh_windows.json @@ -0,0 +1,26 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "host_memory_configuration": "undefined", + "host_networking": "Gig Ethernet", + "host_network_card_count": "1", + "host_networking_topology": "N/A", + "host_processor_interconnect": "", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "status": "available", + "submitter": "cTuning", + "sw_notes": "Automated by MLCommons CM", + "system_type": "edge", + "system_type_detail": "edge server" +} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..bf71db6918094ba69b22dc5bf2e2d4c34ce0be95 GIT binary patch literal 19293 zcmce-V{m6r6!#h1Hh-~g+jcUs?O!snZQIFAY}*r0Y}-yI+Why~XP-A)wOhMY`|6(R zKBxQMQ>VK7c749Y(J&y}vLKcq;FsNAuIthbOlSVJDIv8<7HHpD;@m z&2a+hc6%O&{kVhGk+0Lqtw@xM8Gri#Qo=8xL09rGL*HjRULqx`!`-Ahp)avg_0g}H z+l&4$yR7x#5F^J=c*^F}moJ_?ffu0`nf8qJ`dsxl+~AK_rnakxuSWj28{!9lb{wk4 zRVSCNFD;(j3vK8fVlUhvqt~x0ZXoY2WWTy1#q-I=nOJl%yn- zQtsM`7HikSovTPmd>k#qL9pKcbL-gIFz2X6x{np1;j{#&1qj9U@FmJt{*|ulT=B`09VgH>-2!V6K6Ur zDZIt5yp8~59U7e)dArd($c~pVa6U$eV(|5D)aS)dn{ix}L;q8MGB5^t{x$ybdie~C ze$hU&%0djDAR4w}-^t{@#f=fKI&PdGwvmFMSUPu#rQ@i}&u_e3vCl-~N&)#qF*iu&)>rM#_rOXJTLl!3I+{ zxPHDm2J%#vffufGT-`fvnM7#n(H&0*zN`ckr(-o+1;465EPcEdd8&WZaGabLWyK}` zITG>4)u%wn>xSEk?ztHKNL6RAZZ{coe{Om8gUp|rVen-evxf|+L4z62#HbyVeJ2Tu6!Q%T|}=b@A+7eK!K;*&v#^V9FO-e zf*C4f?w7$GGmCUN_&TmH%wy8t-gi$hI^>>%dzqJbhWzPK*pxsrCy+0(5#CS#xZ~to z!aF3ODag}tFHGpasT=>rQ^6Os86}Mg?MNZi zr8WX8)wob{asv-?Oe=7<)}K984MJCceBE3=J3?>!2?5_OE3sd^5$;l{G{Bhrd2rLf z`SO)mV6*|ne+PcPoQkpLvpXLz#t%EGr?&vw^3Am|;O>oBL$P)Hmsgp?;}H`zK31J> z&uim}{((wk52^@)xX_RJrNwY2pvYIKO)F-4)2-kpTk1ZTl9$59!{htf+F@yUFeS2t zle!9u=Mu#4A`dizPSw*W0=Xep^n|1T9K5%YMj(e5WNx=1Q{(vIz-eC@B8z3 zz5nz7dVN^GHvDvQ@hD;=ETiO7b<&X@k4}*=wo;oYiLlF~@WJ^oi5!x{CyzeX z7Vx-h%=GjY(4*$Y&_WD$Y|;8t<;97)TxL$jc2yG7+J*Eax(5V(vl6u0Ie@=Myni)I zP2_kuN<@#GkruFdqrggwfP<5x9VeGL?SmZ!$&$uMnuwK*DY|SIh*+H8YN_5sG4VEc8z?R?^Km|3I)a(c!S>9qQ<6 zad+lSm47wpEY_Coqnb^l5rqX|I%C0_Cm0kkkbLsMM8yK=8mMW70W7Y;j)D+;s^-|i zbIiF^_Y+~KtmCBdXhHS#kh?Pq2z6y79SzcO2`32Zv0mw_7j;BQLL`c4IaVEYKr4L%Zc0{IA4jGh4r&JVR9!-; z4Z0|a(pURz^Q$&Mb8 zX)<^_CMlU2_Zv2Y3PY(0IvD{<$nr7m5EVd%xQTJ|oWJ?Vw0fF|@kRO)^h*Vck*R%0 z%<)D1iq(+z@m^rsmO2i$K`iqiHlsXmW$m~rjDh$uG9zROHq14&kP|RITEA4HSd0$V3w90f43$tWgeIUmiAPlx<_z(kiIcuPF$e0dCqoX4XEzOerHtVWRdi z__QDzbBTc^wk1iriOf&jYfFc7F<_EF9j@L>ZvWma8BQ!%44#qHSaOTZg>VFxY2@GFE zi`6gD*bDc2by49Q7TKizuG+u(cQ>`YMWi?Q?3O>nkdbJ<5B=&=xCWB`sSi$?21y7H zgV_Bu;6{zdX8-fc>wa@Qkt64N`=⁢W%{)da+4Xaw7Got?)_leoq@DjBWqQ8^~~8 zV4JswMI?7hW-j1<*G-JV&x&kGoTh2ckN~3Bv^g*mX0{&hN##WiIrFJWQhDMOk}faNNC|cS|^gkd5r!sFPJua7#)2!QRXa zGq1h9mxoq<(!S&%&jgJqO6Ia`hZ8g!AGHQFak5Wa`!}u1QYQ46pIa?zl!4?ZjyZ5CaJ9PbpS#^ZXP<=| z@$i9Qhmo9Sc)rntV#>*L!*pAD8p%z>H>9c12hX z07EekXAcL7f84^)dF+dlTQG$a^EBK?l+YCkfGqA!ax|PK;yHWaM zNXf|{UT+9dpkuK!w2Z(DXpEIiVU>Re=@$42qi9H21Hg9n11Q;HB2tv0LC6!TrIBt1 z`sCz6AV>Q8VD4l{kYo?xNFc7{?6fvfNYJzc;8P%A0!~Pq14=+RVXe5iGBO|8Q-Z$& zUcpvDE>sfa!PfBFVbzP6ty5qRsFdRf#KHoUo%+P_me{Cy1B7A53or?Uz?qEL;brRB zD=>!PGLgq2yTUsl$4Wt9L3J@_1Nw2TIH7-#Zjhf*K~62+9}s zg$yW7-~eNQ2#A3QRCh{H6~{Y}5MF(O8qy+t5(XEI9{}l<{0&_SmPoKNRKH82JnJBB z$4uM=W)34EONyDGVY0q!iwtT(JrzKmLLrQy9_W}snP%%`7Q~BIQ zbgJh}R)%1QFhfH!(qK2C*4Cz!7JT1bL6HtSiYi_V08q=_ z;6^C3{6S0!=z>qAajwNODgluIH7B%(M2otMI{@1Ll#u|t)`+2%BU8SRtiMihfhXZ0nq_TLUSfDu--AneI4gO!dF$9S%y~FioJs$ z{s$t};H?S4LLX!^9OO6v*(n60$_b&s{A0=(jyuVC0b1|%F!4U0unE?z>6yp(_!zi5 z7H-C=T~kMpQf#d;(E597i#rK3bL^o%eaA4+&LEeui6+fP!2W?WU`3{f5UKQE zRHzip=vL?Sjd%wr2+L$H?tbzY@ZVH}=Q4glr&%7+ab4 zP)1A~GNL;QUe+V7-g@hym?Z2h9&Y$3HPx-i*ia`axhPF$MCm_7(qcX{{;`lwQrAh% zXJNBFBwQxeTSvT1=uOv!8NcV=)aYBBYv+|~ur~ZF{}la%x9t*@%@4=?{m)japMvp3 zJ^zr931L6iPzbxM#rNh0KjggcTU$Z1x_0O*kXyqI(~f)km?q|b6!cg-fcuav9$8Z* z^x(4ZCv6tsvKu0%5)0T-BZN^LRFPN36G3t3`w7_p7|3tHStC$=PiN4&P8Vr!S!QB3 zLazejT&78bMH1Py&O=VjGQ;TlmGqmx1fj%x^&Y}MBgG#f$I^EL;a74zSILj_S`wa^ z_LkCZ9B%<{ZDiS?bH=%;4i+M>2j(7@*DN}?@khV$LHy)eu4WInCt0HF#CMeGo?&;Q z+GL&z8S`WFo7TTa;VEL8B#@5-#Zt(%N|7Sd`o4ya<2oRxW7aqdcaKJXH{rLsdufJT z&wWhrD;rQfWVXKZj#Mh60@hK8_P3R^Hnj39}gI>%u`6&(N}m&+|s7*lGQXtT0wAPAIYZzD#qh2A8T1 za*qW+z-(jz0 zV9z*+lx$qz3w~Z5?=SD_8NT^H+AV)2xRw%raM53k_~)CyKR-($`@<^AzReqDq} zGCXvLy-K}D>45|%^HxA?E6z??Dy8zm`d#wiwkNY_X5-@iyp@kkiy0W#zCP8Tc!@cE z!YfT?Au9=zT2523x4_sa#RySuIhfCWbz&+>zNJy?q9}Hz1NT^ftr83G)0UGf-=!xm5Fp#H>bJ$4f>|B=*8!!xsPzKBZr zS9)aqo+qjJOlJ~NwN#3aV6|VMK^2pCTUKAz(AYInFR7l~w#7WnTrX=&ARm~~fPGyq3pp>)%+ zI-{0}Ru~y_^IB@iN-D%-5ux7xlFZ95_n(SVVPqzO;+@qPaWFJk9a9MVPqdMUQ4zY8 zPe}ckh1v(rR%=+5i2JiWzt$9B>)})!u<8XP;ooMre}E7PG8FU zzg^v?2@b@RkxAXm)ll*s7ShSn5xsV`^Qojn*H-MM9Z%D;Kqk>!o7YO=nNQhU+bG~;for% z9E>5AtnJoft^o-hnw7j1wIu*mZ9-PNT$)Ho%mCPII&3H$=rY-qnE%YJK~*)=A6k1m zo4p*`GvEuG#VkySNU%G$Ji#MXjH$Nb@*%|?MfVeYUXXbwF=5Sb<=ZXfBiJ8G5zpft z*Zy&76*4fJ3jC`$C#a7|h>Lb5jI_|W2?(h{m=jsN_vqy$+xI#=E4TBMCDRJC@;TJu zYe}>XgGlVD*j$l}L#Q{nZ$}d^#+q4gW^;iyWxg&jB94ib>zy39w5Ja3EIDcS&-pITeg`pR6|9GcDQgET)*-d z0&yGiqv`FVo?P^CGLlW-UMpqgvGjD7#U@^GsD_O&IU~UayJv+%G$vyb`_jPZ0362dFDud2rDgQcHY{IN35Tz zxpJ9QM`HZ5u-b@dEJ6x{n860NUE+*Bvtftzx7KPhX zY!$<1dyocWb)z$&+O+v>H|m;Wq}B5On~r|GqI7A=ZxK=7m3JLmk|mC^G(B zf=DMRx3q)Ia!r%WgPOCmk+$rLk+S?FQf{)e$PR%#ht*gYc##b9_2pJ4w4*dY4ii6f zyR(C$gf@^s4(+V$mO0;=tlmNORBh&EBi0>W%v*X!AH*|wGpI%dMKFNS7_Qi0%2<40 z6YxnY5EePq3F-WUe?q?v|Atbxn7`v}q?3Yb7x@YoI3k`DG0^?M9qursSI4PUAV7yZ zj=Fz0iOw52&MS@A{-h1Vb)qmDz$B0ukr6^`Jv*G9T18qopLN-j=2W~D_7Jz$=r)b$ zE!TlD#70glJ)1qJe8`ojyQyq0FZJhgv-sO72x){rd1C6%_v?#YfvfeiC#i=ZF4{g5c6x@lFHnN7^FqG&WB|^{DHSS{DL5%;JIX%O7h9&vQwb{YuD2)_*MP) zV5BFM6sB@kt&&^;mF1I3;IER&_7yX;NugB6#!72JC7Z8{bkKGV{q^HyJAA{BY)L|2 zv|Xs)(xqfLUi_XzsxR2AHwF=nT&qGNRk!Hv$55I`x|^ zTBU@fp8GJDdtC(X>2{sC`FJyaa8gkwtA7oeT9WBk%i_{3DGEt!Tdb=D{^nKHNEErI z<#8+w#Cng4;MT8dnNx?RPQ2(9_{iqW-rYv_Avg9*WZ&`c6{6$th=9>e4Ake*-zxCC!0Tv!w@ z&6?_e$3JTOFc#w9r!{H$9wkz69TEObXf3ZTJa2!kA!#Rdr1&*+83~<$x#@;`b@qKAhF5Na9zLUFqtjYRDAbvL|QKgVPsfZLx0)TsFmQp*%W;* z9|VJTSEk3?UOwX_wiT3!Q*H^TEN+%4+*L`1f@-|ic{hUS1>d1Oacq3n8a8I~4ov%Dcfkebtat&@Y-}PG4YUYs}k^J3& zpd}z;bPCmV+BCsc2uDu77zbTu{z&g^JxQ$3_%^N?r)h~l~T~^4oeFQ|i z)-*dWwL@-^{%%Pdo?rZarrAmLN+bcfbcpuXr>?^~PZ>Ft4#YaBj>BF%doGp2-e%daABZe+O~7Ru~YR1^65LlOQ_DI}X;058Mj5ZgtI0#?SuA zvh!YN)+GiJ*o#e8{2NJLG$1k>dk?py32_VFTriLK8Fgs$<07}YioBEl9tUza_Ricr zRA%nQ;_XGZqu)WXTQnPfAdwahYGT1k;i{-F!{v~c8`k~nBaO(I>^OBcS6Su4 z`Zph<6t;`E*S>C9+Q-_ZDdeeSH#sl-F^_q8j*jS?=8RiBK*f=FqE>(?0{_v0r)ZY^ z?O*|om;G5@TH3`9)_@w{D-73RA$wjUY{N8|BT_xf08$%0J(4ln?9SX)Q!1_E-CNO{ zjIR3>OnK%9iuKq#>?LcSmkkA5$88N#6aE7hs`I))c)g66k0YM+eGJjRP>6auj2%02`Wtz4l2W%AjzNGi(fChg z2ZE2l61P&WT>gihbIER=+`_x6xg^r>VkbX*vWj&p-bzXq4zbLW=}>X(h#%sbb_(7~ z>gX3dSHSX~4``>Vqa7D@-1N9I_^H{zF}X}KhR$)|4WVR@^C-iIP?7PO|Y!yQkYWtFC> z#S4I&yL7-h-g>RY8p~69oc1>pu>ijFxdGzHg&LLPXfj40VTqh-VwMUb5WS*R*_j=g zqrDHyOoR4xWahnu-l++=*hCw;As`$dbD;dYkiHFU_)eL)fx!7OMx>AJp1%{Z{6QZh zuigPqW0H6eO@Cpy5!ao90O*q7q%z<}L94@WpYJiHiktM|Uz~0!e7E&v&d-L9T!LJT zr-wgLIYZT+gg{!CAy^%U!|!Rovl30{D<&bj{qU|f%dnJ{N-&}@NKB+dR_;8%|85cw z0dJCVA`_6eZwGmsaWa6q`zy8TzKkehl(oIk5p~XWF@uUObA7Rs7D*sE`$1zHc#tsE zbIOxe|77OgIx?pQ6yOXB=|O~Le(j*;dQh0zeS{cF!GF~Z8xqQ>Rds3M6R7I<{M~?$ zSpa?Mm4-pbbyt02b8t*l(~;=t{Uh&K8B1dN$N|^uUqe#qmK*Dut4%ZhYni39*EG3m zQjV+)#DME~gbVm~P)(=bk|0afMjux!EWf>wxMX5mE8xXDtW!1-lgdR4)MI}ui!t`0 zIb|q4;<(5swWCN}s}iXBK_gM-IeyV@xnAV5zvXtls22ZD>(Wwi9`8NyAe;QranJyA zjeBxX;kjo$eECM0`dzoh2s3kbQ8`Yc@m79Ur0FXq23!-=TxSd?b9NhpYFua42nIs5 z{5l53;-0ZTpH{!S28$dE0^vSfLM&QI~ibsa^@G5IC)b5<3ub8p)E4g%2U*BP$L z$7CH9xC(&!9^WUQ;9{av3R1@o*2zX6!#TCiQyIoQ#;6>JzYVkN`xr@~TMI`&j|x-q z@5-Xd1V$fd(Qc21a=a85waiLeNt*>7^W&5cP8_}$Mew9=pEqd%J>^vVTg@28y$2ku zhG{;sW^YCYWx$?7xkaX>kH|FV?hw^O?9SNjW=Shbo-OZ2L@SybU%1o*IhszsqP@G%j=vbwOYVM{we?RVTPSN zMhf|BZjt)3$6=M?&A)KRUVpxqr?>ufN5k&EWmA-Q zVZ}Zm5IV}i^tkp$q z-k-K^_rC8QT!>rp$}2dO>@*}KVbeqp%$B9epsw;Oxcx>SyieX5-|y`K{(E1H{=}h$ zwoSImEvskE9MjG!TdSBnBBN>3NMvxq%2uaDJ(4R74y8@Ea-_ima%K! z#cgYlP}erUWl7Y_>kyEVYDXv>70aRgKDC1~XSo0W zL}yT0{6}Y8yrXr;+XxcIUUZQL!<(8ixQxJ=gU*p1gM=}v_U{1_IhPY#XfgHv{369< zjdv~RnRPZRohq3e3~`-CLy4Yssgba1M2Xgc(^l(c!JCm$OlAfR!&C)z8Ywd_N*1TH zK*tYd?x0;_GYQ3lnPv;tJ_r2S1E`{sS7 z${w>fP&VR{TS$^om6-83iH{+!IwlTNs0Oc%(Wi&CiXf{Xwv#ot@my zD^NYyS%XGfa;H-^Ry79Jwhhn@J@|c5oov<=vYu6EM3S=>pkFAXSrCfb0AD3)p$ZI&oi zKq?CE*~lK%ZzEwuhLDkUZl{e)6kJ=4(Y{-+5Cb8CMp=(13$mXB`qag&qMv=(e1#+| zXHVIOGNOVSo(TFh&JV^0b~h&RqYgu!h7PSIubxx49Z4e#08**UIV}ps*<9y^q?*>C zeoxKPG1k!R170WOk*7ZIMb5;J>`vRr4pACpTu+^{R3A&R;-4G{BXd$-An$i~uEK*j z*P|%LAa=rSir*Sb|B$Kql@3$}3x+ksiD`DQ21nQ;$ZW?ZI4r_5(&J13`o11HI>uL@ z{_LDP#)=tlcn$Ue?Fh1AqggJrLW<`m!pQ>9dkz9Mt!6;G@!#_CbTLyn4rljuu$=cY zzZc=CPC^AnN6R!Lg$nY*qO=Bq>apa;C(Bq|gX8>!J~_hD|AUrD&BiBlGBdy zwkL#`g?Xx;2O5lxNU`Q^`4)=uB7t>VE|iWtV`Hz*rqGAQF9)M7H68A(Ptz|6{jlr>?| z`Z5fQlnX*ZWj`V^S-+#J)EVd7KdG-2jr=gAb5Px!`GIwPdY0N^bDM| z6I4icmk>@#EG?VBGn5R!zZ203oNAMS1_5m|Me2MOYK~6 zFuBWvY!->F`~Nu02xyn!@_6}n0>#*-5bm6kBShto+UDTzoRlL(7rt`GzVL3q5HQNyG>9{XZMKHp{{Tj1wbED75->fR^ z7_}_m>~!v|!;W19eYjJ2l_Mnpz`aJ?qd(h~@C1T$$D52nVOU9z?h~Pec(TAT{sgWR zo$T_|!QF{15+n&T`Pn^8CWy}w+nY*PDR;qx`M$-a|6ouAm<6XXb?E|RH?rajvWRFpgY(?%)c0D z<2~mfU=s#W3snIH5FU<5VaAj?LZqKXok&7p!#@x@c|f=n+lIOkn=Hc_SqUoS&n+2x z+>prrEu1-N9jHJTgyjDMIZ`2&2!{7{cNW2jirj)=|kR1bfg8vJsBMiHhMC~WXoj_MEGz4)Hs5wsD zegf1DB;ZCI*oA?Njk*)-49tcd!^m5wQdSJKk&bxFy6IudsTv6zrsg3*c)=|}<)~e9 zCm|VO^=|9p7B?4dDCj|14MC9L=|`Bw!*HdC_XB$zuH3&h64xh=Y;D*-64LJo!OkQ9 zOTq+9=k5W$)rjy#W||(!j=8;%?jnJfg(SP}u?}03Bdz$GXHl z3ltGUP=B~NQR5%PKqennlYqoSCV@#96vG#}xTTjjV#-sD9F^bmN(A1Jmq`SItKT?@ z^BAeJt`~hfM@;RF`Zy*G)9RX#Wz#`S(&8Dty_C zagP_&jnzZ|ig9aSYRuu=#skn1kb(;5@4FQgf>xmx0yOr!2<#RRC{&{LXxRVR6Gp>D zeR3~(&;t3h*w2SwAnxq5T(E&{WioP2gva+QFi`v%l*Ef#m^%)d9gTn179?f=F7GFR z`=7A>Zuk=pz)2jf|^W+oQx*&iV8c-7C>*_XXfWle=%b3 zvkL&+&H2Gh+^F?H93+#Ed_5ZCN`Z!zUU<E63Z zOU2H)=Tun!4)UN3GvimpDY3x&^?aq}<$ep(f$cp$24>YeT7}(dCWPGYEGl*_LvCRA zZjprDl$_27bW}e{{%9xDOHAuG*3rDKdc0eIDqG56 zh5ps000`M$Rh?9?Z0VsV5=F=(LkD)ZpShx>JO`K7J){jdyI{Q&p2GzhW(kS-wz<}J z(JP&ctt*WV!b)aqI_~;rPK7tx)K!>@Q$70Pk!rP1nqr3C%EVJDNDe{-8*4wDTFf#0 zsyy&XZSyqUd)~oaypM2Ci^Zs{Y}nYggeB40ooTPO)67+z_+e&7?T&Y3okF8wMpgOH z%Sr5a*-p*j^>f~Zj<;mP{zN~q5i!*^tCsqtP|Iw-kA;H5beY)$tmTX5rb65r0wVP zc0egCllMx#p?@^b?C{KO^b{lQ+@*Zef0(*Gc>tzx5IS6E6}jZuo5#w!kC7LX>kkr* z?8a6VZhrE2s*E4&)aEcA3D0nt<5N`m+oEwk>bIUcamFP#G~An-m~P(1g?HZ7HOn+> zoa_8*Ta+-nlcqI(3?!(4U;hKHI=97db=DRLU!B5n@EA1=uno9)PoMU0|N4ZFIi=!8 z^^a=2>*k(lq@KZ0LW7SBgLHlmK)|CvD*lwvnEKN=8ZYGYGCOlTufzMX0-gQP=0gx2 zNzHiKdvJ0rr}Jl2M-{^ZOXmAk<7;=>?XQj(cO~Rm4gc69tdRTJO#AQZ+EAQ#EZ;m^=$( zXj*61|92LK=s4zAMEHt7<)Xyb>T0K_KV4y!=XJsHQ0f*B^Vw#nW$g|$U-rQDqrv9& zmdN!_M>Am;9gT-aJ{kt!# zL-hjURbE;(+I~^Jo>H6SttNP)b~#fQ>3$h?b-|0M0r?Y3+u!<--a+S}PO0DXX1Dd} zWn^q#M7tl=vH@wA3-Q%a187Up>wkz;7ZnV2@%yIh*GG9R>MWu~& zmZMXY8trgk?oOWwA64HZ1bGHPPjsQ?dt(8sA#*!C&ejyru1I7y0mkKja%o$O?sPP~ zyd0KQ?-Zp5o3Mk&B8SAuLZL!gkd&56s*-CJd!E3{?=N~{plZAuwsnc$WJK&~fXw*Q zEkPGcKICIfN!d3J&RJ?Q#joy6Gqf$)pihVW~)!yC{G%h~%Py*klbUMU9g z{ttA_4bI(y)}~L#mVB8<^Pg^UZ*&c({Qgx1ACjT*^(l8Y^N4fC`G!9p82XQsJ51?oKNd<>7Nqh?6`1r7#2u_idx$EC@$az{e^Husj{YuLzH%2gp zs^$Tg+Jt)juL(!^1PI#~OgfoinpWlFB7_>=IhB_2jZr<&Oj5 z#&LhU9^NwttC8*aa}d|M-idMcnd`TRuS>P;UXf30*Y;U4ym;O1Z@)X(Ga zylbj+xbdF0mwHxv;G?g@Arji*22vy67glRl0M-2?W%+1w;i_&q8~Nu4LE^_H>D?AD zY&U3HZ>lY$0+%P~dKM7lro_+Y_LSamh6 zxKWNzXqfM(H`-(+oO^|vojTrxVjJ)7ndGct`@auKWaxix0Up?a1gteL>KheWmQQ!P zU0i*9=Wh(yNBBB=!OKGO8b!T-)ndrk^?`-CxPKDvmutl;6+y7T|f*{3{o2an*w)&ecRM< zio&aVCM=k%j(^j|8+zBZo@DeC^~o)YYu^i^D}67N@5TFs5|Z<=g`fG#8(g=NB9(nk zxYF;LfA8?^d6h*_l`(oeJDNapXrGepp>Rwcd%a}M>!0@|G*vf8H&@9zXx^ zUb%0ax0rYamy_fEo{=@vpfbsg_%3=b!7mfp6~sS6h&7wi@3tU5#U*d-M?YG|2BE;g z?U&j!48HvR^(SNXJpN|_k?+3~cx>c{rp@J<=Sj=gd*%(lQ{25E7-tH1HR$sd?y!!4w%x)m+xCh!$oGry8eTQ$ zoGRNC998V$&A(+t{a1pw;uV(1?0_EnmHbC8{HSpTEU_1~wt*D4^=80EVwa_kM>l7%^sxM0D7K^h);z`hVX354rZ1&@p!K z_Ze33Lpg8yQnpVRsJIMaev|j?wSisU&X6=*fyds;B={q8dB4<6Aj}86Y4pQCb$;BP zFowlcKEQw1`D>3H3<)Zp(=zjj}D?eey~hi3Hd#4 zHcCbl$J?ti9(f?}8VbHl&M@VseU%Xx4Yo4F#H|?$aGmz1*r|Or|K)$TKl`%35em6j z3+EL<_`Q4JbJLGB!}Qxl89@q*iCL2Ppy;>7;p+aV(E2!g{!_VY)BCDho{%Op7%Qz$ z%&&>cE0AAeCiv+`W)78JtU1D`k2#Znb%bygEuw7_s5DqB7%f{o;%13;dK_TdZv`*} zXrfPjx;W3-1WtX@bMyHwzwk4!_s<`BTh4`Qd+naSMCG;~|2wN2+&m3H#X>>OlT|Q= zna|#E2{$9{oL~DlrS~WA__FG*Y(IbV)OS1a*fOv@HFsZr{8N_|h89MAM}yYHN$)+x z@6bnsE+g{|cm`14x5ob0mUg9^ z_?MIfk&~Qob>2rC1AgITKE9X`eikA7sZ%nf40P}-W{2(M6UD4$yu}+{wqEn-?yxwV zRmk9C+T4TTxwP49wYItSGopiYK=;mAXDa-(sKHCwHv@{e>Z3jcTNzmFwLAh}@=r4F*4(!*-n<1x{aA{@4;#hZ8aDr6?8P00}zS~#c71xj~Pim9!>`NVqSYpXa(vHt10#s^zOr5N{!vrO}k4> zHuz~_=h$IHZX!?3G*Ml`(k4SXBi4fF8aT(a37Rd;@pc1+d zH0{9oEP*${u&Vp838?U5GDKYpYR9x(*Rm_knq*iH{KICB7uBjEys7fL?_LF05y27) zes5b!4w6Y8Ocxb??$uwA5JDjeGJtXRU7tgmki_KM{ub}r_skS-MQr; zUv>@#hjN26@7p$k+Dy16DeJ{}gy^cP_0po(g~(4aTXu@x6mr?4c~h9RE^k|!x`W$7 z2wU>nmqauzZ|jV1$O5u@B^m1?yYxiSvH8(Xma+%FmLo|Zu!B0q;3`F{_;}O(F2IIe z*BsPta!a*KS+8HW7qv{W{_Ya2X)?m$Y>{LY=n=qOcNafQz%dnoXxIp_ z_?y7dVrWO;a+Vj<#VD_xPFq0Myy1W2kmn$x>E*^4ajP?T)rW3%dZLzY_NQ5GAp5LE zJJ;UbY5ZqewoU)s82<&>-S|(ay7`~a@-*?^XRii-`2hw`KfD>d{jb+AFpRpCie6CA9Q5$ryJxT92Roa>@1rSn_3z$g35=iKy-Sth zSssPfq^Xjrf?uu|7uCgGOiLI8;ySKnmSsENy?b5HrN4XEELwc`?xTV1Z|XVx^5ou5 ziQlRI&xW%>F}L<-yI}%u(0{>{a_4_l7ytb{Pow|WjJsY|bCN;#KP-mz(ncm!KkCb? z<9klBi)0RN%jpEePb2VM=FNg%dA$PlBm-s5o0XBWA7B4G9f9ARkEYF18m`w5;Dxu7 zYPy)$JCIIzf9-zYlk74ZHT5vm(d#^56=q0I{Z6mFD4>r_7X^+jmg{4IK7IWf>RPQ( zJyCtF*0P=ttHpd&7kgFXH7OU1aWX09Lp#Yw4W(;m8=VaENwQr3eEs8(Lva75qxTJ! zys!FKi{ehkNi`cyD_n_MXN%WRyjtxuxIU$M^U&gx@eB&Mpn$D*aygrwPS$k0dCS); zbPo`qd+RrooL<-@AK9{OPum<0wVeJF>k!X|qtW!k5ce7G^V>vS}U#q6T%Z9}w zT7R`VNNe+msgt$4`J6lJPBvF3Z!X=1m&o=d#lXWglJNhlR~z@jIJc#ol*5lpE}HdUFRyqH zEx#@boaY64j*U>qYdJvNK>%e_J=TuD|Ju!^-s|VC)*eu2;IA*9K7aM1KhAIJ|E1vW z`Y&$GZ^!?C@FkzO7%H(Jfz~0)0ERm1?GXvilq}E)))9%nUr`%8n~p{{tA_7w@Ct+_ zNEX$!S$+`|^JHv4Oy@rae|YxlkN^0?YUK#@r5B472E60PSH0QeY^=tk>8$)?zm6b1 zyRg}h!QY>~3SNEp8juU}X;xkrf#0sr^sg78TtWNCI&wG0U|epiWH>7Of2s%OzYG2X zmN%Gzy~6-}T~F}Cn_#)__AL0%rL=>8Bbwp!Y2AGZCRKF$<~0tUgJ-Y*cAewttLH`q zFO%_bbQuV}oWSP6o662+h!*b!PZt0gC`$&3v4Cb|aa~3;!U+C=i-{kb1%IfC`QtyZjy3+gx~{2qyp~YF z?|)rP0!)@na{Td)wKd?&{V@#>&lkho?uPs0WyZ%13;+4gf2K)!@#rC5fUtZOoc=AS z-9Su`jf43(sK;HwX|eoewt!SPmpmwUhIIUkOKWdYYkNOc&J?BT~cStl&CL zHDj5H3dTaoQdJaEK(UF;Y>~26RnXXG33l-RJ$m>*PEQf;C-CcP4f^#l>w|0EgwGe# z`6^5OdsW%_lWTmNHy0lUxIXI1waHw*s8p5{+RITuv9khop&nE|v>6{*ImoXyBTS znA>bPvwLaw(jb6MSu(WzBl;{@;WF4(B2t%7He%Q=NFTrkffNNRajK(oFrG*zs2Pn= zCOD6bK52ROrwxg2B-bmYI|y={^w%KPy7^aILRVU^Pba)A-vv*zY%x!=%T>(*P?KcYKhJ`HhTuj1vj;DL)pLwqu96}vveC4J-}2%6y3pXC z|L>Ck+))#2_0Lt=uh*6=tQG&JnIXm%4a*fdSQ`)fIy9L1LHB%t=hOLrS26CJ*Z*?q z#(#xmo&Wba9x}s3rwHgIRPvEdvg|ZjPOZ2+bM0FX6a0RR91 literal 0 HcmV?d00001 diff --git a/summary.csv b/summary.csv new file mode 100644 index 0000000000..3a66ce11eb --- /dev/null +++ b/summary.csv @@ -0,0 +1,2 @@ +Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.5764,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json new file mode 100644 index 0000000000..786ef94185 --- /dev/null +++ b/summary.json @@ -0,0 +1,32 @@ +{ + "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_windows", + "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":17.5764, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":1, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.19.2", + "operating_system":"Windows-2022Server-10.0.20348-SP0", + "notes":"Automated by MLCommons CM v2.3.9. ", + "compliance":1, + "errors":0, + "version":"v4.1", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + } +} \ No newline at end of file diff --git a/tmp-run.bat b/tmp-run.bat new file mode 100644 index 0000000000..02f4e7a468 --- /dev/null +++ b/tmp-run.bat @@ -0,0 +1,36 @@ +@echo off + +set PATH=C:\Users\runneradmin\CM\repos\local\cache\b461cd87e6bb43b4\bin;C:\hostedtoolcache\windows\Python\3.12.6\x64\Scripts;%PATH% +set CM_GIT_BRANCH=main +set CM_GIT_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\8bb62b59178d4feb\repo +set CM_GIT_REPO_CHECKOUT=main +set CM_GIT_REPO_CHECKOUT_FOLDER=repo +set CM_GIT_REPO_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\8bb62b59178d4feb\repo +set CM_GIT_REPO_DEPTH=--depth 5 +set CM_GIT_REPO_NAME=mlperf_inference_test_submissions_v5.0 +set CM_GIT_REPO_PATCH=no +set CM_GIT_REPO_RECURSE_SUBMODULES= --recurse-submodules +set CM_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 +set CM_MLPERF_INFERENCE_SUBMISSION_DIR=C:\Users\runneradmin\CM\repos\local\cache\eae25a88b2b14eb7\mlperf-inference-submission +set CM_MLPERF_INFERENCE_SUBMISSION_VERSION=4_1-dev +set CM_MLPERF_RESULTS_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 +set CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE=Results from R50 GH action on windows-latest +set CM_PYTHON_BIN=python.exe +set CM_PYTHON_BIN_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64 +set CM_PYTHON_BIN_WITH_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64\python.exe +set CM_PYTHON_CACHE_TAGS=version-3.12.6,non-virtual +set CM_PYTHON_MAJOR_VERSION=3 +set CM_PYTHON_MINOR_VERSION=12 +set CM_PYTHON_PATCH_VERSION=6 +set CM_PYTHON_VERSION=3.12.6 +set CM_QUIET=yes +set CM_TMP_CURRENT_PATH=D:\a\cm4mlops\cm4mlops +set CM_TMP_CURRENT_SCRIPT_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github +set CM_TMP_CURRENT_SCRIPT_REPO_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops +set CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops +set CM_TMP_CURRENT_SCRIPT_WORK_PATH=D:\a\cm4mlops\cm4mlops +set CM_TMP_PIP_VERSION_STRING= +set CM_WINDOWS=yes + + +call "C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github\run.bat" diff --git a/tmp-state.json b/tmp-state.json new file mode 100644 index 0000000000..e03109d44a --- /dev/null +++ b/tmp-state.json @@ -0,0 +1,3 @@ +{ + "docker": {} +} diff --git a/version_info.json b/version_info.json new file mode 100644 index 0000000000..a15a67dbda --- /dev/null +++ b/version_info.json @@ -0,0 +1,602 @@ +[ + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,src": { + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlperf,inference,utils": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_uid": "e341e5f86d8342e5", + "script_variations": "", + "version": "" + } + }, + { + "get,dataset-aux,imagenet-aux": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-sys-utils-cm,bc90993277e84b8e", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_onnxruntime": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "onnxruntime", + "version": "1.19.2" + } + }, + { + "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-ml-model-resnet50", + "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", + "script_uid": "56203e4e998b4bc0", + "script_variations": "onnx,fp32", + "version": "" + } + }, + { + "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-preprocessed-dataset-imagenet", + "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", + "script_uid": "f259d490bbaf45f5", + "script_variations": "NCHW", + "version": "" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "" + } + }, + { + "get,sut,configs": { + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_uid": "c2fbf72009e2445b", + "script_variations": "", + "version": "" + } + }, + { + "generate,user-conf,mlperf,inference": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_uid": "3af4475745964b93", + "script_variations": "", + "version": "" + } + }, + { + "get,loadgen,_from-pip": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_uid": "64c3d98d0ba04950", + "script_variations": "from-pip", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.psutil", + "version": "6.0.0" + } + }, + { + "get,generic-python-lib,_opencv-python": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "opencv-python", + "version": "4.10.0.84" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "get,generic-python-lib,_pycocotools": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "pycocotools", + "version": "2.0.8" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "benchmark-program,19f369ef47084895", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-program,program": { + "parent": "benchmark-program-mlperf,cfff0132a8aa4018", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_uid": "19f369ef47084895", + "script_variations": "", + "version": "" + } + }, + { + "benchmark-mlperf": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_uid": "cfff0132a8aa4018", + "script_variations": "", + "version": "" + } + }, + { + "save,mlperf,inference,state": { + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", + "script_alias": "save-mlperf-inference-implementation-state", + "script_tags": "save,mlperf,inference,implementation,state", + "script_uid": "b14b813229c444f8", + "script_variations": "", + "version": "" + } + }, + { + "app,mlperf,reference,inference,_cpu,_resnet50,_offline,_onnxruntime,_fp32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "app-mlperf-inference-mlcommons-python", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", + "script_uid": "ff149e9781fc4b65", + "script_variations": "cpu,resnet50,offline,onnxruntime,fp32", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "get,sys-utils-min": { + "parent": "detect-os,863735b7db8c44fc", + "script_alias": "get-sys-utils-min", + "script_tags": "get,sys-utils-min", + "script_uid": "a9af7714d3d94779", + "script_variations": "", + "version": "" + } + }, + { + "detect,os": { + "parent": "detect-cpu,586c8a43320142f7", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_uid": "863735b7db8c44fc", + "script_variations": "", + "version": "" + } + }, + { + "detect,cpu": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_uid": "586c8a43320142f7", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,generic-python-lib,_package.dmiparser": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "package.dmiparser", + "version": "5.1" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-descriptions": { + "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_uid": "48f4622e059b45ce", + "script_variations": "name.mlperf-inference-sut-descriptions", + "version": "" + } + }, + { + "get,mlperf,sut,description": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "get-mlperf-inference-sut-description", + "script_tags": "get,mlperf,sut,description,system-under-test,system-description", + "script_uid": "e49a3f758b2d4e7b", + "script_variations": "", + "version": "" + } + }, + { + "get,python3": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_uid": "d0b5dd74373f4a62", + "script_variations": "", + "version": "3.12.6" + } + }, + { + "get,mlcommons,inference,src": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_uid": "4b57186581024797", + "script_variations": "", + "version": "master" + } + }, + { + "get,dataset-aux,image-classification,imagenet-aux": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-dataset-imagenet-aux", + "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", + "script_uid": "bb2c6dd8c8c64217", + "script_variations": "", + "version": "" + } + }, + { + "get,generic-python-lib,_numpy": { + "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_uid": "94b62a682bc44791", + "script_variations": "numpy", + "version": "1.26.4" + } + }, + { + "run,accuracy,mlperf,_imagenet,_float32": { + "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", + "script_alias": "process-mlperf-accuracy", + "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", + "script_uid": "6e809013816b42ea", + "script_variations": "imagenet,float32", + "version": "" + } + }, + { + "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { + "parent": null, + "script_alias": "app-mlperf-inference", + "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", + "script_uid": "d775cac873ee4231", + "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", + "version": "" + } + } +] From b1497099a36bf02235219d5bf7c3477c4601f853 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 01:42:23 +0530 Subject: [PATCH 025/658] Added an error check for push-mlperf-inference-results to avoid push to unintended repos --- script/push-mlperf-inference-results-to-github/run.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index 2646d1a399..9c4dd0fd18 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -11,7 +11,7 @@ cd "%CM_GIT_REPO_CHECKOUT_PATH%" || ( echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" exit /b 1 ) - +echo %cd% git pull git add * @@ -25,7 +25,7 @@ REM Check if the previous command was successful if %errorlevel% neq 0 exit /b %errorlevel% git commit -a -m "%CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE%" -git push +REM git push REM Check if the previous command was successful if %errorlevel% neq 0 exit /b %errorlevel% From 49b5f5e1cdf29392be22106ab5cd0e68355220d8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 01:42:48 +0530 Subject: [PATCH 026/658] Revert "Results from R50 GH action on windows-latest" This reverts commit 7c44964d8101f6ed9039c18ed842b281851a6724. --- cm-run-script-versions.json | 602 ----------------------------- hardware/gh_windows.json | 26 -- mlperf-inference-submission.tar.gz | Bin 19293 -> 0 bytes summary.csv | 2 - summary.json | 32 -- tmp-run.bat | 36 -- tmp-state.json | 3 - version_info.json | 602 ----------------------------- 8 files changed, 1303 deletions(-) delete mode 100644 cm-run-script-versions.json delete mode 100644 hardware/gh_windows.json delete mode 100644 mlperf-inference-submission.tar.gz delete mode 100644 summary.csv delete mode 100644 summary.json delete mode 100644 tmp-run.bat delete mode 100644 tmp-state.json delete mode 100644 version_info.json diff --git a/cm-run-script-versions.json b/cm-run-script-versions.json deleted file mode 100644 index a15a67dbda..0000000000 --- a/cm-run-script-versions.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "onnx,fp32", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_cpu,_resnet50,_offline,_onnxruntime,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "cpu,resnet50,offline,onnxruntime,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] diff --git a/hardware/gh_windows.json b/hardware/gh_windows.json deleted file mode 100644 index b7cf960dbb..0000000000 --- a/hardware/gh_windows.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "accelerator_frequency": "", - "accelerator_host_interconnect": "N/A", - "accelerator_interconnect": "N/A", - "accelerator_interconnect_topology": "", - "accelerator_memory_capacity": "N/A", - "accelerator_memory_configuration": "N/A", - "accelerator_model_name": "N/A", - "accelerator_on-chip_memories": "", - "accelerators_per_node": "0", - "cooling": "air", - "division": "open", - "host_memory_configuration": "undefined", - "host_networking": "Gig Ethernet", - "host_network_card_count": "1", - "host_networking_topology": "N/A", - "host_processor_interconnect": "", - "host_storage_type": "SSD", - "hw_notes": "", - "number_of_nodes": "1", - "status": "available", - "submitter": "cTuning", - "sw_notes": "Automated by MLCommons CM", - "system_type": "edge", - "system_type_detail": "edge server" -} diff --git a/mlperf-inference-submission.tar.gz b/mlperf-inference-submission.tar.gz deleted file mode 100644 index bf71db6918094ba69b22dc5bf2e2d4c34ce0be95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19293 zcmce-V{m6r6!#h1Hh-~g+jcUs?O!snZQIFAY}*r0Y}-yI+Why~XP-A)wOhMY`|6(R zKBxQMQ>VK7c749Y(J&y}vLKcq;FsNAuIthbOlSVJDIv8<7HHpD;@m z&2a+hc6%O&{kVhGk+0Lqtw@xM8Gri#Qo=8xL09rGL*HjRULqx`!`-Ahp)avg_0g}H z+l&4$yR7x#5F^J=c*^F}moJ_?ffu0`nf8qJ`dsxl+~AK_rnakxuSWj28{!9lb{wk4 zRVSCNFD;(j3vK8fVlUhvqt~x0ZXoY2WWTy1#q-I=nOJl%yn- zQtsM`7HikSovTPmd>k#qL9pKcbL-gIFz2X6x{np1;j{#&1qj9U@FmJt{*|ulT=B`09VgH>-2!V6K6Ur zDZIt5yp8~59U7e)dArd($c~pVa6U$eV(|5D)aS)dn{ix}L;q8MGB5^t{x$ybdie~C ze$hU&%0djDAR4w}-^t{@#f=fKI&PdGwvmFMSUPu#rQ@i}&u_e3vCl-~N&)#qF*iu&)>rM#_rOXJTLl!3I+{ zxPHDm2J%#vffufGT-`fvnM7#n(H&0*zN`ckr(-o+1;465EPcEdd8&WZaGabLWyK}` zITG>4)u%wn>xSEk?ztHKNL6RAZZ{coe{Om8gUp|rVen-evxf|+L4z62#HbyVeJ2Tu6!Q%T|}=b@A+7eK!K;*&v#^V9FO-e zf*C4f?w7$GGmCUN_&TmH%wy8t-gi$hI^>>%dzqJbhWzPK*pxsrCy+0(5#CS#xZ~to z!aF3ODag}tFHGpasT=>rQ^6Os86}Mg?MNZi zr8WX8)wob{asv-?Oe=7<)}K984MJCceBE3=J3?>!2?5_OE3sd^5$;l{G{Bhrd2rLf z`SO)mV6*|ne+PcPoQkpLvpXLz#t%EGr?&vw^3Am|;O>oBL$P)Hmsgp?;}H`zK31J> z&uim}{((wk52^@)xX_RJrNwY2pvYIKO)F-4)2-kpTk1ZTl9$59!{htf+F@yUFeS2t zle!9u=Mu#4A`dizPSw*W0=Xep^n|1T9K5%YMj(e5WNx=1Q{(vIz-eC@B8z3 zz5nz7dVN^GHvDvQ@hD;=ETiO7b<&X@k4}*=wo;oYiLlF~@WJ^oi5!x{CyzeX z7Vx-h%=GjY(4*$Y&_WD$Y|;8t<;97)TxL$jc2yG7+J*Eax(5V(vl6u0Ie@=Myni)I zP2_kuN<@#GkruFdqrggwfP<5x9VeGL?SmZ!$&$uMnuwK*DY|SIh*+H8YN_5sG4VEc8z?R?^Km|3I)a(c!S>9qQ<6 zad+lSm47wpEY_Coqnb^l5rqX|I%C0_Cm0kkkbLsMM8yK=8mMW70W7Y;j)D+;s^-|i zbIiF^_Y+~KtmCBdXhHS#kh?Pq2z6y79SzcO2`32Zv0mw_7j;BQLL`c4IaVEYKr4L%Zc0{IA4jGh4r&JVR9!-; z4Z0|a(pURz^Q$&Mb8 zX)<^_CMlU2_Zv2Y3PY(0IvD{<$nr7m5EVd%xQTJ|oWJ?Vw0fF|@kRO)^h*Vck*R%0 z%<)D1iq(+z@m^rsmO2i$K`iqiHlsXmW$m~rjDh$uG9zROHq14&kP|RITEA4HSd0$V3w90f43$tWgeIUmiAPlx<_z(kiIcuPF$e0dCqoX4XEzOerHtVWRdi z__QDzbBTc^wk1iriOf&jYfFc7F<_EF9j@L>ZvWma8BQ!%44#qHSaOTZg>VFxY2@GFE zi`6gD*bDc2by49Q7TKizuG+u(cQ>`YMWi?Q?3O>nkdbJ<5B=&=xCWB`sSi$?21y7H zgV_Bu;6{zdX8-fc>wa@Qkt64N`=⁢W%{)da+4Xaw7Got?)_leoq@DjBWqQ8^~~8 zV4JswMI?7hW-j1<*G-JV&x&kGoTh2ckN~3Bv^g*mX0{&hN##WiIrFJWQhDMOk}faNNC|cS|^gkd5r!sFPJua7#)2!QRXa zGq1h9mxoq<(!S&%&jgJqO6Ia`hZ8g!AGHQFak5Wa`!}u1QYQ46pIa?zl!4?ZjyZ5CaJ9PbpS#^ZXP<=| z@$i9Qhmo9Sc)rntV#>*L!*pAD8p%z>H>9c12hX z07EekXAcL7f84^)dF+dlTQG$a^EBK?l+YCkfGqA!ax|PK;yHWaM zNXf|{UT+9dpkuK!w2Z(DXpEIiVU>Re=@$42qi9H21Hg9n11Q;HB2tv0LC6!TrIBt1 z`sCz6AV>Q8VD4l{kYo?xNFc7{?6fvfNYJzc;8P%A0!~Pq14=+RVXe5iGBO|8Q-Z$& zUcpvDE>sfa!PfBFVbzP6ty5qRsFdRf#KHoUo%+P_me{Cy1B7A53or?Uz?qEL;brRB zD=>!PGLgq2yTUsl$4Wt9L3J@_1Nw2TIH7-#Zjhf*K~62+9}s zg$yW7-~eNQ2#A3QRCh{H6~{Y}5MF(O8qy+t5(XEI9{}l<{0&_SmPoKNRKH82JnJBB z$4uM=W)34EONyDGVY0q!iwtT(JrzKmLLrQy9_W}snP%%`7Q~BIQ zbgJh}R)%1QFhfH!(qK2C*4Cz!7JT1bL6HtSiYi_V08q=_ z;6^C3{6S0!=z>qAajwNODgluIH7B%(M2otMI{@1Ll#u|t)`+2%BU8SRtiMihfhXZ0nq_TLUSfDu--AneI4gO!dF$9S%y~FioJs$ z{s$t};H?S4LLX!^9OO6v*(n60$_b&s{A0=(jyuVC0b1|%F!4U0unE?z>6yp(_!zi5 z7H-C=T~kMpQf#d;(E597i#rK3bL^o%eaA4+&LEeui6+fP!2W?WU`3{f5UKQE zRHzip=vL?Sjd%wr2+L$H?tbzY@ZVH}=Q4glr&%7+ab4 zP)1A~GNL;QUe+V7-g@hym?Z2h9&Y$3HPx-i*ia`axhPF$MCm_7(qcX{{;`lwQrAh% zXJNBFBwQxeTSvT1=uOv!8NcV=)aYBBYv+|~ur~ZF{}la%x9t*@%@4=?{m)japMvp3 zJ^zr931L6iPzbxM#rNh0KjggcTU$Z1x_0O*kXyqI(~f)km?q|b6!cg-fcuav9$8Z* z^x(4ZCv6tsvKu0%5)0T-BZN^LRFPN36G3t3`w7_p7|3tHStC$=PiN4&P8Vr!S!QB3 zLazejT&78bMH1Py&O=VjGQ;TlmGqmx1fj%x^&Y}MBgG#f$I^EL;a74zSILj_S`wa^ z_LkCZ9B%<{ZDiS?bH=%;4i+M>2j(7@*DN}?@khV$LHy)eu4WInCt0HF#CMeGo?&;Q z+GL&z8S`WFo7TTa;VEL8B#@5-#Zt(%N|7Sd`o4ya<2oRxW7aqdcaKJXH{rLsdufJT z&wWhrD;rQfWVXKZj#Mh60@hK8_P3R^Hnj39}gI>%u`6&(N}m&+|s7*lGQXtT0wAPAIYZzD#qh2A8T1 za*qW+z-(jz0 zV9z*+lx$qz3w~Z5?=SD_8NT^H+AV)2xRw%raM53k_~)CyKR-($`@<^AzReqDq} zGCXvLy-K}D>45|%^HxA?E6z??Dy8zm`d#wiwkNY_X5-@iyp@kkiy0W#zCP8Tc!@cE z!YfT?Au9=zT2523x4_sa#RySuIhfCWbz&+>zNJy?q9}Hz1NT^ftr83G)0UGf-=!xm5Fp#H>bJ$4f>|B=*8!!xsPzKBZr zS9)aqo+qjJOlJ~NwN#3aV6|VMK^2pCTUKAz(AYInFR7l~w#7WnTrX=&ARm~~fPGyq3pp>)%+ zI-{0}Ru~y_^IB@iN-D%-5ux7xlFZ95_n(SVVPqzO;+@qPaWFJk9a9MVPqdMUQ4zY8 zPe}ckh1v(rR%=+5i2JiWzt$9B>)})!u<8XP;ooMre}E7PG8FU zzg^v?2@b@RkxAXm)ll*s7ShSn5xsV`^Qojn*H-MM9Z%D;Kqk>!o7YO=nNQhU+bG~;for% z9E>5AtnJoft^o-hnw7j1wIu*mZ9-PNT$)Ho%mCPII&3H$=rY-qnE%YJK~*)=A6k1m zo4p*`GvEuG#VkySNU%G$Ji#MXjH$Nb@*%|?MfVeYUXXbwF=5Sb<=ZXfBiJ8G5zpft z*Zy&76*4fJ3jC`$C#a7|h>Lb5jI_|W2?(h{m=jsN_vqy$+xI#=E4TBMCDRJC@;TJu zYe}>XgGlVD*j$l}L#Q{nZ$}d^#+q4gW^;iyWxg&jB94ib>zy39w5Ja3EIDcS&-pITeg`pR6|9GcDQgET)*-d z0&yGiqv`FVo?P^CGLlW-UMpqgvGjD7#U@^GsD_O&IU~UayJv+%G$vyb`_jPZ0362dFDud2rDgQcHY{IN35Tz zxpJ9QM`HZ5u-b@dEJ6x{n860NUE+*Bvtftzx7KPhX zY!$<1dyocWb)z$&+O+v>H|m;Wq}B5On~r|GqI7A=ZxK=7m3JLmk|mC^G(B zf=DMRx3q)Ia!r%WgPOCmk+$rLk+S?FQf{)e$PR%#ht*gYc##b9_2pJ4w4*dY4ii6f zyR(C$gf@^s4(+V$mO0;=tlmNORBh&EBi0>W%v*X!AH*|wGpI%dMKFNS7_Qi0%2<40 z6YxnY5EePq3F-WUe?q?v|Atbxn7`v}q?3Yb7x@YoI3k`DG0^?M9qursSI4PUAV7yZ zj=Fz0iOw52&MS@A{-h1Vb)qmDz$B0ukr6^`Jv*G9T18qopLN-j=2W~D_7Jz$=r)b$ zE!TlD#70glJ)1qJe8`ojyQyq0FZJhgv-sO72x){rd1C6%_v?#YfvfeiC#i=ZF4{g5c6x@lFHnN7^FqG&WB|^{DHSS{DL5%;JIX%O7h9&vQwb{YuD2)_*MP) zV5BFM6sB@kt&&^;mF1I3;IER&_7yX;NugB6#!72JC7Z8{bkKGV{q^HyJAA{BY)L|2 zv|Xs)(xqfLUi_XzsxR2AHwF=nT&qGNRk!Hv$55I`x|^ zTBU@fp8GJDdtC(X>2{sC`FJyaa8gkwtA7oeT9WBk%i_{3DGEt!Tdb=D{^nKHNEErI z<#8+w#Cng4;MT8dnNx?RPQ2(9_{iqW-rYv_Avg9*WZ&`c6{6$th=9>e4Ake*-zxCC!0Tv!w@ z&6?_e$3JTOFc#w9r!{H$9wkz69TEObXf3ZTJa2!kA!#Rdr1&*+83~<$x#@;`b@qKAhF5Na9zLUFqtjYRDAbvL|QKgVPsfZLx0)TsFmQp*%W;* z9|VJTSEk3?UOwX_wiT3!Q*H^TEN+%4+*L`1f@-|ic{hUS1>d1Oacq3n8a8I~4ov%Dcfkebtat&@Y-}PG4YUYs}k^J3& zpd}z;bPCmV+BCsc2uDu77zbTu{z&g^JxQ$3_%^N?r)h~l~T~^4oeFQ|i z)-*dWwL@-^{%%Pdo?rZarrAmLN+bcfbcpuXr>?^~PZ>Ft4#YaBj>BF%doGp2-e%daABZe+O~7Ru~YR1^65LlOQ_DI}X;058Mj5ZgtI0#?SuA zvh!YN)+GiJ*o#e8{2NJLG$1k>dk?py32_VFTriLK8Fgs$<07}YioBEl9tUza_Ricr zRA%nQ;_XGZqu)WXTQnPfAdwahYGT1k;i{-F!{v~c8`k~nBaO(I>^OBcS6Su4 z`Zph<6t;`E*S>C9+Q-_ZDdeeSH#sl-F^_q8j*jS?=8RiBK*f=FqE>(?0{_v0r)ZY^ z?O*|om;G5@TH3`9)_@w{D-73RA$wjUY{N8|BT_xf08$%0J(4ln?9SX)Q!1_E-CNO{ zjIR3>OnK%9iuKq#>?LcSmkkA5$88N#6aE7hs`I))c)g66k0YM+eGJjRP>6auj2%02`Wtz4l2W%AjzNGi(fChg z2ZE2l61P&WT>gihbIER=+`_x6xg^r>VkbX*vWj&p-bzXq4zbLW=}>X(h#%sbb_(7~ z>gX3dSHSX~4``>Vqa7D@-1N9I_^H{zF}X}KhR$)|4WVR@^C-iIP?7PO|Y!yQkYWtFC> z#S4I&yL7-h-g>RY8p~69oc1>pu>ijFxdGzHg&LLPXfj40VTqh-VwMUb5WS*R*_j=g zqrDHyOoR4xWahnu-l++=*hCw;As`$dbD;dYkiHFU_)eL)fx!7OMx>AJp1%{Z{6QZh zuigPqW0H6eO@Cpy5!ao90O*q7q%z<}L94@WpYJiHiktM|Uz~0!e7E&v&d-L9T!LJT zr-wgLIYZT+gg{!CAy^%U!|!Rovl30{D<&bj{qU|f%dnJ{N-&}@NKB+dR_;8%|85cw z0dJCVA`_6eZwGmsaWa6q`zy8TzKkehl(oIk5p~XWF@uUObA7Rs7D*sE`$1zHc#tsE zbIOxe|77OgIx?pQ6yOXB=|O~Le(j*;dQh0zeS{cF!GF~Z8xqQ>Rds3M6R7I<{M~?$ zSpa?Mm4-pbbyt02b8t*l(~;=t{Uh&K8B1dN$N|^uUqe#qmK*Dut4%ZhYni39*EG3m zQjV+)#DME~gbVm~P)(=bk|0afMjux!EWf>wxMX5mE8xXDtW!1-lgdR4)MI}ui!t`0 zIb|q4;<(5swWCN}s}iXBK_gM-IeyV@xnAV5zvXtls22ZD>(Wwi9`8NyAe;QranJyA zjeBxX;kjo$eECM0`dzoh2s3kbQ8`Yc@m79Ur0FXq23!-=TxSd?b9NhpYFua42nIs5 z{5l53;-0ZTpH{!S28$dE0^vSfLM&QI~ibsa^@G5IC)b5<3ub8p)E4g%2U*BP$L z$7CH9xC(&!9^WUQ;9{av3R1@o*2zX6!#TCiQyIoQ#;6>JzYVkN`xr@~TMI`&j|x-q z@5-Xd1V$fd(Qc21a=a85waiLeNt*>7^W&5cP8_}$Mew9=pEqd%J>^vVTg@28y$2ku zhG{;sW^YCYWx$?7xkaX>kH|FV?hw^O?9SNjW=Shbo-OZ2L@SybU%1o*IhszsqP@G%j=vbwOYVM{we?RVTPSN zMhf|BZjt)3$6=M?&A)KRUVpxqr?>ufN5k&EWmA-Q zVZ}Zm5IV}i^tkp$q z-k-K^_rC8QT!>rp$}2dO>@*}KVbeqp%$B9epsw;Oxcx>SyieX5-|y`K{(E1H{=}h$ zwoSImEvskE9MjG!TdSBnBBN>3NMvxq%2uaDJ(4R74y8@Ea-_ima%K! z#cgYlP}erUWl7Y_>kyEVYDXv>70aRgKDC1~XSo0W zL}yT0{6}Y8yrXr;+XxcIUUZQL!<(8ixQxJ=gU*p1gM=}v_U{1_IhPY#XfgHv{369< zjdv~RnRPZRohq3e3~`-CLy4Yssgba1M2Xgc(^l(c!JCm$OlAfR!&C)z8Ywd_N*1TH zK*tYd?x0;_GYQ3lnPv;tJ_r2S1E`{sS7 z${w>fP&VR{TS$^om6-83iH{+!IwlTNs0Oc%(Wi&CiXf{Xwv#ot@my zD^NYyS%XGfa;H-^Ry79Jwhhn@J@|c5oov<=vYu6EM3S=>pkFAXSrCfb0AD3)p$ZI&oi zKq?CE*~lK%ZzEwuhLDkUZl{e)6kJ=4(Y{-+5Cb8CMp=(13$mXB`qag&qMv=(e1#+| zXHVIOGNOVSo(TFh&JV^0b~h&RqYgu!h7PSIubxx49Z4e#08**UIV}ps*<9y^q?*>C zeoxKPG1k!R170WOk*7ZIMb5;J>`vRr4pACpTu+^{R3A&R;-4G{BXd$-An$i~uEK*j z*P|%LAa=rSir*Sb|B$Kql@3$}3x+ksiD`DQ21nQ;$ZW?ZI4r_5(&J13`o11HI>uL@ z{_LDP#)=tlcn$Ue?Fh1AqggJrLW<`m!pQ>9dkz9Mt!6;G@!#_CbTLyn4rljuu$=cY zzZc=CPC^AnN6R!Lg$nY*qO=Bq>apa;C(Bq|gX8>!J~_hD|AUrD&BiBlGBdy zwkL#`g?Xx;2O5lxNU`Q^`4)=uB7t>VE|iWtV`Hz*rqGAQF9)M7H68A(Ptz|6{jlr>?| z`Z5fQlnX*ZWj`V^S-+#J)EVd7KdG-2jr=gAb5Px!`GIwPdY0N^bDM| z6I4icmk>@#EG?VBGn5R!zZ203oNAMS1_5m|Me2MOYK~6 zFuBWvY!->F`~Nu02xyn!@_6}n0>#*-5bm6kBShto+UDTzoRlL(7rt`GzVL3q5HQNyG>9{XZMKHp{{Tj1wbED75->fR^ z7_}_m>~!v|!;W19eYjJ2l_Mnpz`aJ?qd(h~@C1T$$D52nVOU9z?h~Pec(TAT{sgWR zo$T_|!QF{15+n&T`Pn^8CWy}w+nY*PDR;qx`M$-a|6ouAm<6XXb?E|RH?rajvWRFpgY(?%)c0D z<2~mfU=s#W3snIH5FU<5VaAj?LZqKXok&7p!#@x@c|f=n+lIOkn=Hc_SqUoS&n+2x z+>prrEu1-N9jHJTgyjDMIZ`2&2!{7{cNW2jirj)=|kR1bfg8vJsBMiHhMC~WXoj_MEGz4)Hs5wsD zegf1DB;ZCI*oA?Njk*)-49tcd!^m5wQdSJKk&bxFy6IudsTv6zrsg3*c)=|}<)~e9 zCm|VO^=|9p7B?4dDCj|14MC9L=|`Bw!*HdC_XB$zuH3&h64xh=Y;D*-64LJo!OkQ9 zOTq+9=k5W$)rjy#W||(!j=8;%?jnJfg(SP}u?}03Bdz$GXHl z3ltGUP=B~NQR5%PKqennlYqoSCV@#96vG#}xTTjjV#-sD9F^bmN(A1Jmq`SItKT?@ z^BAeJt`~hfM@;RF`Zy*G)9RX#Wz#`S(&8Dty_C zagP_&jnzZ|ig9aSYRuu=#skn1kb(;5@4FQgf>xmx0yOr!2<#RRC{&{LXxRVR6Gp>D zeR3~(&;t3h*w2SwAnxq5T(E&{WioP2gva+QFi`v%l*Ef#m^%)d9gTn179?f=F7GFR z`=7A>Zuk=pz)2jf|^W+oQx*&iV8c-7C>*_XXfWle=%b3 zvkL&+&H2Gh+^F?H93+#Ed_5ZCN`Z!zUU<E63Z zOU2H)=Tun!4)UN3GvimpDY3x&^?aq}<$ep(f$cp$24>YeT7}(dCWPGYEGl*_LvCRA zZjprDl$_27bW}e{{%9xDOHAuG*3rDKdc0eIDqG56 zh5ps000`M$Rh?9?Z0VsV5=F=(LkD)ZpShx>JO`K7J){jdyI{Q&p2GzhW(kS-wz<}J z(JP&ctt*WV!b)aqI_~;rPK7tx)K!>@Q$70Pk!rP1nqr3C%EVJDNDe{-8*4wDTFf#0 zsyy&XZSyqUd)~oaypM2Ci^Zs{Y}nYggeB40ooTPO)67+z_+e&7?T&Y3okF8wMpgOH z%Sr5a*-p*j^>f~Zj<;mP{zN~q5i!*^tCsqtP|Iw-kA;H5beY)$tmTX5rb65r0wVP zc0egCllMx#p?@^b?C{KO^b{lQ+@*Zef0(*Gc>tzx5IS6E6}jZuo5#w!kC7LX>kkr* z?8a6VZhrE2s*E4&)aEcA3D0nt<5N`m+oEwk>bIUcamFP#G~An-m~P(1g?HZ7HOn+> zoa_8*Ta+-nlcqI(3?!(4U;hKHI=97db=DRLU!B5n@EA1=uno9)PoMU0|N4ZFIi=!8 z^^a=2>*k(lq@KZ0LW7SBgLHlmK)|CvD*lwvnEKN=8ZYGYGCOlTufzMX0-gQP=0gx2 zNzHiKdvJ0rr}Jl2M-{^ZOXmAk<7;=>?XQj(cO~Rm4gc69tdRTJO#AQZ+EAQ#EZ;m^=$( zXj*61|92LK=s4zAMEHt7<)Xyb>T0K_KV4y!=XJsHQ0f*B^Vw#nW$g|$U-rQDqrv9& zmdN!_M>Am;9gT-aJ{kt!# zL-hjURbE;(+I~^Jo>H6SttNP)b~#fQ>3$h?b-|0M0r?Y3+u!<--a+S}PO0DXX1Dd} zWn^q#M7tl=vH@wA3-Q%a187Up>wkz;7ZnV2@%yIh*GG9R>MWu~& zmZMXY8trgk?oOWwA64HZ1bGHPPjsQ?dt(8sA#*!C&ejyru1I7y0mkKja%o$O?sPP~ zyd0KQ?-Zp5o3Mk&B8SAuLZL!gkd&56s*-CJd!E3{?=N~{plZAuwsnc$WJK&~fXw*Q zEkPGcKICIfN!d3J&RJ?Q#joy6Gqf$)pihVW~)!yC{G%h~%Py*klbUMU9g z{ttA_4bI(y)}~L#mVB8<^Pg^UZ*&c({Qgx1ACjT*^(l8Y^N4fC`G!9p82XQsJ51?oKNd<>7Nqh?6`1r7#2u_idx$EC@$az{e^Husj{YuLzH%2gp zs^$Tg+Jt)juL(!^1PI#~OgfoinpWlFB7_>=IhB_2jZr<&Oj5 z#&LhU9^NwttC8*aa}d|M-idMcnd`TRuS>P;UXf30*Y;U4ym;O1Z@)X(Ga zylbj+xbdF0mwHxv;G?g@Arji*22vy67glRl0M-2?W%+1w;i_&q8~Nu4LE^_H>D?AD zY&U3HZ>lY$0+%P~dKM7lro_+Y_LSamh6 zxKWNzXqfM(H`-(+oO^|vojTrxVjJ)7ndGct`@auKWaxix0Up?a1gteL>KheWmQQ!P zU0i*9=Wh(yNBBB=!OKGO8b!T-)ndrk^?`-CxPKDvmutl;6+y7T|f*{3{o2an*w)&ecRM< zio&aVCM=k%j(^j|8+zBZo@DeC^~o)YYu^i^D}67N@5TFs5|Z<=g`fG#8(g=NB9(nk zxYF;LfA8?^d6h*_l`(oeJDNapXrGepp>Rwcd%a}M>!0@|G*vf8H&@9zXx^ zUb%0ax0rYamy_fEo{=@vpfbsg_%3=b!7mfp6~sS6h&7wi@3tU5#U*d-M?YG|2BE;g z?U&j!48HvR^(SNXJpN|_k?+3~cx>c{rp@J<=Sj=gd*%(lQ{25E7-tH1HR$sd?y!!4w%x)m+xCh!$oGry8eTQ$ zoGRNC998V$&A(+t{a1pw;uV(1?0_EnmHbC8{HSpTEU_1~wt*D4^=80EVwa_kM>l7%^sxM0D7K^h);z`hVX354rZ1&@p!K z_Ze33Lpg8yQnpVRsJIMaev|j?wSisU&X6=*fyds;B={q8dB4<6Aj}86Y4pQCb$;BP zFowlcKEQw1`D>3H3<)Zp(=zjj}D?eey~hi3Hd#4 zHcCbl$J?ti9(f?}8VbHl&M@VseU%Xx4Yo4F#H|?$aGmz1*r|Or|K)$TKl`%35em6j z3+EL<_`Q4JbJLGB!}Qxl89@q*iCL2Ppy;>7;p+aV(E2!g{!_VY)BCDho{%Op7%Qz$ z%&&>cE0AAeCiv+`W)78JtU1D`k2#Znb%bygEuw7_s5DqB7%f{o;%13;dK_TdZv`*} zXrfPjx;W3-1WtX@bMyHwzwk4!_s<`BTh4`Qd+naSMCG;~|2wN2+&m3H#X>>OlT|Q= zna|#E2{$9{oL~DlrS~WA__FG*Y(IbV)OS1a*fOv@HFsZr{8N_|h89MAM}yYHN$)+x z@6bnsE+g{|cm`14x5ob0mUg9^ z_?MIfk&~Qob>2rC1AgITKE9X`eikA7sZ%nf40P}-W{2(M6UD4$yu}+{wqEn-?yxwV zRmk9C+T4TTxwP49wYItSGopiYK=;mAXDa-(sKHCwHv@{e>Z3jcTNzmFwLAh}@=r4F*4(!*-n<1x{aA{@4;#hZ8aDr6?8P00}zS~#c71xj~Pim9!>`NVqSYpXa(vHt10#s^zOr5N{!vrO}k4> zHuz~_=h$IHZX!?3G*Ml`(k4SXBi4fF8aT(a37Rd;@pc1+d zH0{9oEP*${u&Vp838?U5GDKYpYR9x(*Rm_knq*iH{KICB7uBjEys7fL?_LF05y27) zes5b!4w6Y8Ocxb??$uwA5JDjeGJtXRU7tgmki_KM{ub}r_skS-MQr; zUv>@#hjN26@7p$k+Dy16DeJ{}gy^cP_0po(g~(4aTXu@x6mr?4c~h9RE^k|!x`W$7 z2wU>nmqauzZ|jV1$O5u@B^m1?yYxiSvH8(Xma+%FmLo|Zu!B0q;3`F{_;}O(F2IIe z*BsPta!a*KS+8HW7qv{W{_Ya2X)?m$Y>{LY=n=qOcNafQz%dnoXxIp_ z_?y7dVrWO;a+Vj<#VD_xPFq0Myy1W2kmn$x>E*^4ajP?T)rW3%dZLzY_NQ5GAp5LE zJJ;UbY5ZqewoU)s82<&>-S|(ay7`~a@-*?^XRii-`2hw`KfD>d{jb+AFpRpCie6CA9Q5$ryJxT92Roa>@1rSn_3z$g35=iKy-Sth zSssPfq^Xjrf?uu|7uCgGOiLI8;ySKnmSsENy?b5HrN4XEELwc`?xTV1Z|XVx^5ou5 ziQlRI&xW%>F}L<-yI}%u(0{>{a_4_l7ytb{Pow|WjJsY|bCN;#KP-mz(ncm!KkCb? z<9klBi)0RN%jpEePb2VM=FNg%dA$PlBm-s5o0XBWA7B4G9f9ARkEYF18m`w5;Dxu7 zYPy)$JCIIzf9-zYlk74ZHT5vm(d#^56=q0I{Z6mFD4>r_7X^+jmg{4IK7IWf>RPQ( zJyCtF*0P=ttHpd&7kgFXH7OU1aWX09Lp#Yw4W(;m8=VaENwQr3eEs8(Lva75qxTJ! zys!FKi{ehkNi`cyD_n_MXN%WRyjtxuxIU$M^U&gx@eB&Mpn$D*aygrwPS$k0dCS); zbPo`qd+RrooL<-@AK9{OPum<0wVeJF>k!X|qtW!k5ce7G^V>vS}U#q6T%Z9}w zT7R`VNNe+msgt$4`J6lJPBvF3Z!X=1m&o=d#lXWglJNhlR~z@jIJc#ol*5lpE}HdUFRyqH zEx#@boaY64j*U>qYdJvNK>%e_J=TuD|Ju!^-s|VC)*eu2;IA*9K7aM1KhAIJ|E1vW z`Y&$GZ^!?C@FkzO7%H(Jfz~0)0ERm1?GXvilq}E)))9%nUr`%8n~p{{tA_7w@Ct+_ zNEX$!S$+`|^JHv4Oy@rae|YxlkN^0?YUK#@r5B472E60PSH0QeY^=tk>8$)?zm6b1 zyRg}h!QY>~3SNEp8juU}X;xkrf#0sr^sg78TtWNCI&wG0U|epiWH>7Of2s%OzYG2X zmN%Gzy~6-}T~F}Cn_#)__AL0%rL=>8Bbwp!Y2AGZCRKF$<~0tUgJ-Y*cAewttLH`q zFO%_bbQuV}oWSP6o662+h!*b!PZt0gC`$&3v4Cb|aa~3;!U+C=i-{kb1%IfC`QtyZjy3+gx~{2qyp~YF z?|)rP0!)@na{Td)wKd?&{V@#>&lkho?uPs0WyZ%13;+4gf2K)!@#rC5fUtZOoc=AS z-9Su`jf43(sK;HwX|eoewt!SPmpmwUhIIUkOKWdYYkNOc&J?BT~cStl&CL zHDj5H3dTaoQdJaEK(UF;Y>~26RnXXG33l-RJ$m>*PEQf;C-CcP4f^#l>w|0EgwGe# z`6^5OdsW%_lWTmNHy0lUxIXI1waHw*s8p5{+RITuv9khop&nE|v>6{*ImoXyBTS znA>bPvwLaw(jb6MSu(WzBl;{@;WF4(B2t%7He%Q=NFTrkffNNRajK(oFrG*zs2Pn= zCOD6bK52ROrwxg2B-bmYI|y={^w%KPy7^aILRVU^Pba)A-vv*zY%x!=%T>(*P?KcYKhJ`HhTuj1vj;DL)pLwqu96}vveC4J-}2%6y3pXC z|L>Ck+))#2_0Lt=uh*6=tQG&JnIXm%4a*fdSQ`)fIy9L1LHB%t=hOLrS26CJ*Z*?q z#(#xmo&Wba9x}s3rwHgIRPvEdvg|ZjPOZ2+bM0FX6a0RR91 diff --git a/summary.csv b/summary.csv deleted file mode 100644 index 3a66ce11eb..0000000000 --- a/summary.csv +++ /dev/null @@ -1,2 +0,0 @@ -Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.5764,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json deleted file mode 100644 index 786ef94185..0000000000 --- a/summary.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "0":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows", - "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.5764, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v2.3.9. ", - "compliance":1, - "errors":0, - "version":"v4.1", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - } -} \ No newline at end of file diff --git a/tmp-run.bat b/tmp-run.bat deleted file mode 100644 index 02f4e7a468..0000000000 --- a/tmp-run.bat +++ /dev/null @@ -1,36 +0,0 @@ -@echo off - -set PATH=C:\Users\runneradmin\CM\repos\local\cache\b461cd87e6bb43b4\bin;C:\hostedtoolcache\windows\Python\3.12.6\x64\Scripts;%PATH% -set CM_GIT_BRANCH=main -set CM_GIT_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\8bb62b59178d4feb\repo -set CM_GIT_REPO_CHECKOUT=main -set CM_GIT_REPO_CHECKOUT_FOLDER=repo -set CM_GIT_REPO_CHECKOUT_PATH=C:\Users\runneradmin\CM\repos\local\cache\8bb62b59178d4feb\repo -set CM_GIT_REPO_DEPTH=--depth 5 -set CM_GIT_REPO_NAME=mlperf_inference_test_submissions_v5.0 -set CM_GIT_REPO_PATCH=no -set CM_GIT_REPO_RECURSE_SUBMODULES= --recurse-submodules -set CM_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 -set CM_MLPERF_INFERENCE_SUBMISSION_DIR=C:\Users\runneradmin\CM\repos\local\cache\eae25a88b2b14eb7\mlperf-inference-submission -set CM_MLPERF_INFERENCE_SUBMISSION_VERSION=4_1-dev -set CM_MLPERF_RESULTS_GIT_REPO_URL=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 -set CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE=Results from R50 GH action on windows-latest -set CM_PYTHON_BIN=python.exe -set CM_PYTHON_BIN_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64 -set CM_PYTHON_BIN_WITH_PATH=C:\hostedtoolcache\windows\Python\3.12.6\x64\python.exe -set CM_PYTHON_CACHE_TAGS=version-3.12.6,non-virtual -set CM_PYTHON_MAJOR_VERSION=3 -set CM_PYTHON_MINOR_VERSION=12 -set CM_PYTHON_PATCH_VERSION=6 -set CM_PYTHON_VERSION=3.12.6 -set CM_QUIET=yes -set CM_TMP_CURRENT_PATH=D:\a\cm4mlops\cm4mlops -set CM_TMP_CURRENT_SCRIPT_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github -set CM_TMP_CURRENT_SCRIPT_REPO_PATH=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops -set CM_TMP_CURRENT_SCRIPT_REPO_PATH_WITH_PREFIX=C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops -set CM_TMP_CURRENT_SCRIPT_WORK_PATH=D:\a\cm4mlops\cm4mlops -set CM_TMP_PIP_VERSION_STRING= -set CM_WINDOWS=yes - - -call "C:\Users\runneradmin\CM\repos\anandhu-eng@cm4mlops\script\push-mlperf-inference-results-to-github\run.bat" diff --git a/tmp-state.json b/tmp-state.json deleted file mode 100644 index e03109d44a..0000000000 --- a/tmp-state.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "docker": {} -} diff --git a/version_info.json b/version_info.json deleted file mode 100644 index a15a67dbda..0000000000 --- a/version_info.json +++ /dev/null @@ -1,602 +0,0 @@ -[ - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,src": { - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlperf,inference,utils": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_uid": "e341e5f86d8342e5", - "script_variations": "", - "version": "" - } - }, - { - "get,dataset-aux,imagenet-aux": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-sys-utils-cm,bc90993277e84b8e", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_onnxruntime": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "onnxruntime", - "version": "1.19.2" - } - }, - { - "get,ml-model,image-classification,resnet50,raw,_onnx,_fp32": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-ml-model-resnet50", - "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification", - "script_uid": "56203e4e998b4bc0", - "script_variations": "onnx,fp32", - "version": "" - } - }, - { - "get,dataset,image-classification,imagenet,preprocessed,_NCHW": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-preprocessed-dataset-imagenet", - "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed", - "script_uid": "f259d490bbaf45f5", - "script_variations": "NCHW", - "version": "" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "" - } - }, - { - "get,sut,configs": { - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_uid": "c2fbf72009e2445b", - "script_variations": "", - "version": "" - } - }, - { - "generate,user-conf,mlperf,inference": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_uid": "3af4475745964b93", - "script_variations": "", - "version": "" - } - }, - { - "get,loadgen,_from-pip": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_uid": "64c3d98d0ba04950", - "script_variations": "from-pip", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.psutil", - "version": "6.0.0" - } - }, - { - "get,generic-python-lib,_opencv-python": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "opencv-python", - "version": "4.10.0.84" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "get,generic-python-lib,_pycocotools": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "pycocotools", - "version": "2.0.8" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "benchmark-program,19f369ef47084895", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-program,program": { - "parent": "benchmark-program-mlperf,cfff0132a8aa4018", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_uid": "19f369ef47084895", - "script_variations": "", - "version": "" - } - }, - { - "benchmark-mlperf": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_uid": "cfff0132a8aa4018", - "script_variations": "", - "version": "" - } - }, - { - "save,mlperf,inference,state": { - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_offline,_onnxruntime,_fp32 )", - "script_alias": "save-mlperf-inference-implementation-state", - "script_tags": "save,mlperf,inference,implementation,state", - "script_uid": "b14b813229c444f8", - "script_variations": "", - "version": "" - } - }, - { - "app,mlperf,reference,inference,_cpu,_resnet50,_offline,_onnxruntime,_fp32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "app-mlperf-inference-mlcommons-python", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,reference,ref", - "script_uid": "ff149e9781fc4b65", - "script_variations": "cpu,resnet50,offline,onnxruntime,fp32", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "get,sys-utils-min": { - "parent": "detect-os,863735b7db8c44fc", - "script_alias": "get-sys-utils-min", - "script_tags": "get,sys-utils-min", - "script_uid": "a9af7714d3d94779", - "script_variations": "", - "version": "" - } - }, - { - "detect,os": { - "parent": "detect-cpu,586c8a43320142f7", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_uid": "863735b7db8c44fc", - "script_variations": "", - "version": "" - } - }, - { - "detect,cpu": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_uid": "586c8a43320142f7", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,generic-python-lib,_package.dmiparser": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "package.dmiparser", - "version": "5.1" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-descriptions": { - "parent": "get-mlperf-inference-sut-description,e49a3f758b2d4e7b", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_uid": "48f4622e059b45ce", - "script_variations": "name.mlperf-inference-sut-descriptions", - "version": "" - } - }, - { - "get,mlperf,sut,description": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "get-mlperf-inference-sut-description", - "script_tags": "get,mlperf,sut,description,system-under-test,system-description", - "script_uid": "e49a3f758b2d4e7b", - "script_variations": "", - "version": "" - } - }, - { - "get,python3": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_uid": "d0b5dd74373f4a62", - "script_variations": "", - "version": "3.12.6" - } - }, - { - "get,mlcommons,inference,src": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_uid": "4b57186581024797", - "script_variations": "", - "version": "master" - } - }, - { - "get,dataset-aux,image-classification,imagenet-aux": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-dataset-imagenet-aux", - "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux", - "script_uid": "bb2c6dd8c8c64217", - "script_variations": "", - "version": "" - } - }, - { - "get,generic-python-lib,_numpy": { - "parent": "process-mlperf-accuracy,6e809013816b42ea ( imagenet,_float32 )", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,generic-python-lib", - "script_uid": "94b62a682bc44791", - "script_variations": "numpy", - "version": "1.26.4" - } - }, - { - "run,accuracy,mlperf,_imagenet,_float32": { - "parent": "app-mlperf-inference,d775cac873ee4231 ( python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )", - "script_alias": "process-mlperf-accuracy", - "script_tags": "run,mlperf,mlcommons,accuracy,mlc,process,process-accuracy", - "script_uid": "6e809013816b42ea", - "script_variations": "imagenet,float32", - "version": "" - } - }, - { - "app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline": { - "parent": null, - "script_alias": "app-mlperf-inference", - "script_tags": "app,vision,language,mlcommons,mlperf,inference,generic", - "script_uid": "d775cac873ee4231", - "script_variations": "python,resnet50,onnxruntime,cpu,test,r4.1-dev_default,offline", - "version": "" - } - } -] From 42f1444d23c7f84ae4b6947b666f0c996ed0c743 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 01:45:32 +0530 Subject: [PATCH 027/658] Added an error check for push-mlperf-inference-results to avoid push to unintended repos --- script/get-git-repo/run.bat | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/script/get-git-repo/run.bat b/script/get-git-repo/run.bat index 583d324ee2..d00f32b15d 100644 --- a/script/get-git-repo/run.bat +++ b/script/get-git-repo/run.bat @@ -6,7 +6,7 @@ rem echo Cloning MLCommons from %CM_GIT_URL% with branch %CM_GIT_CHECKOUT% %CM_G rem git clone %CM_GIT_RECURSE_SUBMODULES% %CM_GIT_URL% %CM_GIT_DEPTH% inference rem cd inference rem git checkout -b "%CM_GIT_CHECKOUT%" -rem +rem rem Next line allows ERRORLEVEL inside if statements! setlocal enabledelayedexpansion @@ -18,9 +18,10 @@ set folder=%CM_GIT_CHECKOUT_FOLDER% if not exist "%CM_TMP_GIT_PATH%" ( - if exist %folder% ( - deltree %folder% + if exist "%folder%" ( + rmdir /S /Q "%folder%" rem Use rmdir instead of deltree ) + echo ****************************************************** echo Current directory: %CUR_DIR% echo. @@ -28,9 +29,12 @@ if not exist "%CM_TMP_GIT_PATH%" ( echo. echo "%CM_GIT_CLONE_CMD%" echo. + %CM_GIT_CLONE_CMD% IF !ERRORLEVEL! NEQ 0 EXIT !ERRORLEVEL! - cd %folder% + + cd "%folder%" + if not "%CM_GIT_SHA%" == "" ( echo. echo. @@ -39,9 +43,7 @@ if not exist "%CM_TMP_GIT_PATH%" ( ) ) else ( - - cd %folder% - + cd "%folder%" ) if not "%CM_GIT_SUBMODULES%" == "" ( @@ -62,6 +64,7 @@ if "%CM_GIT_PATCH%" == "yes" ( ) ) -cd %CUR_DIR% +cd "%CUR_DIR%" exit /b 0 + From c4a0c413b65be56b67eebdb2f062b5e51d2018a2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 01:58:17 +0530 Subject: [PATCH 028/658] Added an error check for push-mlperf-inference-results to avoid push to unintended repos --- script/push-mlperf-inference-results-to-github/run.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index 9c4dd0fd18..ebf497a8bf 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -6,12 +6,15 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH ( exit /b 1 ) +echo %cd% +echo "Doing cd" REM Change directory cd "%CM_GIT_REPO_CHECKOUT_PATH%" || ( echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" exit /b 1 ) echo %cd% +echo "%CM_GIT_REPO_CHECKOUT_PATH%" git pull git add * From 32f5bf8e287692f805ed2e7a713f851c6912d2e1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Oct 2024 23:33:30 +0100 Subject: [PATCH 029/658] Update customize.py --- script/benchmark-program/customize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/benchmark-program/customize.py b/script/benchmark-program/customize.py index af8c73a323..422d82c482 100644 --- a/script/benchmark-program/customize.py +++ b/script/benchmark-program/customize.py @@ -45,7 +45,7 @@ def preprocess(i): if os_info['platform'] != 'windows' and str(env.get('CM_SAVE_CONSOLE_LOG', True)).lower() not in [ "no", "false", "0"]: logs_dir = env.get('CM_LOGS_DIR', env['CM_RUN_DIR']) - env['CM_RUN_CMD'] += " 2>&1 ; echo \$? > exitstatus | tee " + q+ os.path.join(logs_dir, "console.out") + q + env['CM_RUN_CMD'] += r" 2>&1 ; echo \$? > exitstatus | tee " + q+ os.path.join(logs_dir, "console.out") + q # additional arguments and tags for measuring system informations(only if 'CM_PROFILE_NVIDIA_POWER' is 'on') if env.get('CM_PROFILE_NVIDIA_POWER', '') == "on": @@ -73,7 +73,7 @@ def preprocess(i): # running the script as a process in background pre_run_cmd = pre_run_cmd + 'cm run script --tags=runtime,system,utilisation' + env['CM_SYS_UTILISATION_SCRIPT_TAGS'] + ' --quiet & ' # obtain the command if of the background process - pre_run_cmd += ' cmd_pid=\$!' + ' && ' + 'echo CMD_PID=\$cmd_pid' + pre_run_cmd += r" cmd_pid=\$! && echo CMD_PID=\$cmd_pid" print(f"Pre run command for recording the runtime system information: {pre_run_cmd}") env['CM_PRE_RUN_CMD'] = pre_run_cmd @@ -81,7 +81,7 @@ def preprocess(i): # generate the post run cmd - for killing the process that records runtime system infos post_run_cmd = "" if env.get('CM_PROFILE_NVIDIA_POWER', '') == "on": - post_run_cmd += "echo killing process \$cmd_pid && kill -TERM \${cmd_pid}" + post_run_cmd += r"echo killing process \$cmd_pid && kill -TERM \${cmd_pid}" print(f"Post run command for killing the process that measures the runtime system information: {post_run_cmd}") env['CM_POST_RUN_CMD'] = post_run_cmd From 9a1215a3ee641552dc8fa3be3fb9cf07e40c8a63 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Oct 2024 23:40:34 +0100 Subject: [PATCH 030/658] Improve benchmark-program-mlperf --- script/benchmark-program-mlperf/customize.py | 39 +++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/script/benchmark-program-mlperf/customize.py b/script/benchmark-program-mlperf/customize.py index a333b6c078..7d109051d9 100644 --- a/script/benchmark-program-mlperf/customize.py +++ b/script/benchmark-program-mlperf/customize.py @@ -16,13 +16,40 @@ def postprocess(i): if env.get('CM_MLPERF_POWER', '') == "yes": - if os_info['platform'] == 'windows': - return {'return':1, 'error':'TBD: this script is not yet supported on Windows'} - + if env.get('CM_MLPERF_SHORT_RANGING_RUN', '') != 'no': - os.system("echo '0' > "+env.get('CM_RUN_DIR','')+ "/count.txt") - env['CM_MLPERF_RUN_CMD'] = "CM_MLPERF_RUN_COUNT=\$(cat \${CM_RUN_DIR}/count.txt); echo \${CM_MLPERF_RUN_COUNT}; CM_MLPERF_RUN_COUNT=\$((CM_MLPERF_RUN_COUNT+1)); echo \${CM_MLPERF_RUN_COUNT} > \${CM_RUN_DIR}/count.txt && if [ \${CM_MLPERF_RUN_COUNT} -eq \'1\' ]; then export CM_MLPERF_USER_CONF=\${CM_MLPERF_RANGING_USER_CONF}; else export CM_MLPERF_USER_CONF=\${CM_MLPERF_TESTING_USER_CONF}; fi && "+env.get('CM_RUN_CMD','').strip() + # Write '0' to the count.txt file in CM_RUN_DIR + count_file = os.path.join(env.get('CM_RUN_DIR', ''), 'count.txt') + with open(count_file, 'w') as f: + f.write('0') + + if os_info['platform'] != 'windows': + # Construct the shell command with proper escaping + env['CM_MLPERF_RUN_CMD'] = ( + "CM_MLPERF_RUN_COUNT=$(cat ${CM_RUN_DIR}/count.txt); " + "echo ${CM_MLPERF_RUN_COUNT}; " + "CM_MLPERF_RUN_COUNT=$((CM_MLPERF_RUN_COUNT+1)); " + "echo ${CM_MLPERF_RUN_COUNT} > ${CM_RUN_DIR}/count.txt && " + "if [ ${CM_MLPERF_RUN_COUNT} -eq 1 ]; then " + "export CM_MLPERF_USER_CONF=${CM_MLPERF_RANGING_USER_CONF}; " + "else " + "export CM_MLPERF_USER_CONF=${CM_MLPERF_TESTING_USER_CONF}; " + "fi && " + env.get('CM_RUN_CMD', '').strip() + ) + else: + env['CM_MLPERF_RUN_CMD'] = ( + "set /P CM_MLPERF_RUN_COUNT=<\"%CM_RUN_DIR%\\count.txt\" & " + "echo %CM_MLPERF_RUN_COUNT% & " + "set /A CM_MLPERF_RUN_COUNT=!CM_MLPERF_RUN_COUNT!+1 & " + "echo !CM_MLPERF_RUN_COUNT! > \"%CM_RUN_DIR%\\count.txt\" & " + "if !CM_MLPERF_RUN_COUNT! EQU 1 ( " + "set CM_MLPERF_USER_CONF=%CM_MLPERF_RANGING_USER_CONF% " + ") else ( " + "set CM_MLPERF_USER_CONF=%CM_MLPERF_TESTING_USER_CONF% " + ") & " + env.get('CM_RUN_CMD', '').strip() + ) else: - env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD','').strip() + # Just use the existing CM_RUN_CMD if no ranging run is needed + env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD', '').strip() return {'return':0} From 14f2901d38e4daf030cbd711703ddff99028b37b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 5 Oct 2024 10:34:55 +0530 Subject: [PATCH 031/658] enabled for windows + push to github added --- .../test-mlperf-inference-retinanet.yml | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index 61f9d30cd2..54d6198e15 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -13,11 +13,11 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, windows-latest, macos-latest] python-version: [ "3.12" ] backend: [ "onnxruntime", "pytorch" ] implementation: [ "python", "cpp" ] @@ -35,6 +35,27 @@ jobs: run: | python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} + - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=retinanet --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --quiet -v --target_qps=1 + - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os != 'windows-latest' run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --hw_name=default --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=retinanet --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.compiler.tags=gcc --quiet -v --target_qps=1 + - name: Push Results + if: github.repository_owner == 'gateoverflow' + env: + USER: "GitHub Action" + EMAIL: "admin@gateoverflow.com" + run: | + git config --global user.name "${{ env.USER }}" + git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + + cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet \ No newline at end of file From f19ce8ab5053e61696839f7060a14229e5a9260a Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 5 Oct 2024 15:27:30 +0530 Subject: [PATCH 032/658] gh action update: enabled windows run + push results to github --- ...bert-deepsparse-tf-onnxruntime-pytorch.yml | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index a6e6b86ef7..4b714965a3 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -13,8 +13,7 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: @@ -22,6 +21,7 @@ jobs: python-version: [ "3.11" ] backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ] precision: [ "int8", "fp32" ] + os: [ubuntu-latest, windows-latest, macos-latest] exclude: - backend: tf - backend: pytorch @@ -38,6 +38,26 @@ jobs: run: | python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - - name: Test MLPerf Inference Bert (DeepSparse, TF, ONNX, PyTorch) + - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os == 'windows-latest' + run: | + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --precision=${{ matrix.precision }} --target_qps=1 -v --quiet + - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} + if: matrix.os != 'windows-latest' run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 -v --quiet + - name: Push Results + if: github.repository_owner == 'gateoverflow' + env: + USER: "GitHub Action" + EMAIL: "admin@gateoverflow.com" + run: | + git config --global user.name "${{ env.USER }}" + git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + + cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet \ No newline at end of file From 7567c3fd8922b085e6d8302f4140dd09beaa321b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 5 Oct 2024 15:35:23 +0530 Subject: [PATCH 033/658] code clean --- .github/workflows/test-qaic-software-kit.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-qaic-software-kit.yml b/.github/workflows/test-qaic-software-kit.yml index e3a186daae..4b877da008 100644 --- a/.github/workflows/test-qaic-software-kit.yml +++ b/.github/workflows/test-qaic-software-kit.yml @@ -38,4 +38,3 @@ jobs: - name: Test Software Kit for compilation on Ubuntu 20.04 run: | cm run script --tags=get,qaic,software,kit --adr.compiler.tags=${{ matrix.compiler }} --adr.compiler.version=${{ matrix.llvm-version }} --quiet - cm run script --tags=get,qaic,software,kit --adr.compiler.tags=${{ matrix.compiler }} --adr.compiler.version=${{ matrix.llvm-version }} --quiet From 101591d83582bd8d0926980b0eb8354ef81662f3 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 5 Oct 2024 15:50:20 +0530 Subject: [PATCH 034/658] updated gh action for resnet50 cpp --- ...lperf-inference-mlcommons-cpp-resnet50.yml | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index f726b8ecc5..f82e00750e 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -13,16 +13,16 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: [ "3.12" ] llvm-version: [ "15.0.6", "16.0.4", "17.0.6" ] + os: [ubuntu-latest, windows-latest, macos-latest] exclude: - - llvm-version: "15.0.6" - - llvm-version: "16.0.4" + - llvm-version: "15.0.6" + - llvm-version: "16.0.4" steps: - uses: actions/checkout@v3 @@ -36,6 +36,26 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm run script --quiet --tags=get,sys-utils-cm cm run script --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }} - - name: Test MLPerf Inference MLCommons C++ ResNet50 + - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }} + if: matrix.os == 'windows-latest' + run: | + cmr "app mlperf inference mlcommons cpp" --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet + - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }} + if: matrix.os != 'windows-latest' run: | - cmr "app mlperf inference mlcommons cpp" -v --quiet + cmr "app mlperf inference mlcommons cpp" --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} -v --quiet + - name: Push Results + if: github.repository_owner == 'gateoverflow' + env: + USER: "GitHub Action" + EMAIL: "admin@gateoverflow.com" + run: | + git config --global user.name "${{ env.USER }}" + git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + + cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from MLCommons C++ ResNet50 GH action on ${{ matrix.os }}" --quiet From 6affd71bfbc54c94c2f33f58e3a8ae6ad1ea635e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 22:37:43 +0530 Subject: [PATCH 035/658] Fixes for dropbox files getting blocked (imagenet and windows packages) --- script/detect-os/customize.py | 6 ++++++ script/get-dataset-imagenet-val/_cm.yaml | 4 ++-- script/get-generic-sys-util/_cm.json | 6 ++++-- script/get-sys-utils-min/_cm.yaml | 12 +++++++++--- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/script/detect-os/customize.py b/script/detect-os/customize.py index 2f8dd4c76d..d59a511f6a 100644 --- a/script/detect-os/customize.py +++ b/script/detect-os/customize.py @@ -55,6 +55,9 @@ def postprocess(i): env['CM_HOST_OS_MACHINE'] = state['os_uname_machine'] + else: + env['CM_HOST_OS_PACKAGE_MANAGER'] = "choco" + import platform env['CM_HOST_SYSTEM_NAME'] = platform.node() @@ -93,6 +96,9 @@ def postprocess(i): elif env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "zypper": env['CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD'] = "zypper install -y" env['CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD'] = "zypper update -y" + elif env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "choco": + env['CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD'] = "choco install -y" + env['CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD'] = "choco upgrade -y" if os.path.exists("/.dockerenv"): env['CM_RUN_INSIDE_DOCKER'] = "yes" diff --git a/script/get-dataset-imagenet-val/_cm.yaml b/script/get-dataset-imagenet-val/_cm.yaml index d21cdad2a5..0b9923927b 100644 --- a/script/get-dataset-imagenet-val/_cm.yaml +++ b/script/get-dataset-imagenet-val/_cm.yaml @@ -93,8 +93,8 @@ variations: default: true env: CM_DAE_FILENAME: ILSVRC2012_img_val_500.tar -# CM_DAE_URL: http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar - CM_DAE_URL: https://www.dropbox.com/scl/fi/a7fhjnzxi6x3ceapxh5bm/ILSVRC2012_img_val_500.tar?rlkey=hz4rabo9ve43co3c303y9r6l7&st=ulcgb3av&dl=1 + CM_DAE_URL: http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar + CM_DOWNLOAD_URL1: https://www.dropbox.com/scl/fi/a7fhjnzxi6x3ceapxh5bm/ILSVRC2012_img_val_500.tar?rlkey=hz4rabo9ve43co3c303y9r6l7&st=ulcgb3av&dl=1 CM_DATASET_SIZE: '500' CM_DOWNLOAD_CHECKSUM: 8627befdd8c2bcf305729020e9db354e CM_DOWNLOAD_FILENAME: ILSVRC2012_img_val_500.tar diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 90fba4fc98..bb2223f855 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -203,7 +203,8 @@ "brew": "bzip2", "dnf": "libbzip2-devel", "yum": "libbzip2-devel", - "zlib-devel": "libbz2-devel" + "zlib-devel": "libbz2-devel", + "choco": "zlib" } } }, @@ -676,7 +677,8 @@ "apt": "xz-utils", "brew": "xz", "dnf": "xz", - "yum": "xz" + "yum": "xz", + "choco": "xz" } } }, diff --git a/script/get-sys-utils-min/_cm.yaml b/script/get-sys-utils-min/_cm.yaml index 8f623e2c1c..6623d9a79d 100644 --- a/script/get-sys-utils-min/_cm.yaml +++ b/script/get-sys-utils-min/_cm.yaml @@ -8,13 +8,19 @@ cache: true category: Detection or installation of tools and artifacts -deps: [] +deps: + - tags: get,generic,sys-util,_xz + enable_if_env: + CM_HOST_OS_TYPE: + - windows + - tags: get,generic,sys-util,_zlib + enable_if_env: + CM_HOST_OS_TYPE: + - windows env: CM_CLEAN_DIRS: bin CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 - ; https://www.dropbox.com/scl/fi/cj7xmgmk4wu1r0iuoedcq/zlib123dllx64-bin.zip?rlkey=sqmldotiste2zhk6nhmijg09e&st=i5w7esjx&dl=1 - ; https://www.dropbox.com/scl/fi/x94qvrmac55ht03257139/xz-5.2.9-win64.zip?rlkey=adzixlsqrapncrusy9ndd2qji&st=tqobzlfo&dl=1 CM_SUDO: sudo new_env_keys: From 4adfb0bd4cef7c132a51b5e24c93be4367877ae0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 22:50:40 +0530 Subject: [PATCH 036/658] Fixed recursion bug in get-sys-utils-min --- script/detect-os/_cm.json | 5 +++++ script/get-generic-sys-util/customize.py | 13 ++++++------- script/get-sys-utils-min/_cm.yaml | 2 ++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/script/detect-os/_cm.json b/script/detect-os/_cm.json index 383b7f9302..7d64c9397b 100644 --- a/script/detect-os/_cm.json +++ b/script/detect-os/_cm.json @@ -25,6 +25,11 @@ "windows" ] }, + "skip_if_env": { + "CM_WINDOWS_SYS_UTILS_MIN_INSTALL": [ + "yes" + ] + }, "tags": "get,sys-utils-min" } ], diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 648f139159..3263acf4f0 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -29,8 +29,13 @@ def preprocess(i): # Only "install" mode reaches here pm = env.get('CM_HOST_OS_PACKAGE_MANAGER') + util = env.get('CM_SYS_UTIL_NAME', '') + if util == '': + return {'return': 1, 'error': 'Please select a variation specifying the sys util name'} - if os_info['platform'] == 'windows': + package = state.get(util) + + if os_info['platform'] == 'windows' and not package: print ('') print ('WARNING: for now skipping get-generic-sys-util on Windows ...') print ('') @@ -40,12 +45,6 @@ def preprocess(i): if not pm: return {'return': 1, 'error': 'Package manager not detected for the given OS'} - util = env.get('CM_SYS_UTIL_NAME', '') - if util == '': - return {'return': 1, 'error': 'Please select a variation specifying the sys util name'} - - - package = state.get(util) if not package: return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} diff --git a/script/get-sys-utils-min/_cm.yaml b/script/get-sys-utils-min/_cm.yaml index 6623d9a79d..c07f46eb57 100644 --- a/script/get-sys-utils-min/_cm.yaml +++ b/script/get-sys-utils-min/_cm.yaml @@ -9,6 +9,7 @@ cache: true category: Detection or installation of tools and artifacts deps: + - tags: detect,os - tags: get,generic,sys-util,_xz enable_if_env: CM_HOST_OS_TYPE: @@ -20,6 +21,7 @@ deps: env: CM_CLEAN_DIRS: bin + CM_WINDOWS_SYS_UTILS_MIN_INSTALL: yes CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 CM_SUDO: sudo From 8364936a58f7ff1c8c218fc93e267a4c92d16a11 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 22:56:01 +0530 Subject: [PATCH 037/658] Fixed recursion bug in get-sys-utils-min --- script/get-generic-sys-util/_cm.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index bb2223f855..94abbf371d 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -203,8 +203,7 @@ "brew": "bzip2", "dnf": "libbzip2-devel", "yum": "libbzip2-devel", - "zlib-devel": "libbz2-devel", - "choco": "zlib" + "zlib-devel": "libbz2-devel" } } }, @@ -688,7 +687,8 @@ }, "state": { "zlib": { - "apt": "zlib1g" + "apt": "zlib1g", + "choco": "zlib" } } }, From 1f27d0a5130b5afdeb5d00e715d618f3206e48d9 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sat, 5 Oct 2024 23:06:23 +0530 Subject: [PATCH 038/658] Commit to support custom git clone branch in docker --- automation/script/module_misc.py | 3 +++ script/app-mlperf-inference/_cm.yaml | 2 +- script/build-dockerfile/_cm.yaml | 2 ++ script/build-dockerfile/customize.py | 9 +++------ 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index b68d90e279..54ab2dd489 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1470,6 +1470,8 @@ def dockerfile(i): run_cmd = r['run_cmd_string'] cm_repo = i.get('docker_cm_repo', docker_settings.get('cm_repo', 'mlcommons@cm4mlops')) + cm_repo_branch = i.get('docker_cm_repo_branch', docker_settings.get('cm_repo_branch', 'mlperf-inference')) + cm_repo_flags = i.get('docker_cm_repo_flags', docker_settings.get('cm_repo_flags', '')) docker_base_image = i.get('docker_base_image', docker_settings.get('base_image')) @@ -1542,6 +1544,7 @@ def dockerfile(i): 'automation': 'script', 'tags': 'build,dockerfile', 'cm_repo': cm_repo, + 'cm_repo_branch': cm_repo_branch, 'cm_repo_flags': cm_repo_flags, 'docker_base_image': docker_base_image, 'docker_os': docker_os, diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 2b8186c886..36211180bb 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -1611,7 +1611,7 @@ docker: extra_run_args: ' --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined' os: ubuntu cm_repo: mlcommons@cm4mlops - cm_repo_flags: ' --branch=mlperf-inference ' + cm_repo_branch: mlperf-inference real_run: False os_version: '22.04' docker_input_mapping: diff --git a/script/build-dockerfile/_cm.yaml b/script/build-dockerfile/_cm.yaml index da00f24de8..7706597ee5 100644 --- a/script/build-dockerfile/_cm.yaml +++ b/script/build-dockerfile/_cm.yaml @@ -19,6 +19,7 @@ default_env: ' CM_DOCKER_OS: ubuntu CM_DOCKER_NOT_PULL_UPDATE: False + CM_MLOPS_REPO_BRANCH: mlperf-inference input_mapping: build: CM_BUILD_DOCKER_IMAGE @@ -26,6 +27,7 @@ input_mapping: cm_repo: CM_MLOPS_REPO cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO cm_repos: CM_DOCKER_EXTRA_CM_REPOS + cm_repo_branch: CM_MLOPS_REPO_BRANCH comments: CM_DOCKER_RUN_COMMENTS copy_files: CM_DOCKER_COPY_FILES docker_base_image: CM_DOCKER_IMAGE_BASE diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 896454fb93..2a8c96b635 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -69,11 +69,8 @@ def preprocess(i): else: cm_mlops_repo = "mlcommons@cm4mlops" - if env.get("CM_MLOPS_REPO_BRANCH", '') != '': - cm_mlops_repo_branch_string = f" --branch {env['CM_MLOPS_REPO_BRANCH']}" - else: - cm_mlops_repo_branch_string = "" - + cm_mlops_repo_branch_string = f" --branch={env['CM_MLOPS_REPO_BRANCH']}" + if env.get('CM_DOCKERFILE_WITH_PATH', '') == '': env['CM_DOCKERFILE_WITH_PATH'] = os.path.join(os.getcwd(), "Dockerfile") @@ -206,7 +203,7 @@ def preprocess(i): x = env.get('CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO','') if x!='': x=' '+x - f.write('RUN cm pull repo ' + cm_mlops_repo + x + EOL) + f.write('RUN cm pull repo ' + cm_mlops_repo + cm_mlops_repo_branch_string + x + EOL) # Check extra repositories x = env.get('CM_DOCKER_EXTRA_CM_REPOS','') From c86c23d3f046169594e64bacbfdeda29b43244ae Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 23:17:03 +0530 Subject: [PATCH 039/658] Fixed recursion bug in get-sys-utils-min --- script/get-generic-sys-util/customize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 3263acf4f0..d74fc58669 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -34,8 +34,11 @@ def preprocess(i): return {'return': 1, 'error': 'Please select a variation specifying the sys util name'} package = state.get(util) + package_name = None + if package and pm: + package_name = package.get(pm) - if os_info['platform'] == 'windows' and not package: + if os_info['platform'] == 'windows' and not package_name: print ('') print ('WARNING: for now skipping get-generic-sys-util on Windows ...') print ('') @@ -49,7 +52,6 @@ def preprocess(i): if not package: return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} - package_name = package.get(pm) if not package_name: return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} From 4a451c20d5f534c7136206065163b84c880f2255 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:20:24 +0530 Subject: [PATCH 040/658] initial commit for enabling download to host - mixtral --- .../_cm.yaml | 6 ++++++ .../customize.py | 4 ++-- script/app-mlperf-inference/_cm.yaml | 21 +++++++++++++++++++ script/get-dataset-mixtral/customize.py | 18 ++++++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 script/get-dataset-mixtral/customize.py diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index b648dc54e0..5d392b5e68 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -434,6 +434,9 @@ deps: skip_if_env: CM_MLPERF_CUSTOM_MODEL_PATH: - "on" + skip_if_env: + CM_MLPERF_MODEL_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + - 'yes' ## 3d-unet - tags: get,ml-model,medical-imaging,3d-unet @@ -555,6 +558,9 @@ deps: enable_if_env: CM_MODEL: - mixtral-8x7b + skip_if_env: + CM_MLPERF_DATASET_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + - 'yes' ## Kits19 for 3d-unet - tags: get,dataset,kits19,preprocessed diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index ca97bbf0d0..0c2646df5e 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -332,13 +332,13 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio device = env['CM_MLPERF_DEVICE'] if env['CM_MLPERF_DEVICE'] != "gpu" else "cuda" cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " main.py " \ " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + \ - " --dataset-path " + env['CM_DATASET_PREPROCESSED_PATH'] + \ + " --dataset-path " + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] + \ " --device " + device.replace("cuda", "cuda:0") + \ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ scenario_extra_options + mode_extra_options + \ " --output-log-dir " + env['CM_MLPERF_OUTPUT_DIR'] + \ ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \ - " --model-path " + env['MODEL_DIR'] + " --model-path " + env['MIXTRAL_CHECKPOINT_PATH'] cmd = cmd.replace("--count", "--total-sample-count") elif "3d-unet" in env['CM_MODEL']: diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 2b8186c886..6fa0135088 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -799,6 +799,27 @@ variations: - openorca-gsm8k-mbxp-combined-accuracy-script tags: run,accuracy,mlperf,_openorca-gsm8k-mbxp,_int32 + mixtral-8x7b,reference: + - tags: get,ml-model,mixtral + names: + - ml-model + - mixtral-model + enable_if_env: + CM_MLPERF_MODEL_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + - 'yes' + docker: + mounts: + - "${{ MIXTRAL_CHECKPOINT_PATH }}:${{ MIXTRAL_CHECKPOINT_PATH }}" + - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined + names: + - openorca-mbxp-gsm8k-combined-preprocessed + enable_if_env: + CM_MLPERF_DATASET_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + - 'yes' + docker: + mounts: + - "${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}:${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}" + rnnt: group: model diff --git a/script/get-dataset-mixtral/customize.py b/script/get-dataset-mixtral/customize.py new file mode 100644 index 0000000000..31d72127ea --- /dev/null +++ b/script/get-dataset-mixtral/customize.py @@ -0,0 +1,18 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + return {'return':0} + + +def postprocess(i): + env = i['env'] + + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] = env['CM_DATASET_PREPROCESSED_PATH'] + + return {'return':0} From 97d792757f017628d4bd25b6c8a57c3396dfa887 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:33:08 +0530 Subject: [PATCH 041/658] fixed syntax --- script/app-mlperf-inference/_cm.yaml | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 6fa0135088..014910d4db 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -800,24 +800,23 @@ variations: tags: run,accuracy,mlperf,_openorca-gsm8k-mbxp,_int32 mixtral-8x7b,reference: - - tags: get,ml-model,mixtral - names: - - ml-model - - mixtral-model - enable_if_env: - CM_MLPERF_MODEL_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: - - 'yes' - docker: - mounts: + docker: + deps: + - tags: get,ml-model,mixtral + names: + - ml-model + - mixtral-model + enable_if_env: + CM_MLPERF_MODEL_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + - 'yes' + - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined + names: + - openorca-mbxp-gsm8k-combined-preprocessed + enable_if_env: + CM_MLPERF_DATASET_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + - 'yes' + mounts: - "${{ MIXTRAL_CHECKPOINT_PATH }}:${{ MIXTRAL_CHECKPOINT_PATH }}" - - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined - names: - - openorca-mbxp-gsm8k-combined-preprocessed - enable_if_env: - CM_MLPERF_DATASET_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: - - 'yes' - docker: - mounts: - "${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}:${{ CM_DATASET_MIXTRAL_PREPROCESSED_PATH }}" rnnt: From 4d5ab8b519fee9c89bebe68e8c7ac82d96008a87 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:52:55 +0530 Subject: [PATCH 042/658] use _ instead of - in env variables --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 4 ++-- script/app-mlperf-inference/_cm.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 5d392b5e68..0576f6eca0 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -435,7 +435,7 @@ deps: CM_MLPERF_CUSTOM_MODEL_PATH: - "on" skip_if_env: - CM_MLPERF_MODEL_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - 'yes' ## 3d-unet @@ -559,7 +559,7 @@ deps: CM_MODEL: - mixtral-8x7b skip_if_env: - CM_MLPERF_DATASET_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - 'yes' ## Kits19 for 3d-unet diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 014910d4db..bea7ae604b 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -807,13 +807,13 @@ variations: - ml-model - mixtral-model enable_if_env: - CM_MLPERF_MODEL_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - 'yes' - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined names: - openorca-mbxp-gsm8k-combined-preprocessed enable_if_env: - CM_MLPERF_DATASET_MIXTRAL-8X7B_DOWNLOAD_TO_HOST: + CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST: - 'yes' mounts: - "${{ MIXTRAL_CHECKPOINT_PATH }}:${{ MIXTRAL_CHECKPOINT_PATH }}" From 19152ede94955aa95e851ef5b66bac102798b6c7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 5 Oct 2024 18:37:19 +0530 Subject: [PATCH 043/658] assigned default string to be taken in env variable --- script/app-mlperf-inference-mlcommons-python/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 0c2646df5e..d9b91b26d5 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -78,7 +78,7 @@ def preprocess(i): if env.get('CM_NETWORK_LOADGEN', '') != "lon" and env.get('CM_MLPERF_INFERENCE_API_SERVER','')=='' and "llama2-70b" not in env['CM_MODEL']: env['MODEL_DIR'] = env.get('CM_ML_MODEL_PATH') if not env['MODEL_DIR']: - env['MODEL_DIR'] = os.path.dirname(env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH'))) + env['MODEL_DIR'] = os.path.dirname(env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH', ''))) RUN_CMD = "" state['RUN'] = {} From e98da2361d96dcd17bea784492a0f79a640c0cc3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 5 Oct 2024 19:38:02 +0530 Subject: [PATCH 044/658] argument batch size changed for reference mixtral --- script/app-mlperf-inference-mlcommons-python/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index d9b91b26d5..0376293484 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -340,6 +340,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \ " --model-path " + env['MIXTRAL_CHECKPOINT_PATH'] cmd = cmd.replace("--count", "--total-sample-count") + cmd = cmd.replace("--max-batchsize", "--batch-size") elif "3d-unet" in env['CM_MODEL']: From c6eadd917821a06856951f4f5a7fdbfbb3569162 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 5 Oct 2024 19:54:43 +0530 Subject: [PATCH 045/658] Update env variables for processing accuracy --- script/process-mlperf-accuracy/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index d6536e13fd..765f96a353 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -87,8 +87,8 @@ def preprocess(i): elif dataset == "openorca-gsm8k-mbxp-combined": accuracy_checker_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b", "evaluate-accuracy.py") - CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['CM_ML_MODEL_MIXTRAL_FILE_WITH_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --dataset-file '" + env['CM_DATASET_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32") +" > '" + out_file + "'" + CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['MIXTRAL_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --dataset-file '" + env['CM_DATASET_MIXTRAL_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32") +" > '" + out_file + "'" elif dataset == "coco2014": From ed7f42507f7993782e9147d3a04bdbd225ad8089 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 5 Oct 2024 19:58:55 +0530 Subject: [PATCH 046/658] skip model and dataset deps when download to host is set --- script/process-mlperf-accuracy/_cm.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index cd4028a533..35508e1703 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -344,12 +344,22 @@ "names": [ "openorca-gsm8k-mbxp-combined" ], + "skip_if_env": { + "CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [ + "yes" + ] + }, "tags": "get,dataset-mixtral,openorca-mbxp-gsm8k-combined" }, { "names": [ "mixtral-8x7b-model" ], + "skip_if_env": { + "CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [ + "yes" + ] + }, "tags": "get,ml-model,mixtral" } ], From 12801d7199184fc44b79c4afc1023d15f3e98438 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 5 Oct 2024 22:34:05 +0530 Subject: [PATCH 047/658] added rouge score module dependency --- script/process-mlperf-accuracy/_cm.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index 35508e1703..1addc3cbf0 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -340,6 +340,14 @@ }, "openorca-gsm8k-mbxp": { "deps": [ + { + "tags": "get,generic-python-lib,_package.rouge_score", + "names": + [ + "pip-package", + "rouge-score" + ] + }, { "names": [ "openorca-gsm8k-mbxp-combined" From 4a63a2d8f57d6d23b614c3ed62484393a111d1a6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 19:53:45 +0100 Subject: [PATCH 048/658] Update test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml --- ...mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index 4b714965a3..44ec1b2792 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -27,6 +27,7 @@ jobs: - backend: pytorch - backend: onnxruntime - precision: fp32 + - os: windows-latest steps: - uses: actions/checkout@v3 @@ -60,4 +61,4 @@ jobs: git config --global credential.https://gist.github.com.helper "!gh auth git-credential" cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet \ No newline at end of file + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet From d470b0af36b0ba5a039e924984a7992d6900858d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 20:02:47 +0100 Subject: [PATCH 049/658] Update test-mlperf-inference-retinanet.yml --- .github/workflows/test-mlperf-inference-retinanet.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index 54d6198e15..54217d71af 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -4,7 +4,7 @@ name: MLPerf inference retinanet on: - pull_request: + pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-retinanet.yml' @@ -24,6 +24,8 @@ jobs: exclude: - backend: pytorch implementation: cpp + - os: windows + implementation: cpp steps: - uses: actions/checkout@v3 @@ -58,4 +60,4 @@ jobs: git config --global credential.https://gist.github.com.helper "!gh auth git-credential" cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet \ No newline at end of file + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet From a51e6f6ee1cb04313c326b2b408ec497a11d41c8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 20:07:18 +0100 Subject: [PATCH 050/658] Update test-mlperf-inference-mlcommons-cpp-resnet50.yml --- .../workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index f82e00750e..afa00114de 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -23,6 +23,7 @@ jobs: exclude: - llvm-version: "15.0.6" - llvm-version: "16.0.4" + - os: windows-latest steps: - uses: actions/checkout@v3 From cc7f9ddc81864af0a913239f5bef4f9da12f4317 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 20:43:59 +0100 Subject: [PATCH 051/658] Update _cm.yaml --- script/get-sys-utils-cm/_cm.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 3310c28e8a..4d3e755ed7 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -14,8 +14,6 @@ deps: env: CM_CLEAN_DIRS: bin CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 - ; https://cKnowledge.org/ai/data/zlib123dllx64-bin.zip - ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip CM_SUDO: sudo input_mapping: From 92961271ea4657e609c744352a21f497fedb6cc2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 20:45:41 +0100 Subject: [PATCH 052/658] Update test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml --- ...-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index 44ec1b2792..9fed7358ce 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -4,7 +4,7 @@ name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch) on: - pull_request: + pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml' From 73434fde2825a4ce18ca8b6c3a265ecad6bce6a4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:10:14 +0530 Subject: [PATCH 053/658] Test windows bat --- script/push-mlperf-inference-results-to-github/run.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index ebf497a8bf..d482a1ecfe 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -8,8 +8,9 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH ( echo %cd% echo "Doing cd" +echo %CM_GIT_REPO_CHECKOUT_PATH% REM Change directory -cd "%CM_GIT_REPO_CHECKOUT_PATH%" || ( +chdir %CM_GIT_REPO_CHECKOUT_PATH% || ( echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" exit /b 1 ) From 684cf46abb883c88a8ce86dd83335475f469429a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:21:18 +0530 Subject: [PATCH 054/658] Test windows bat --- script/push-mlperf-inference-results-to-github/run.bat | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index d482a1ecfe..93adf1f238 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -10,10 +10,7 @@ echo %cd% echo "Doing cd" echo %CM_GIT_REPO_CHECKOUT_PATH% REM Change directory -chdir %CM_GIT_REPO_CHECKOUT_PATH% || ( - echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" - exit /b 1 -) +chdir %CM_GIT_REPO_CHECKOUT_PATH% echo %cd% echo "%CM_GIT_REPO_CHECKOUT_PATH%" git pull From 0c7a0a4b9ff7dd0640beb7beca4c28ffa82071b4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 22:04:43 +0100 Subject: [PATCH 055/658] Update test-mlperf-inference-retinanet.yml --- .github/workflows/test-mlperf-inference-retinanet.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index 54217d71af..e0cfbe6b6c 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -24,7 +24,9 @@ jobs: exclude: - backend: pytorch implementation: cpp - - os: windows + - os: windows-latest + implementation: cpp + - os: macos-latest implementation: cpp steps: From 30dcc3b30a92cac9ac4cfaca1e93d150ce04e061 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:47:27 +0530 Subject: [PATCH 056/658] Fix windows bat file for push-mlperf-inference-results --- .../push-mlperf-inference-results-to-github/run.bat | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index 93adf1f238..e0da85d90f 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -6,13 +6,12 @@ if not defined CM_GIT_REPO_CHECKOUT_PATH ( exit /b 1 ) -echo %cd% -echo "Doing cd" -echo %CM_GIT_REPO_CHECKOUT_PATH% -REM Change directory -chdir %CM_GIT_REPO_CHECKOUT_PATH% -echo %cd% -echo "%CM_GIT_REPO_CHECKOUT_PATH%" +cd /d "%CM_GIT_REPO_CHECKOUT_PATH%" +if %errorlevel% neq 0 ( + echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%" + exit /b 1 +) + git pull git add * From 71ca44966b741447759111de2feec3e86667eb58 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 22:31:43 +0100 Subject: [PATCH 057/658] Update README.md --- README.md | 118 ++---------------------------------------------------- 1 file changed, 3 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index deff5458ce..9e049ec68d 100644 --- a/README.md +++ b/README.md @@ -13,125 +13,13 @@ [![Test QAIC Software kit Compilation](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml) -This repository contains reusable and cross-platform automation recipes to run DevOps, MLOps, and MLPerf -via a simple and human-readable [Collective Mind interface (CM)](https://github.com/mlcommons/ck) -while adapting to different operating systems, software and hardware. -All СM scripts have a simple Python API, extensible JSON/YAML meta description -and unified input/output to make them reusable in different projects either individually -or by chaining them together into portable automation workflows, applications -and web services adaptable to continuously changing models, data sets, software and hardware. +The `mlperf-branch` of cm4mlops repository is where we have the updatd CM scripts for MLPerf inference including Automotive. The general CM scripts in this branch may not be compatible with any other project. -We develop and test [CM scripts](script) as a community effort to support the following projects: -* [CM for MLPerf](https://docs.mlcommons.org/inference): modularize and automate MLPerf benchmarks - * [Modular C++ harness for MLPerf loadgen](https://github.com/mlcommons/cm4mlops/tree/main/script/app-mlperf-inference-mlcommons-cpp) - * [Modular Python harness for MLPerf loadgen](https://github.com/mlcommons/cm4mlops/tree/main/script/app-loadgen-generic-python) -* [CM for research and education](https://cTuning.org/ae): provide a common interface to automate and reproduce results from research papers - and MLPerf benchmarks; -* [CM for ABTF](https://github.com/mlcommons/cm4abtf): provide a unified CM interface to run automotive benchmarks; -* [CM for optimization](https://access.cknowledge.org/playground/?action=challenges): co-design efficient and cost-effective - software and hardware for AI, ML and other emerging workloads via open challenges. +[Main MLPerf Inference Documentation Site](https://docs.mlcommons.org/inference/) -You can read this [ArXiv paper](https://arxiv.org/abs/2406.16791) to learn more about the CM motivation and long-term vision. +[![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) -Please provide your feedback or submit your issues [here](https://github.com/mlcommons/cm4mlops/issues). - - -## Catalog - -Online catalog: [cKnowledge](https://access.cknowledge.org/playground/?action=scripts), [MLCommons](https://docs.mlcommons.org/cm4mlops/scripts). - -## Citation - -Please use this [BibTeX file](https://github.com/mlcommons/ck/blob/master/citation.bib) to cite this project. - -## A few demos - -### Install CM and virtual env - -Install the [MLCommons CM automation language](https://access.cknowledge.org/playground/?action=install). - -### Pull this repository - -```bash -cm pull repo mlcommons@cm4mlops --branch=dev -``` - -### Run image classification using CM - -```bash - -cm run script "python app image-classification onnx _cpu" --help - -cm run script "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e -cm run script "python app image-classification onnx _cpu" --input=computer_mouse.jpg - -cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg -cmr --tags=python,app,image-classification,onnx,_cpu --input=computer_mouse.jpg -cmr 3d5e908e472b417e --input=computer_mouse.jpg - -cm docker script "python app image-classification onnx _cpu" --input=computer_mouse.jpg - -cm gui script "python app image-classification onnx _cpu" -``` - -### Re-run experiments from the ACM/IEEE MICRO'23 paper - -Check this [script/reproduce-ieee-acm-micro2023-paper-96](README.md). - -### Run MLPerf ResNet CPU inference benchmark via CM - -```bash -cm run script --tags=run-mlperf,inference,_performance-only,_short \ - --division=open \ - --category=edge \ - --device=cpu \ - --model=resnet50 \ - --precision=float32 \ - --implementation=mlcommons-python \ - --backend=onnxruntime \ - --scenario=Offline \ - --execution_mode=test \ - --power=no \ - --adr.python.version_min=3.8 \ - --clean \ - --compliance=no \ - --quiet \ - --time -``` - -### Run MLPerf BERT CUDA inference benchmark v4.1 via CM - -```bash -cmr "run-mlperf inference _find-performance _full _r4.1" \ - --model=bert-99 \ - --implementation=nvidia \ - --framework=tensorrt \ - --category=datacenter \ - --scenario=Offline \ - --execution_mode=test \ - --device=cuda \ - --docker \ - --docker_cm_repo=mlcommons@cm4mlops \ - --docker_cm_repo_flags="--branch=mlperf-inference" \ - --test_query_count=100 \ - --quiet -``` - -### Run MLPerf SDXL reference inference benchmark v4.1 via CM - -```bash -cm run script \ - --tags=run-mlperf,inference,_r4.1 \ - --model=sdxl \ - --implementation=reference \ - --framework=pytorch \ - --category=datacenter \ - --scenario=Offline \ - --execution_mode=valid \ - --device=cuda \ - --quiet -``` ## License From 65d3cf5081f54d6bb0777f60271d04b0fa1d8c71 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 22:33:47 +0100 Subject: [PATCH 058/658] Update README.md --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 9e049ec68d..b1c1fb58cb 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,7 @@ [![Test QAIC Software kit Compilation](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml) - -The `mlperf-branch` of cm4mlops repository is where we have the updatd CM scripts for MLPerf inference including Automotive. The general CM scripts in this branch may not be compatible with any other project. - -[Main MLPerf Inference Documentation Site](https://docs.mlcommons.org/inference/) +The `mlperf-branch` of the **cm4mlops** repository contains updated CM scripts specifically for MLPerf Inference, including support for Automotive. Please note that the general CM scripts in this branch may not be compatible with other projects. For more information on using CM for MLPerf Inference, visit the [MLPerf Inference Documentation site](https://docs.mlcommons.org/inference/). [![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) From b042b255cc8ccbf9345c21e79767c4a771f98fef Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 03:08:17 +0530 Subject: [PATCH 059/658] Fix windows bat file for push-mlperf-inference-results --- script/push-mlperf-inference-results-to-github/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index e0da85d90f..e35929810c 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -17,7 +17,7 @@ git add * REM Check if the CM_MLPERF_INFERENCE_SUBMISSION_DIR variable is set if defined CM_MLPERF_INFERENCE_SUBMISSION_DIR ( - robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /MIR + robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /MIR /A+H git add * ) From 3a1c49200d829b34f3d492a5bcc150d6d5a00f6f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 03:22:26 +0530 Subject: [PATCH 060/658] Fix windows bat file for push-mlperf-inference-results --- script/push-mlperf-inference-results-to-github/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index e35929810c..3bfb551412 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -17,7 +17,7 @@ git add * REM Check if the CM_MLPERF_INFERENCE_SUBMISSION_DIR variable is set if defined CM_MLPERF_INFERENCE_SUBMISSION_DIR ( - robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /MIR /A+H + robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /E /COPYALL /DCOPY:DAT /A-:SH git add * ) From 7fe1da113a5583ed2bfa723fa3845e462733c683 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 03:30:23 +0530 Subject: [PATCH 061/658] Fix windows bat file for push-mlperf-inference-results --- script/push-mlperf-inference-results-to-github/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index 3bfb551412..c4b11ba74a 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -25,7 +25,7 @@ REM Check if the previous command was successful if %errorlevel% neq 0 exit /b %errorlevel% git commit -a -m "%CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE%" -REM git push +git push REM Check if the previous command was successful if %errorlevel% neq 0 exit /b %errorlevel% From 4931614b5d90b239d188bde493a0f13834bda160 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 5 Oct 2024 23:16:26 +0100 Subject: [PATCH 062/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index c1e249f2ff..9b66b98773 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -60,6 +60,7 @@ jobs: git config --global credential.https://github.com.helper "!gh auth git-credential" git config --global credential.https://gist.github.com.helper "" git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + git config --global credential.helper 'cache --timeout=3600' cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet From 3bd0a264fb76a273e36b80861c03053210c3ff10 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 03:36:58 +0530 Subject: [PATCH 063/658] Use raw string in generate-mlperf-inference-user-conf --- script/generate-mlperf-inference-user-conf/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-user-conf/customize.py b/script/generate-mlperf-inference-user-conf/customize.py index e2569db27f..9acc4da978 100644 --- a/script/generate-mlperf-inference-user-conf/customize.py +++ b/script/generate-mlperf-inference-user-conf/customize.py @@ -346,7 +346,7 @@ def preprocess(i): env['CM_MLPERF_RANGING_USER_CONF'] = os.path.join(os.path.dirname(user_conf_path), "ranging_"+key+".conf")# ranging_user_conf_path for a shorter run if short_ranging: - env['CM_MLPERF_USER_CONF'] = "\${CM_MLPERF_USER_CONF}" + env['CM_MLPERF_USER_CONF'] = r"\${CM_MLPERF_USER_CONF}" else: env['CM_MLPERF_USER_CONF'] = os.path.join(os.path.dirname(user_conf_path), key+".conf")# user_conf_path else: From 7fd221967976291b4cd5e1d7abcdb77df301d15d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 03:55:54 +0530 Subject: [PATCH 064/658] Use raw string in generate-mlperf-inference-user-conf --- script/push-mlperf-inference-results-to-github/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/push-mlperf-inference-results-to-github/run.bat b/script/push-mlperf-inference-results-to-github/run.bat index c4b11ba74a..2052eb5644 100644 --- a/script/push-mlperf-inference-results-to-github/run.bat +++ b/script/push-mlperf-inference-results-to-github/run.bat @@ -17,7 +17,7 @@ git add * REM Check if the CM_MLPERF_INFERENCE_SUBMISSION_DIR variable is set if defined CM_MLPERF_INFERENCE_SUBMISSION_DIR ( - robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /E /COPYALL /DCOPY:DAT /A-:SH + robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /E /COPYALL /DCOPY:DAT git add * ) From fdf11dae03d6d50c16ca95f4e8bcad143a6fdef5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 04:11:58 +0530 Subject: [PATCH 065/658] Fix github-auth-cli on windows --- script/authenticate-github-cli/customize.py | 5 ++++- script/authenticate-github-cli/run.bat | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/script/authenticate-github-cli/customize.py b/script/authenticate-github-cli/customize.py index 14b762e0f2..bb4bbef2d6 100644 --- a/script/authenticate-github-cli/customize.py +++ b/script/authenticate-github-cli/customize.py @@ -13,7 +13,10 @@ def preprocess(i): cmd = "gh auth login" if env.get('CM_GH_AUTH_TOKEN', '') != '': - cmd = f" echo {env['CM_GH_AUTH_TOKEN']} | {cmd} --with-token" + if os_info['platform'] == 'windows': + cmd = f" echo %CM_GH_AUTH_TOKEN% | {cmd} --with-token" + else: + cmd = f" echo {env['CM_GH_AUTH_TOKEN']} | {cmd} --with-token" env['CM_RUN_CMD'] = cmd quiet = (env.get('CM_QUIET', False) == 'yes') diff --git a/script/authenticate-github-cli/run.bat b/script/authenticate-github-cli/run.bat index 648302ca71..ea2ecc7422 100644 --- a/script/authenticate-github-cli/run.bat +++ b/script/authenticate-github-cli/run.bat @@ -1 +1,18 @@ -rem native script +@echo off +echo Running gh auth: +REM Not printing CM_RUN_CMD as it can contain secret +REM echo %CM_RUN_CMD% +echo. + +REM Check if CM_FAKE_RUN is not equal to "yes" +if not "%CM_FAKE_RUN%"=="yes" ( + + REM Execute the command stored in CM_RUN_CMD + call %CM_RUN_CMD% + + REM Check the exit code and exit with error if non-zero + if %ERRORLEVEL% neq 0 ( + exit /b 1 + ) +) + From 7a095b34937a3154e0b54dcd283f0d54a0574038 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 04:31:24 +0530 Subject: [PATCH 066/658] Fix github-auth-cli on windows --- script/authenticate-github-cli/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/authenticate-github-cli/run.bat b/script/authenticate-github-cli/run.bat index ea2ecc7422..ef634aa61a 100644 --- a/script/authenticate-github-cli/run.bat +++ b/script/authenticate-github-cli/run.bat @@ -8,7 +8,7 @@ REM Check if CM_FAKE_RUN is not equal to "yes" if not "%CM_FAKE_RUN%"=="yes" ( REM Execute the command stored in CM_RUN_CMD - call %CM_RUN_CMD% + %CM_RUN_CMD% REM Check the exit code and exit with error if non-zero if %ERRORLEVEL% neq 0 ( From d741897c9bdb7442096bf6280274a1d8a71e8960 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 04:51:05 +0530 Subject: [PATCH 067/658] Fix github-auth-cli on windows --- script/authenticate-github-cli/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/authenticate-github-cli/customize.py b/script/authenticate-github-cli/customize.py index bb4bbef2d6..0a93d5038f 100644 --- a/script/authenticate-github-cli/customize.py +++ b/script/authenticate-github-cli/customize.py @@ -14,7 +14,7 @@ def preprocess(i): cmd = "gh auth login" if env.get('CM_GH_AUTH_TOKEN', '') != '': if os_info['platform'] == 'windows': - cmd = f" echo %CM_GH_AUTH_TOKEN% | {cmd} --with-token" + cmd = f"{cmd} --with-token < %CM_GH_AUTH_TOKEN%" else: cmd = f" echo {env['CM_GH_AUTH_TOKEN']} | {cmd} --with-token" From 4828582e4d0c83c351e6386b64f6d198f295d3f5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 05:09:49 +0530 Subject: [PATCH 068/658] Fix github-auth-cli on windows --- script/authenticate-github-cli/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/authenticate-github-cli/customize.py b/script/authenticate-github-cli/customize.py index 0a93d5038f..814e29a72c 100644 --- a/script/authenticate-github-cli/customize.py +++ b/script/authenticate-github-cli/customize.py @@ -14,7 +14,7 @@ def preprocess(i): cmd = "gh auth login" if env.get('CM_GH_AUTH_TOKEN', '') != '': if os_info['platform'] == 'windows': - cmd = f"{cmd} --with-token < %CM_GH_AUTH_TOKEN%" + cmd = f"{cmd} --with-token %CM_GH_AUTH_TOKEN%" else: cmd = f" echo {env['CM_GH_AUTH_TOKEN']} | {cmd} --with-token" From 20ca636e0624f2001dd8bcabd2a283eb0cb92062 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 00:58:28 +0100 Subject: [PATCH 069/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 9b66b98773..e9634aac35 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -35,6 +35,7 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + github-token: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }} - name: Install dependencies run: | python3 -m pip install cmind From bd0b1e9cf5a3e5a8894fd0769cea58035e7036c5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 05:31:54 +0530 Subject: [PATCH 070/658] Fix github-auth-cli on windows --- script/authenticate-github-cli/customize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/authenticate-github-cli/customize.py b/script/authenticate-github-cli/customize.py index 814e29a72c..a873791f43 100644 --- a/script/authenticate-github-cli/customize.py +++ b/script/authenticate-github-cli/customize.py @@ -14,7 +14,9 @@ def preprocess(i): cmd = "gh auth login" if env.get('CM_GH_AUTH_TOKEN', '') != '': if os_info['platform'] == 'windows': - cmd = f"{cmd} --with-token %CM_GH_AUTH_TOKEN%" + with open("token", "w") as f: + f.write(env['CM_GH_AUTH_TOKEN']) + cmd = f"{cmd} --with-token < token" else: cmd = f" echo {env['CM_GH_AUTH_TOKEN']} | {cmd} --with-token" From 8d45120986a5caa14dde8bf546ec672fad777d57 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 06:14:51 +0530 Subject: [PATCH 071/658] Fix github-auth-cli on windows --- script/authenticate-github-cli/run.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/authenticate-github-cli/run.bat b/script/authenticate-github-cli/run.bat index ef634aa61a..2366ffc076 100644 --- a/script/authenticate-github-cli/run.bat +++ b/script/authenticate-github-cli/run.bat @@ -8,7 +8,8 @@ REM Check if CM_FAKE_RUN is not equal to "yes" if not "%CM_FAKE_RUN%"=="yes" ( REM Execute the command stored in CM_RUN_CMD - %CM_RUN_CMD% + REM %CM_RUN_CMD% + echo %CM_GH_AUTH_TOKEN% | gh auth login --with-token REM Check the exit code and exit with error if non-zero if %ERRORLEVEL% neq 0 ( From f0af3c42de11149fc5e0f4335a9b8a901f1c56cb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 01:49:34 +0100 Subject: [PATCH 072/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index e9634aac35..244fd6a8f1 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -54,6 +54,7 @@ jobs: env: USER: "GitHub Action" EMAIL: "admin@gateoverflow.com" + GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }} run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" From 46e42fb7a16e3654d139bad76fbd8c63f4e71205 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 01:55:23 +0100 Subject: [PATCH 073/658] Update test-mlperf-inference-resnet50.yml | update token passing --- .github/workflows/test-mlperf-inference-resnet50.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 244fd6a8f1..3a2af98b1d 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -35,7 +35,6 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - github-token: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }} - name: Install dependencies run: | python3 -m pip install cmind @@ -58,11 +57,4 @@ jobs: run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" - git config --global credential.https://github.com.helper "" - git config --global credential.https://github.com.helper "!gh auth git-credential" - git config --global credential.https://gist.github.com.helper "" - git config --global credential.https://gist.github.com.helper "!gh auth git-credential" - git config --global credential.helper 'cache --timeout=3600' - - cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet From 90c4e645c4c266095df10028b1e620dffbd82ddc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 01:57:21 +0100 Subject: [PATCH 074/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 3a2af98b1d..38e5e276af 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -43,7 +43,7 @@ jobs: if: matrix.os == 'windows-latest' run: | git config --system core.longpaths true - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_windows --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet - name: Test MLPerf Inference ResNet50 (Linux/macOS) if: matrix.os != 'windows-latest' run: | From 4d9c5919cc1f1396f8d152263c16d92bd13cb19e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:03:53 +0100 Subject: [PATCH 075/658] Update test-mlperf-inference-retinanet.yml --- .github/workflows/test-mlperf-inference-retinanet.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index e0cfbe6b6c..429fdd3737 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -53,13 +53,8 @@ jobs: env: USER: "GitHub Action" EMAIL: "admin@gateoverflow.com" + GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }} run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" - git config --global credential.https://github.com.helper "" - git config --global credential.https://github.com.helper "!gh auth git-credential" - git config --global credential.https://gist.github.com.helper "" - git config --global credential.https://gist.github.com.helper "!gh auth git-credential" - - cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet From 4f5d80144cc47adeb91524dfbca35902f3ed13b7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:04:44 +0100 Subject: [PATCH 076/658] Update test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml --- ...rf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index 9fed7358ce..99bb6d3b27 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -52,13 +52,8 @@ jobs: env: USER: "GitHub Action" EMAIL: "admin@gateoverflow.com" + GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }} run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" - git config --global credential.https://github.com.helper "" - git config --global credential.https://github.com.helper "!gh auth git-credential" - git config --global credential.https://gist.github.com.helper "" - git config --global credential.https://gist.github.com.helper "!gh auth git-credential" - - cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet From ec53df4e87cf665f26eb9ff4e9dd72008be43ce1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:11:22 +0100 Subject: [PATCH 077/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index fd452c3ba6..0f68195d1a 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -22,7 +22,10 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_performance-only,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_short --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=edge --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=edge --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/gh_action_submissions --results_dir=$HOME/gh_action_results/test_results + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions build_nvidia: if: github.repository_owner == 'gateoverflow_off' @@ -41,4 +44,8 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_performance-only,_short --submitter="MLCommons" --docker --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_short --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --category=edge --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=edge --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/gh_action_submissions --results_dir=$HOME/gh_action_results/test_results + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions + From 1cff8389151ddcc78b5a3615ab91887eec32435d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:27:18 +0100 Subject: [PATCH 078/658] Update test-mlperf-inference-resnet50.yml --- .github/workflows/test-mlperf-inference-resnet50.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 38e5e276af..5e1f00c478 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -57,4 +57,8 @@ jobs: run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet From 55a728bb8f6ffa638303f9191b2237ffd53241a8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:28:25 +0100 Subject: [PATCH 079/658] Update test-mlperf-inference-retinanet.yml --- .github/workflows/test-mlperf-inference-retinanet.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-retinanet.yml b/.github/workflows/test-mlperf-inference-retinanet.yml index 429fdd3737..5077ad19e7 100644 --- a/.github/workflows/test-mlperf-inference-retinanet.yml +++ b/.github/workflows/test-mlperf-inference-retinanet.yml @@ -57,4 +57,8 @@ jobs: run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Retinanet GH action on ${{ matrix.os }}" --quiet From eac7598c2b35fb8e28ab00345d5d3c5cbdb8bb43 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 02:33:24 +0100 Subject: [PATCH 080/658] Update test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml --- ...lperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index 99bb6d3b27..7ac57d8e70 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -56,4 +56,8 @@ jobs: run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from Bert GH action on ${{ matrix.os }}" --quiet From cb39efb873fbe85502992a30060385c13ede28b7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 13:44:55 +0100 Subject: [PATCH 081/658] Update test-mlperf-inference-sdxl.yaml | Enable back --- .github/workflows/test-mlperf-inference-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 0f68195d1a..d9b3c5a47a 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -6,7 +6,7 @@ on: jobs: build_reference: - if: github.repository_owner == 'gateoverflow_off' + if: github.repository_owner == 'gateoverflow' runs-on: [ self-hosted, linux, x64 ] strategy: fail-fast: false From b351b11b5f04f0f2898c63f6543c250ed2089d0b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sun, 6 Oct 2024 22:50:15 +0530 Subject: [PATCH 082/658] disabled mac os + added env variable(TOKEN) --- .../workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index afa00114de..61f59392f5 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -24,6 +24,7 @@ jobs: - llvm-version: "15.0.6" - llvm-version: "16.0.4" - os: windows-latest + - os: macos-latest steps: - uses: actions/checkout@v3 @@ -50,6 +51,7 @@ jobs: env: USER: "GitHub Action" EMAIL: "admin@gateoverflow.com" + GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }} run: | git config --global user.name "${{ env.USER }}" git config --global user.email "${{ env.EMAIL }}" From 9ebb6b2c63c621dc229c047750a3c8be872e2ad0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 6 Oct 2024 18:34:14 +0100 Subject: [PATCH 083/658] Update test-mlperf-inference-mlcommons-cpp-resnet50.yml --- .../workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index 61f59392f5..d537fc8964 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -4,7 +4,7 @@ name: MLPerf inference MLCommons C++ ResNet50 on: - pull_request: + pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml' From f2c0690243a4bb004c628e71ed930449ba6ffd04 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 7 Oct 2024 14:24:47 +0100 Subject: [PATCH 084/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index d9b3c5a47a..09c7f6aae1 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -22,9 +22,7 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_performance-only,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean - cm run script --tags=run-mlperf,inference,_short --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=edge --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=edge --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/gh_action_submissions --results_dir=$HOME/gh_action_results/test_results + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions build_nvidia: From 7408609c73004dd3d4086da573544260f8c90ea0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 7 Oct 2024 14:26:07 +0100 Subject: [PATCH 085/658] Update test-mlperf-inference-sdxl.yaml --- .../workflows/test-mlperf-inference-sdxl.yaml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 09c7f6aae1..2b616e9603 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -24,26 +24,3 @@ jobs: cm pull repo cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions - - build_nvidia: - if: github.repository_owner == 'gateoverflow_off' - runs-on: [ self-hosted, linux, x64 ] - strategy: - fail-fast: false - matrix: - python-version: [ "3.12" ] - backend: [ "tensorrt" ] - precision: [ "float16" ] - implementation: [ "nvidia" ] - steps: - - name: Test MLPerf Inference SDXL Nvidia - run: | - source gh_action/bin/deactivate || python3 -m venv gh_action - source gh_action/bin/activate - export CM_REPOS=$HOME/GH_CM - cm pull repo - cm run script --tags=run-mlperf,inference,_performance-only,_short --submitter="MLCommons" --docker --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean - cm run script --tags=run-mlperf,inference,_short --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --category=edge --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=edge --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/gh_action_submissions --results_dir=$HOME/gh_action_results/test_results - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions - From ad8a267d917472be65d5bbb660c573f90945c31f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 7 Oct 2024 17:03:10 +0100 Subject: [PATCH 086/658] Update test-mlperf-inference-mlcommons-cpp-resnet50.yml --- .../workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index d537fc8964..4b71896296 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -59,6 +59,4 @@ jobs: git config --global credential.https://github.com.helper "!gh auth git-credential" git config --global credential.https://gist.github.com.helper "" git config --global credential.https://gist.github.com.helper "!gh auth git-credential" - - cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}" cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from MLCommons C++ ResNet50 GH action on ${{ matrix.os }}" --quiet From 769bf6afb15058ab9699c8b708fc484cc9c7b7fb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 7 Oct 2024 22:03:19 +0530 Subject: [PATCH 087/658] Fix potential env corruption in get-cuda-devices --- script/get-cuda-devices/_cm.yaml | 5 ++++- script/get-cuda-devices/customize.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/script/get-cuda-devices/_cm.yaml b/script/get-cuda-devices/_cm.yaml index e0d348b831..2cd9073d7b 100644 --- a/script/get-cuda-devices/_cm.yaml +++ b/script/get-cuda-devices/_cm.yaml @@ -31,6 +31,9 @@ docker: use_host_group_id: 'yes' image_tag_extra: '-cm-dev' +env: + CM_DETECT_USING_PYCUDA: 'no' + new_env_keys: - CM_CUDA_DEVICE_* - CM_CUDA_NUM_DEVICES @@ -46,7 +49,7 @@ print_files_if_script_error: variations: with-pycuda: env: - CM_CUDA_DEVICES_DETECT_USING_PYCUDA: 'yes' + CM_DETECT_USING_PYCUDA: 'yes' deps: - tags: get,python3 names: diff --git a/script/get-cuda-devices/customize.py b/script/get-cuda-devices/customize.py index 4aaf215484..4184d3b54b 100644 --- a/script/get-cuda-devices/customize.py +++ b/script/get-cuda-devices/customize.py @@ -6,7 +6,7 @@ def preprocess(i): env = i['env'] - if str(env.get('CM_CUDA_DEVICES_DETECT_USING_PYCUDA', '')).lower() in [ "1", "yes", "true"]: + if str(env.get('CM_DETECT_USING_PYCUDA', '')).lower() in [ "1", "yes", "true"]: i['run_script_input']['script_name'] = 'detect' return {'return':0} From 3df4c3c1de665fe98d612b6485e4a3fa67fdebb7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:00:28 +0530 Subject: [PATCH 088/658] added cuda version to run suffix 5 --- script/get-cuda/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-cuda/customize.py b/script/get-cuda/customize.py index 03e6c7bf79..11de3c6cd7 100644 --- a/script/get-cuda/customize.py +++ b/script/get-cuda/customize.py @@ -214,5 +214,6 @@ def postprocess(i): env['+ LDFLAGS'].append("-L"+x) env['CM_CUDA_VERSION_STRING'] = "cu"+env['CM_CUDA_VERSION'].replace(".", "") + env['CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX5'] = env['CM_CUDA_VERSION_STRING'] return {'return':0, 'version': version} From e3febed3beb9df7ea4b8fa2932112abd68dfd35c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:01:39 +0530 Subject: [PATCH 089/658] updated new env keys for run suffix 5 to reflect --- script/get-cuda/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-cuda/_cm.yaml b/script/get-cuda/_cm.yaml index d68e3fd7f8..db5a30b0bb 100644 --- a/script/get-cuda/_cm.yaml +++ b/script/get-cuda/_cm.yaml @@ -46,6 +46,7 @@ new_env_keys: - CUDA_PATH - CM_CUDA_* - CM_NVCC_* +- CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX5 - +PATH - +C_INCLUDE_PATH - +CPLUS_INCLUDE_PATH From d1a71e5bcb2d03903ce4fc0a450b783753295c0f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:07:25 +0530 Subject: [PATCH 090/658] Modified to take upto 5 run config suffixes --- script/get-mlperf-inference-sut-configs/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-sut-configs/customize.py b/script/get-mlperf-inference-sut-configs/customize.py index e4a6a40481..75ef95b94a 100644 --- a/script/get-mlperf-inference-sut-configs/customize.py +++ b/script/get-mlperf-inference-sut-configs/customize.py @@ -27,7 +27,7 @@ def postprocess(i): implementation_string = env['CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX'] if env.get('CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX', '') != '' else env.get('CM_MLPERF_IMPLEMENTATION', 'default') run_config = [] - for i in range(1,5): + for i in range(1,6): if env.get(f'CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX{i}', '') != '': run_config.append(env.get(f'CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX{i}')) From bcec9ec54eb3b0408cc2033e6f86e05a995c5b23 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 8 Oct 2024 00:46:41 +0530 Subject: [PATCH 091/658] Updated run commands --- .github/workflows/test-mlperf-inference-mixtral.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index 9de8f2f70c..e09988f29d 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -28,4 +28,5 @@ jobs: cm pull repo - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions From c33cee3299850a498b2aef67a46cd8678e8be0e4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 7 Oct 2024 19:55:23 +0530 Subject: [PATCH 092/658] Fix behaviour of const --- automation/script/module.py | 36 ++++++++++++++++++---------- script/app-mlperf-inference/_cm.yaml | 10 ++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 9494e7a39a..09a16f0df3 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -838,8 +838,8 @@ def _run(self, i): script_artifact_env = meta.get('env',{}) env.update(script_artifact_env) - - + script_artifact_state = meta.get('state',{}) + utils.merge_dicts({'dict1':state, 'dict2':script_artifact_state, 'append_lists':True, 'append_unique':True}) @@ -853,7 +853,7 @@ def _run(self, i): # STEP 700: Overwrite env with keys from the script input (to allow user friendly CLI) - # IT HAS THE PRIORITY OVER meta['default_env'] and meta['env'] + # IT HAS THE PRIORITY OVER meta['default_env'] and meta['env'] but not over the meta from versions/variations # (env OVERWRITE - user enforces it from CLI) # (it becomes const) if input_mapping: @@ -866,7 +866,9 @@ def _run(self, i): # update_env_from_input_mapping(const, i, docker_input_mapping) - + #Update env/state with cost + env.update(const) + utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True}) @@ -952,7 +954,7 @@ def _run(self, i): if version!='' and version in versions: versions_meta = versions[version] - r = update_state_from_meta(versions_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) + r = update_state_from_meta(versions_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) if r['return']>0: return r adr=get_adr(versions_meta) if adr: @@ -1328,7 +1330,7 @@ def _run(self, i): if default_version in versions: versions_meta = versions[default_version] - r = update_state_from_meta(versions_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) + r = update_state_from_meta(versions_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) if r['return']>0: return r if "add_deps_recursive" in versions_meta: @@ -1453,6 +1455,7 @@ def _run(self, i): r = _update_env(env, 'CM_TMP_PIP_VERSION_STRING', pip_version_string) if r['return']>0: return r + if pip_version_string != '': logging.debug(recursion_spaces+' # potential PIP version string (if needed): '+pip_version_string) @@ -1462,10 +1465,6 @@ def _run(self, i): logging.debug(recursion_spaces+' - Running preprocess ...') - # Update env and state with const - utils.merge_dicts({'dict1':env, 'dict2':const, 'append_lists':True, 'append_unique':True}) - utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True}) - run_script_input['run_state'] = run_state ii = copy.deepcopy(customize_common_input) @@ -2019,7 +2018,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env if variation_tag_dynamic_suffix: self._update_variation_meta_with_dynamic_suffix(variation_meta, variation_tag_dynamic_suffix) - r = update_state_from_meta(variation_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) + r = update_state_from_meta(variation_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) if r['return']>0: return r if variation_meta.get('script_name', '')!='': @@ -2050,7 +2049,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env combined_variation_meta = variations[combined_variation] - r = update_state_from_meta(combined_variation_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) + r = update_state_from_meta(combined_variation_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i) if r['return']>0: return r adr=get_adr(combined_variation_meta) @@ -5110,7 +5109,7 @@ def update_env_from_input_mapping(env, inp, input_mapping): env[input_mapping[key]] = inp[key] ############################################################################## -def update_state_from_meta(meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys, new_state_keys, i): +def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys, new_state_keys, i): """ Internal: update env and state from meta """ @@ -5118,12 +5117,23 @@ def update_state_from_meta(meta, env, state, deps, post_deps, prehook_deps, post default_env = meta.get('default_env',{}) for key in default_env: env.setdefault(key, default_env[key]) + update_env = meta.get('env', {}) env.update(update_env) + update_const = meta.get('const', {}) + if update_const: + const.update(update_const) + env.update(const) + update_state = meta.get('state', {}) utils.merge_dicts({'dict1':state, 'dict2':update_state, 'append_lists':True, 'append_unique':True}) + update_const_state = meta.get('const_state', {}) + if const_state: + utils.merge_dicts({'dict1':const_state, 'dict2':update_const_state, 'append_lists':True, 'append_unique':True}) + utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True}) + new_deps = meta.get('deps', []) if len(new_deps)>0: append_deps(deps, new_deps) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 0eb89752bc..c5cad1a5f0 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -658,6 +658,8 @@ variations: tags: run,accuracy,mlperf,_kits19,_int8 3d-unet_,reference: + env: + CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-3d-unet docker: deps: - enable_if_env: @@ -697,6 +699,8 @@ variations: tags: get,ml-model,sdxl,_fp16,_rclone sdxl,reference,float16: + env: + CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-sdxl-float16 docker: deps: - enable_if_env: @@ -705,6 +709,8 @@ variations: tags: get,ml-model,sdxl,_fp16,_rclone sdxl,reference,bfloat16: + env: + CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-sdxl-bfloat16 docker: deps: - enable_if_env: @@ -713,6 +719,8 @@ variations: tags: get,ml-model,sdxl,_fp16,_rclone sdxl,reference,float32: + env: + CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-sdxl-float32 docker: deps: - enable_if_env: @@ -764,6 +772,8 @@ variations: tags: _llama2-70b-99.9 llama2-70b_,reference: + env: + CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-llama2-70b docker: deps: - enable_if_env: From a1fa5b93e14aae7431e373bc4a6b7070c616c540 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 7 Oct 2024 21:27:54 +0530 Subject: [PATCH 093/658] Testing docker image name for scc24 --- automation/script/module.py | 8 ++------ automation/script/module_misc.py | 8 ++++++-- script/app-mlperf-inference/_cm.yaml | 15 +++++---------- script/run-mlperf-inference-app/_cm.yaml | 5 ++++- script/run-mlperf-inference-app/customize.py | 10 ++++++++-- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 09a16f0df3..1eee8b48b2 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -413,10 +413,6 @@ def _run(self, i): ignore_script_error = i.get('ignore_script_error', False) - # Get constant env and state - const = i.get('const',{}) - const_state = i.get('const_state',{}) - # Detect current path and record in env for further use in native scripts current_path = os.path.abspath(os.getcwd()) r = _update_env(env, 'CM_TMP_CURRENT_PATH', current_path) @@ -884,7 +880,7 @@ def _run(self, i): variations = script_artifact.meta.get('variations', {}) state['docker'] = meta.get('docker', {}) - r = self._update_state_from_variations(i, meta, variation_tags, variations, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose) + r = self._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose) if r['return'] > 0: return r @@ -1915,7 +1911,7 @@ def _dump_version_info_for_script(self, output_dir = os.getcwd(), quiet = False, return {'return': 0} ###################################################################################### - def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose): + def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose): # Save current explicit variations import copy diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 54ab2dd489..91ce181af5 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1393,6 +1393,8 @@ def dockerfile(i): env=i.get('env', {}) state = i.get('state', {}) + const=i.get('const', {}) + const_state = i.get('const_state', {}) script_automation = i['self_module'] dockerfile_env=i.get('dockerfile_env', {}) @@ -1420,7 +1422,7 @@ def dockerfile(i): state['docker'] = docker_settings add_deps_recursive = i.get('add_deps_recursive', {}) - r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False) + r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False) if r['return'] > 0: return r @@ -1741,6 +1743,8 @@ def docker(i): env['CM_RUN_STATE_DOCKER'] = False script_automation = i['self_module'] state = i.get('state', {}) + const = i.get('const', {}) + const_state = i.get('const_state', {}) tags_split = i.get('tags', '').split(",") variation_tags = [ t[1:] for t in tags_split if t.startswith("_") ] @@ -1793,7 +1797,7 @@ def docker(i): state['docker'] = docker_settings add_deps_recursive = i.get('add_deps_recursive', {}) - r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False) + r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False) if r['return'] > 0: return r diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index c5cad1a5f0..8d74a8f454 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -658,9 +658,8 @@ variations: tags: run,accuracy,mlperf,_kits19,_int8 3d-unet_,reference: - env: - CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-3d-unet docker: + image_name: mlperf-inference-mlcommons-python-implementation-3d-unet deps: - enable_if_env: CM_MLPERF_DATASET_3DUNET_DOWNLOAD_TO_HOST: @@ -699,9 +698,8 @@ variations: tags: get,ml-model,sdxl,_fp16,_rclone sdxl,reference,float16: - env: - CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-sdxl-float16 docker: + image_name: mlperf-inference-mlcommons-python-implementation-sdxl-float16 deps: - enable_if_env: CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST: @@ -709,9 +707,8 @@ variations: tags: get,ml-model,sdxl,_fp16,_rclone sdxl,reference,bfloat16: - env: - CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-sdxl-bfloat16 docker: + image_name: mlperf-inference-mlcommons-python-implementation-sdxl-bfloat16 deps: - enable_if_env: CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST: @@ -719,9 +716,8 @@ variations: tags: get,ml-model,sdxl,_fp16,_rclone sdxl,reference,float32: - env: - CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-sdxl-float32 docker: + image_name: mlperf-inference-mlcommons-python-implementation-sdxl-float32 deps: - enable_if_env: CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST: @@ -772,9 +768,8 @@ variations: tags: _llama2-70b-99.9 llama2-70b_,reference: - env: - CM_DOCKER_IMAGE_NAME: mlperf-inference-mlcommons-python-implementation-llama2-70b docker: + image_name: mlperf-inference-mlcommons-python-implementation-llama2-70b deps: - enable_if_env: CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index cefdf55d40..1054a1bb6d 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -140,7 +140,8 @@ deps: - tags: install,pip-package,for-cmind-python,_package.tabulate - tags: get,mlperf,inference,utils -docker: +#We use this script as a command generator to run docker via app-mlperf-inference script +docker_off: mounts: - ${{ INSTALL_DATA_PATH }}:/install_data - ${{ DATA_PATH }}:/data @@ -248,6 +249,7 @@ variations: - short env: CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX4: scc24-base + CM_DOCKER_IMAGE_NAME: scc24 adr: coco2014-preprocessed: tags: _size.50,_with-sample-ids @@ -271,6 +273,7 @@ variations: extra_cache_tags: "scc24-main" env: CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX4: scc24-main + CM_DOCKER_IMAGE_NAME: scc24 deps: - tags: clean,nvidia,scratch,_sdxl,_downloaded-data extra_cache_rm_tags: scc24-base diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index e2361f2dee..7b60e779ec 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -14,6 +14,7 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + const = i.get('const', {}) inp = i['input'] state = i['state'] @@ -194,6 +195,9 @@ def preprocess(i): if k.startswith("docker_"): docker_extra_input[k] = inp[k] inp = {} + + if env.get('CM_DOCKER_IMAGE_NAME', '') != '': + docker_extra_input['docker_image_name'] = env['CM_DOCKER_IMAGE_NAME'] else: action = "run" @@ -220,14 +224,16 @@ def preprocess(i): env['CM_MLPERF_LOADGEN_MODE'] = mode env_copy = copy.deepcopy(env) + const_copy = copy.deepcopy(const) print(f"\nRunning loadgen scenario: {scenario} and mode: {mode}") ii = {'action':action, 'automation':'script', 'tags': scenario_tags, 'quiet': 'true', - 'env': env_copy, 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': + 'env': env_copy, 'const': const_copy, 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': copy.deepcopy(add_deps_recursive), 'ad': ad, 'adr': copy.deepcopy(adr), 'v': verbose, 'print_env': print_env, 'print_deps': print_deps, 'dump_version_info': dump_version_info} if action == "docker": for k in docker_extra_input: ii[k] = docker_extra_input[k] + r = cm.access(ii) if r['return'] > 0: return r @@ -252,7 +258,7 @@ def preprocess(i): env['CM_MLPERF_LOADGEN_COMPLIANCE_TEST'] = test env['CM_MLPERF_LOADGEN_MODE'] = "compliance" ii = {'action':action, 'automation':'script', 'tags': scenario_tags, 'quiet': 'true', - 'env': copy.deepcopy(env), 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': + 'env': copy.deepcopy(env), 'const': copy.deepcopy(const), 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': copy.deepcopy(add_deps_recursive), 'adr': copy.deepcopy(adr), 'ad': ad, 'v': verbose, 'print_env': print_env, 'print_deps': print_deps, 'dump_version_info': dump_version_info} if action == "docker": for k in docker_extra_input: From 7b5d4adf3ffeb95a118a2100e13550127faec62c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 00:14:35 +0530 Subject: [PATCH 094/658] Fix const corruption from deps --- automation/script/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 1eee8b48b2..6db2c7960e 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3007,8 +3007,8 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a 'remembered_selections': remembered_selections, 'env':env, 'state':state, - 'const':const, - 'const_state':const_state, + 'const':copy.deepcopy(const), + 'const_state':copy.deepcopy(const_state), 'add_deps_recursive':add_deps_recursive, 'debug_script_tags':debug_script_tags, 'verbose':verbose, From e4a8a2aa24900a64ebd9027f02149cd240f228e5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 01:05:57 +0530 Subject: [PATCH 095/658] Fix unnecessary bool->str conversion --- automation/script/module.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 6db2c7960e..b2b527b218 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1372,7 +1372,6 @@ def _run(self, i): r = update_env_with_values(env) if r['return']>0: return r - # Clean some output files clean_tmp_files(clean_files, recursion_spaces) @@ -1449,6 +1448,9 @@ def _run(self, i): elif pip_version_max != '': pip_version_string = '<='+pip_version_max + env.update(const) + utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True}) + r = _update_env(env, 'CM_TMP_PIP_VERSION_STRING', pip_version_string) if r['return']>0: return r @@ -3035,6 +3037,11 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a r = update_env_with_values(env) if r['return']>0: return r + #Update env/state with cost + env.update(const) + utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True}) + + return {'return': 0} ############################################################################## @@ -4413,7 +4420,7 @@ def update_env_with_values(env, fail_on_not_found=False, extra_env={}): # Check cases such as --env.CM_SKIP_COMPILE if type(value)==bool: - env[key] = str(value) + env[key] = value continue tmp_values = re.findall(r'<<<(.*?)>>>', str(value)) From 587622809c2c2e0659524cd46b6be144ca8c0e30 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 01:56:52 +0530 Subject: [PATCH 096/658] Use different docker image names for SCC nvidia/reference --- script/run-mlperf-inference-app/customize.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 7b60e779ec..0aef8551c2 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -23,6 +23,12 @@ def preprocess(i): if env.get('CM_RUN_DOCKER_CONTAINER', '') == "yes": return {'return':0} + if env.get('CM_DOCKER_IMAGE_NAME', '') == 'scc24': + if env.get("CM_MLPERF_IMPLEMENTATION", "reference") == "reference": + env['CM_DOCKER_IMAGE_NAME'] = "scc24-reference" + elif "nvidia" in env.get("CM_MLPERF_IMPLEMENTATION", "reference"): + env['CM_DOCKER_IMAGE_NAME'] = "scc24-nvidia" + dump_version_info = env.get('CM_DUMP_VERSION_INFO', True) system_meta = state.get('CM_SUT_META', {}) From 0d06290ff23fc8ee6cc57c738dc38476b09f0ab8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 02:18:05 +0530 Subject: [PATCH 097/658] Improve cm tests --- tests/script/check.py | 10 ++++++++++ tests/script/test_features.py | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/script/check.py b/tests/script/check.py index 7394406d8f..aba7f7831c 100644 --- a/tests/script/check.py +++ b/tests/script/check.py @@ -12,3 +12,13 @@ def check_list(r, string, found=True): raise Exception('CM search returned an empty list for ' + string) if len(r['list']) > 0 and not found: raise Exception('CM search returned at lease one entry for ' + string) + +def check_key_value(d, key, value, absent_ok=False): + if not d.get(key): + if absent_ok: + return True + else: + raise Exception(f"{key} is missing. Current values are {d}") + elif d[key] != value: + raise Exception(f"{key} is not having the expected value of {value}. Current value is {d[key]}") + diff --git a/tests/script/test_features.py b/tests/script/test_features.py index d116cbd5bf..0679099321 100644 --- a/tests/script/test_features.py +++ b/tests/script/test_features.py @@ -18,4 +18,10 @@ checks.check_list(r, "_NHWC") r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) -checks.check_list(r, "_NHWC", False) +#checks.check_list(r, "-_NHWC", False) + + +r = cm.access({'action':'run', 'automation': 'script', 'tags': 'test-scripts,_v1,_v2'}) +new_env = r['new_env'] +checks.check_key_value(new_env, "CM_VAR1", "combv1v2") +checks.check_key_value(new_env, "CM_VAR2", "constv2") From d9fa259a9a0ee541d34b4a7f2beafd95a1381c0e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 02:18:54 +0530 Subject: [PATCH 098/658] Added test-cm-scripts script --- script/test-cm-scripts/_cm.yaml | 31 +++++++++++++++++++++++++++++ script/test-cm-scripts/customize.py | 22 ++++++++++++++++++++ script/test-cm-scripts/run.bat | 1 + script/test-cm-scripts/run.sh | 17 ++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 script/test-cm-scripts/_cm.yaml create mode 100644 script/test-cm-scripts/customize.py create mode 100644 script/test-cm-scripts/run.bat create mode 100644 script/test-cm-scripts/run.sh diff --git a/script/test-cm-scripts/_cm.yaml b/script/test-cm-scripts/_cm.yaml new file mode 100644 index 0000000000..b8b7a9c791 --- /dev/null +++ b/script/test-cm-scripts/_cm.yaml @@ -0,0 +1,31 @@ +alias: test-cm-scripts +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: false +tags: +- test +- test-scripts +- cm-test +env: + CM_VAR1: orig +const: + CM_CVAR1: orig +new_env_keys: + - CM_VAR* + - CM_CVAR* + +uid: 6fbe3884575c4e51 +variations: + v1: + env: + CM_VAR1: v1 + v2: + env: + CM_VAR1: v2 + CM_VAR2: v2 + const: + CM_VAR2: constv2 + v1,v2: + env: + CM_VAR1: combv1v2 + CM_VAR2: combv1v2 diff --git a/script/test-cm-scripts/customize.py b/script/test-cm-scripts/customize.py new file mode 100644 index 0000000000..d12f9b3e1d --- /dev/null +++ b/script/test-cm-scripts/customize.py @@ -0,0 +1,22 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + return {'return':0} diff --git a/script/test-cm-scripts/run.bat b/script/test-cm-scripts/run.bat new file mode 100644 index 0000000000..648302ca71 --- /dev/null +++ b/script/test-cm-scripts/run.bat @@ -0,0 +1 @@ +rem native script diff --git a/script/test-cm-scripts/run.sh b/script/test-cm-scripts/run.sh new file mode 100644 index 0000000000..4c23c380ea --- /dev/null +++ b/script/test-cm-scripts/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency + +echo "Running: " +echo "${CM_RUN_CMD}" +echo "" + +if [[ ${CM_FAKE_RUN} != "yes" ]]; then + eval "${CM_RUN_CMD}" + test $? -eq 0 || exit 1 +fi From 008de67521caf33a14293c66d8f8aba38122e3b5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 22:42:15 +0530 Subject: [PATCH 099/658] Support custom_system_nvidia run option --- script/run-mlperf-inference-app/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 1054a1bb6d..54aa957754 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -43,6 +43,7 @@ input_mapping: category: CM_MLPERF_SUBMISSION_SYSTEM_TYPE clean: CM_MLPERF_CLEAN_ALL compliance: CM_MLPERF_LOADGEN_COMPLIANCE + custom_system_nvidia: CM_CUSTOM_SYSTEM_NVIDIA dashboard_wb_project: CM_MLPERF_DASHBOARD_WANDB_PROJECT dashboard_wb_user: CM_MLPERF_DASHBOARD_WANDB_USER debug: CM_DEBUG_SCRIPT_BENCHMARK_PROGRAM From 9f6fdc1dfc517cb6e9e366923e1fc3886860f61f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 18:53:58 +0100 Subject: [PATCH 100/658] Update test-mlperf-inference-dlrm.yml --- .../workflows/test-mlperf-inference-dlrm.yml | 34 +++---------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index 27025acde6..9357481a77 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -10,7 +10,7 @@ on: jobs: build_reference: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, GO-i9, linux, x64 ] + runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false matrix: @@ -19,42 +19,18 @@ jobs: device: [ "cpu", "cuda" ] steps: - - name: Install dependencies - run: | - source gh_action/bin/deactivate || python3 -m venv gh_action - source gh_action/bin/activate - export CM_REPOS=$HOME/GH_CM - python3 -m pip install cm4mlops - cm pull repo - name: Test MLPerf Inference DLRM-v2 reference implementation - run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean - - build_nvidia: - if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, GO-i9, linux, x64 ] - strategy: - fail-fast: false - matrix: - python-version: [ "3.12" ] - backend: [ "tensorrt" ] - device: [ "cuda" ] - - steps: - - name: Install dependencies run: | source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - - name: Test MLPerf Inference DLRM-v2 NVIDIA implementation - run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=nvidia --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean build_intel: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, GO-i9, linux, x64 ] + runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false matrix: @@ -63,13 +39,11 @@ jobs: device: [ "cpu" ] steps: - - name: Install dependencies + - name: Test MLPerf Inference DLRM-v2 INTEL implementation run: | source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - - name: Test MLPerf Inference DLRM-v2 INTEL implementation - run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean From 7b8cacf9a167e8656b6cef9a47f4f7860cf68ba9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 18:55:49 +0100 Subject: [PATCH 101/658] Update test-mlperf-inference-mixtral.yml --- .github/workflows/test-mlperf-inference-mixtral.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index e09988f29d..d7a358ae13 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -19,14 +19,12 @@ jobs: device: [ "cpu", "cuda" ] steps: - - name: Install dependencies + - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation run: | source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation - run: | cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions From 17114558ec5c24995158d14d8ec6780fd95eab96 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 8 Oct 2024 23:41:28 +0530 Subject: [PATCH 102/658] removed cuda device --- .github/workflows/test-mlperf-inference-mixtral.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index d7a358ae13..b5716e78f8 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -16,7 +16,7 @@ jobs: matrix: python-version: [ "3.12" ] backend: [ "pytorch" ] - device: [ "cpu", "cuda" ] + device: [ "cpu" ] steps: - name: Test MLPerf Inference MIXTRAL-8X7B reference implementation From 3be017a8f145510ddee9056d64ea251ca1c34030 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 00:12:09 +0530 Subject: [PATCH 103/658] Use --custom_system_nvidia for nvidia runs --- .github/workflows/test-mlperf-inference-sdxl.yaml | 2 +- .github/workflows/test-scc24-sdxl.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 2b616e9603..a43fc5791b 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -22,5 +22,5 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --custom_system_nvidia=yes --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 680d0f5f43..b2ae1e1847 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -52,7 +52,7 @@ jobs: pip install --upgrade cm4mlops pip install tabulate cm pull repo - cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --clean + cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes--clean cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions From 13d0330c945717952e0873e838c3bf8c59c132f8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 00:12:35 +0530 Subject: [PATCH 104/658] Use --custom_system_nvidia for nvidia runs --- .github/workflows/test-mlperf-inference-gptj.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml index 1c59dea469..ed1a50aa03 100644 --- a/.github/workflows/test-mlperf-inference-gptj.yml +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -28,6 +28,6 @@ jobs: cm pull repo - name: Test MLPerf Inference GPTJ run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --custom_system_nvidia=yes --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From dcf49170e31720bbeaee0209d273c1c1b8571f07 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 19:48:22 +0100 Subject: [PATCH 105/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index b2ae1e1847..028d5ea6c4 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -52,7 +52,7 @@ jobs: pip install --upgrade cm4mlops pip install tabulate cm pull repo - cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes--clean + cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes --clean cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions From a1471b7c422f3f873f292025974089cbcbb01a8d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 01:07:24 +0530 Subject: [PATCH 106/658] Fix CUDA num_devices --- script/get-cuda-devices/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-cuda-devices/customize.py b/script/get-cuda-devices/customize.py index 4184d3b54b..242044e7e8 100644 --- a/script/get-cuda-devices/customize.py +++ b/script/get-cuda-devices/customize.py @@ -53,8 +53,8 @@ def postprocess(i): key_env = 'CM_CUDA_DEVICE_PROP_'+key.upper().replace(' ','_') env[key_env] = val - state['cm_cuda_num_devices'] = gpu_id - env['CM_CUDA_NUM_DEVICES'] = gpu_id + state['cm_cuda_num_devices'] = gpu_id + 1 + env['CM_CUDA_NUM_DEVICES'] = gpu_id + 1 state['cm_cuda_device_prop'] = p state['cm_cuda_devices_prop'] = gpu From 34b9c2cfd53e46b53240ff617104e86d07031a5b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 8 Oct 2024 22:39:47 +0100 Subject: [PATCH 107/658] Support cleaning of Nvidia SDXL model --- .../clean-nvidia-mlperf-inference-scratch-space/customize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py index 5a0a95e765..977a9993bb 100644 --- a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py +++ b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py @@ -25,6 +25,9 @@ def preprocess(i): if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'preprocessed_data': clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "preprocessed_data", "coco2014-tokenized-sdxl")} """ cache_rm_tags = "nvidia-harness,_preprocess_data,_sdxl" + if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'downloaded_model': + clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "models", "SDXL")} """ + cache_rm_tags = "nvidia-harness,_download_model,_sdxl" cache_rm_tags = cache_rm_tags + extra_cache_rm_tags From 387dff3c3dbd958ff54abd8ba75770574b7a35cf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 09:46:09 +0100 Subject: [PATCH 108/658] Update test-mlperf-inference-dlrm.yml --- .github/workflows/test-mlperf-inference-dlrm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index 9357481a77..3247806656 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -9,14 +9,14 @@ on: jobs: build_reference: - if: github.repository_owner == 'gateoverflow' + if: github.repository_owner == 'gateoverflow_off' runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false matrix: python-version: [ "3.12" ] backend: [ "pytorch" ] - device: [ "cpu", "cuda" ] + device: [ "cpu" ] steps: - name: Test MLPerf Inference DLRM-v2 reference implementation @@ -29,7 +29,7 @@ jobs: cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean build_intel: - if: github.repository_owner == 'gateoverflow' + if: github.repository_owner == 'gateoverflow_off' runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false From 0efa8f35d073e68543317b04ee7e5487f88af5cc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 11:31:08 +0100 Subject: [PATCH 109/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index a43fc5791b..d4347dfac9 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -22,5 +22,5 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --custom_system_nvidia=yes --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 45d38b3d40a0fc883509a1be58941b60e9fd3102 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 16:28:17 +0530 Subject: [PATCH 110/658] Support nvidia_system_name in run-mlperf-inference-app --- script/run-mlperf-inference-app/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 54aa957754..7a09a3afbe 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -67,6 +67,7 @@ input_mapping: model: CM_MLPERF_MODEL multistream_target_latency: CM_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY network: CM_NETWORK_LOADGEN + nvidia_system_name: CM_NVIDIA_SYSTEM_NAME offline_target_qps: CM_MLPERF_LOADGEN_OFFLINE_TARGET_QPS output_dir: OUTPUT_BASE_DIR output_summary: MLPERF_INFERENCE_SUBMISSION_SUMMARY From 490a2df3ee43c7860884a51bb18cea2eb4b87619 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 18:04:32 +0530 Subject: [PATCH 111/658] Support pulling of code changes for Nvidia implementation --- script/app-mlperf-inference-nvidia/_cm.yaml | 7 +++++++ script/process-mlperf-accuracy/customize.py | 2 +- script/pull-git-repo/run.sh | 4 ++-- script/run-mlperf-inference-app/_cm.yaml | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 15d6e45193..71f9a6e94e 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -242,6 +242,13 @@ deps: names: - nvidia-inference-common-code + - tags: pull,git,repo + env: + CM_GIT_CHECKOUT_PATH: '<<>>' + enable_if_env: + CM_MLPERF_INFERENCE_PULL_CODE_CHANGES: + 'yes' + # Creates user conf for given SUT - tags: generate,user-conf,mlperf,inference names: diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 765f96a353..e83482258b 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -101,7 +101,7 @@ def preprocess(i): if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': extra_options += f" --compliance-images-path '{env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " else: - extra_options += f" --compliance-images-path {os.path.join(result_dir, 'images')} " + extra_options += f""" --compliance-images-path '{os.path.join(result_dir, "images")'} """ if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': extra_options += f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " diff --git a/script/pull-git-repo/run.sh b/script/pull-git-repo/run.sh index 66cf8406f0..34b83cc63e 100644 --- a/script/pull-git-repo/run.sh +++ b/script/pull-git-repo/run.sh @@ -7,10 +7,10 @@ path=${CM_GIT_CHECKOUT_PATH} echo "cd $path" cd $path -test $? -eq 0 || exit 1 +test $? -eq 0 || exit $? echo ${CM_GIT_PULL_CMD} eval ${CM_GIT_PULL_CMD} -test $? -eq 0 || exit 1 +test $? -eq 0 || exit $? cd $CUR_DIR diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 7a09a3afbe..497919093c 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -77,6 +77,7 @@ input_mapping: precision: CM_MLPERF_MODEL_PRECISION preprocess_submission: CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR push_to_github: CM_MLPERF_RESULT_PUSH_TO_GITHUB + pull_changes: CM_MLPERF_INFERENCE_PULL_CODE_CHANGES readme: CM_MLPERF_README regenerate_accuracy_file: CM_MLPERF_REGENERATE_ACCURACY_FILE regenerate_files: CM_REGENERATE_MEASURE_FILES From 61b098a6540c9ce3483fd261f344cec08e35c8d2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 18:07:56 +0530 Subject: [PATCH 112/658] Fix typo --- script/process-mlperf-accuracy/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index e83482258b..72e7aec9e8 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -101,7 +101,7 @@ def preprocess(i): if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': extra_options += f" --compliance-images-path '{env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " else: - extra_options += f""" --compliance-images-path '{os.path.join(result_dir, "images")'} """ + extra_options += f""" --compliance-images-path '{os.path.join(result_dir, "images")}' """ if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': extra_options += f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " From d81510b7809e94b708659393462d9066256a1e83 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 18:12:57 +0530 Subject: [PATCH 113/658] Fix typo --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 71f9a6e94e..09820e5683 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -247,7 +247,7 @@ deps: CM_GIT_CHECKOUT_PATH: '<<>>' enable_if_env: CM_MLPERF_INFERENCE_PULL_CODE_CHANGES: - 'yes' + - 'yes' # Creates user conf for given SUT - tags: generate,user-conf,mlperf,inference From f26f37db604032b61c426a9d088167d860e3b87e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 13:45:00 +0100 Subject: [PATCH 114/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 028d5ea6c4..673577f29d 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -52,7 +52,7 @@ jobs: pip install --upgrade cm4mlops pip install tabulate cm pull repo - cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes --clean + cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --pull_changes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes --clean cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions From 50431589d01cd60060a0d1be3c1f8193d99c6144 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 16:48:59 +0100 Subject: [PATCH 115/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 673577f29d..9b8954ceff 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -2,7 +2,7 @@ name: MLPerf inference SDXL (SCC) on: schedule: - - cron: "1 3 * * *" + - cron: "55 15 * * *" jobs: build_reference: From 5f2784b547fd05053ae792f9ab0ab121a04e97f6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 19:05:40 +0100 Subject: [PATCH 116/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 9b8954ceff..40f563f460 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -2,7 +2,7 @@ name: MLPerf inference SDXL (SCC) on: schedule: - - cron: "55 15 * * *" + - cron: "5 2 * * *" jobs: build_reference: From 55e3784ed4938537b312a1948a5f2594cd27037b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 01:11:35 +0530 Subject: [PATCH 117/658] Fixed criteo preprocessed mount for reference implementation --- script/app-mlperf-inference/_cm.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 8d74a8f454..92aac8c4ac 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -985,11 +985,15 @@ variations: docker: deps: - tags: get,dlrm,data,mlperf,inference,_nvidia + mounts: + - "${{ DLRM_DATA_PATH }}:/home/mlperf_inf_dlrmv2" dlrm_,intel: docker: deps: - tags: get,preprocessed,dataset,criteo,_mlc + mounts: + - "${{ DLRM_DATA_PATH }}:${{ DLRM_DATA_PATH }}" dlrm_,reference: docker: @@ -997,7 +1001,8 @@ variations: - tags: get,preprocessed,dataset,criteo,_mlc - tags: get,ml-model,dlrm,_pytorch,_fp32 mounts: - - ${{ CM_ML_MODEL_FILE_WITH_PATH }}:${{ CM_ML_MODEL_FILE_WITH_PATH }} + - "${{ CM_ML_MODEL_FILE_WITH_PATH }}:${{ CM_ML_MODEL_FILE_WITH_PATH }}" + - "${{ DLRM_DATA_PATH }}:${{ DLRM_DATA_PATH }}" dockerfile_env: CM_ML_MODEL_FILE_WITH_PATH: "on" @@ -1626,7 +1631,6 @@ docker: - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" - "${{ CM_CRITEO_PREPROCESSED_PATH }}:${{ CM_CRITEO_PREPROCESSED_PATH }}" - "${{ LLAMA2_CHECKPOINT_PATH }}:${{ LLAMA2_CHECKPOINT_PATH }}" - - "${{ DLRM_DATA_PATH }}:/home/mlperf_inf_dlrmv2" - "${{ CM_NVIDIA_LLAMA_DATASET_FILE_PATH }}:${{ CM_NVIDIA_LLAMA_DATASET_FILE_PATH }}" - "${{ SDXL_CHECKPOINT_PATH }}:${{ SDXL_CHECKPOINT_PATH }}" - "${{ CM_DATASET_KITS19_PREPROCESSED_PATH }}:${{ CM_DATASET_KITS19_PREPROCESSED_PATH }}" From fefe028d331c0b65c4586f72295baa2e34aeeea2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 21:07:33 +0100 Subject: [PATCH 118/658] Update test-mlperf-inference-dlrm.yml | Enable dlrm reference run --- .github/workflows/test-mlperf-inference-dlrm.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index 3247806656..5024336aef 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -5,17 +5,16 @@ name: MLPerf inference DLRM-v2 on: schedule: - - cron: "30 21 * * *" + - cron: "30 1 * * *" jobs: build_reference: - if: github.repository_owner == 'gateoverflow_off' + if: github.repository_owner == 'gateoverflow' runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false matrix: python-version: [ "3.12" ] - backend: [ "pytorch" ] device: [ "cpu" ] steps: @@ -26,7 +25,7 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean + cm run script --tags=run-mlperf,inference,_performance-only --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --backend=pytorch --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --clean build_intel: if: github.repository_owner == 'gateoverflow_off' From 165a1fd04337790b197dc5d3abd6161ebd0a5d11 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 02:41:19 +0530 Subject: [PATCH 119/658] Support cxx11-abi for torch --- script/get-generic-python-lib/_cm.json | 20 +++++++++++++++++--- script/get-generic-python-lib/customize.py | 1 + 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/script/get-generic-python-lib/_cm.json b/script/get-generic-python-lib/_cm.json index b3757091c1..19b6f41441 100644 --- a/script/get-generic-python-lib/_cm.json +++ b/script/get-generic-python-lib/_cm.json @@ -874,12 +874,26 @@ "deps": [ { "tags": "get,generic-python-lib,_package.networkx", - "enable_if_env": { - "CM_PYTHON_MINOR_VERSION": [ "7", "8" ] - } + "enable_if_env": { + "CM_PYTHON_MINOR_VERSION": [ "7", "8" ] + } } ] }, + "cxx11-abi": { + "env": { + } + }, + "torch,cxx11-abi": { + "env": { + "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL": "https://download.pytorch.org/whl/nightly/cpu-cxx11-abi" + } + }, + "package.torch,cxx11-abi": { + "env": { + "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/nightly/cpu-cxx11-abi" + } + }, "torch,pre": { "default_env": { "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torch" diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index 18ae0149aa..e0e30c7f67 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -78,6 +78,7 @@ def preprocess(i): # Check extra index URL extra_index_url = env.get('CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL','').strip() + if extra_index_url != '': # Check special cases if '${CM_TORCH_CUDA}' in extra_index_url: From 52f2ebb1436d6e255d60a3dc0acf99e4f24f492d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 03:49:07 +0530 Subject: [PATCH 120/658] Build torch whl file for intel mlperf inference dlrm --- script/app-mlperf-inference-intel/_cm.yaml | 8 ++++---- script/install-ipex-from-src/_cm.json | 6 +++--- script/install-pytorch-from-src/_cm.json | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/script/app-mlperf-inference-intel/_cm.yaml b/script/app-mlperf-inference-intel/_cm.yaml index 4f0cdbacec..0975f0b0b6 100644 --- a/script/app-mlperf-inference-intel/_cm.yaml +++ b/script/app-mlperf-inference-intel/_cm.yaml @@ -940,10 +940,10 @@ variations: names: - pip-package - accelerate - - tags: get,generic-python-lib,_package.torch,_path.https://download.pytorch.org/whl/nightly/cpu-cxx11-abi/torch-2.1.0.dev20230715%2Bcpu.cxx11.abi-cp39-cp39-linux_x86_64.whl - names: - - pip-package - - pip-torch + - tags: install,pytorch,from-src,_for-intel-mlperf-inference-v3.1-dlrm-v2 + names: + - pytorch + - torch dlrm-v2_: env: {} diff --git a/script/install-ipex-from-src/_cm.json b/script/install-ipex-from-src/_cm.json index e41efea533..7f2318c45a 100644 --- a/script/install-ipex-from-src/_cm.json +++ b/script/install-ipex-from-src/_cm.json @@ -214,10 +214,10 @@ "version": "1.23.5" }, { - "tags": "get,generic-python-lib,_package.torch,_path.https://download.pytorch.org/whl/nightly/cpu-cxx11-abi/torch-2.1.0.dev20230715%2Bcpu.cxx11.abi-cp39-cp39-linux_x86_64.whl", + "tags": "install,pytorch,from-src,_for-intel-mlperf-inference-v3.1-dlrm-v2", "names": [ - "pip-package", - "pip-torch" + "pytorch", + "torch" ] } ], diff --git a/script/install-pytorch-from-src/_cm.json b/script/install-pytorch-from-src/_cm.json index 9beb7e8c33..95693576de 100644 --- a/script/install-pytorch-from-src/_cm.json +++ b/script/install-pytorch-from-src/_cm.json @@ -63,6 +63,7 @@ "install", "get", "src", + "from-src", "from.src", "pytorch", "src-pytorch" @@ -419,6 +420,22 @@ } } }, + "for-intel-mlperf-inference-v3.1-dlrm-v2": { + "base": [ + "sha.927dc662386af052018212c7d01309a506fc94cd" + ], + "deps": [ + { + "tags": "get,cmake", + "version_min": "3.25.0" + } + ], + "ad": { + "pytorch-src-repo": { + "tags": "_no-recurse-submodules,_full-history" + } + } + }, "cuda": { "deps": [ { From c65367f6eadb2990cffd212f6c2ac405df0757e7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 09:55:45 +0100 Subject: [PATCH 121/658] Update test-mlperf-inference-gptj.yml --- .github/workflows/test-mlperf-inference-gptj.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml index ed1a50aa03..643b10115e 100644 --- a/.github/workflows/test-mlperf-inference-gptj.yml +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -19,15 +19,13 @@ jobs: precision: [ "float16" ] steps: - - name: Install dependencies + - name: Test MLPerf Inference GPTJ run: | source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - - name: Test MLPerf Inference GPTJ - run: | - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --custom_system_nvidia=yes --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --implementation=reference --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 9c75b3b0cb920216558cfa34485cfbe4b38a4d1a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 10:05:01 +0100 Subject: [PATCH 122/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index d4347dfac9..a285d4447c 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -23,4 +23,4 @@ jobs: python3 -m pip install cm4mlops cm pull repo cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From e1afd28f4b176c10f3402c9e5764138b311515d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 10:05:29 +0100 Subject: [PATCH 123/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index a285d4447c..1bde01a10a 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -2,7 +2,7 @@ name: MLPerf inference SDXL #off now as we have SCC24 test doing the same on: schedule: - - cron: "1 2 * * *" + - cron: "10 9 * * *" jobs: build_reference: From 594c4c74415af3b29642fe52e1732bf62623f201 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 10 Oct 2024 10:22:03 +0100 Subject: [PATCH 124/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 1bde01a10a..5b73cda70b 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -1,8 +1,7 @@ name: MLPerf inference SDXL -#off now as we have SCC24 test doing the same on: schedule: - - cron: "10 9 * * *" + - cron: "30 9 * * *" jobs: build_reference: From 4c76cf567fd78116e22db578a72ebbbfd9a46e03 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 17:38:10 +0530 Subject: [PATCH 125/658] Kill the docker container at the end of mlperf-run --- script/run-mlperf-inference-app/_cm.yaml | 1 + script/run-mlperf-inference-app/customize.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 497919093c..e08182194c 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -56,6 +56,7 @@ input_mapping: execution_mode: CM_MLPERF_RUN_STYLE find_performance: CM_MLPERF_FIND_PERFORMANCE_MODE framework: CM_MLPERF_BACKEND + docker_keep_alive: CM_DOCKER_CONTAINER_KEEP_ALIVE gpu_name: CM_NVIDIA_GPU_NAME hw_name: CM_HW_NAME pip_loadgen: CM_MLPERF_INFERENCE_LOADGEN_INSTALL_FROM_PIP diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 0aef8551c2..a60060edfb 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -275,6 +275,11 @@ def preprocess(i): if state.get('docker', {}): del(state['docker']) + if env.get('CM_DOCKER_CONTAINER_ID', '') != '' and str(env.get('CM_DOCKER_CONTAINER_KEEP_ALIVE', '')).lower() not in ["yes", "1", "true"]: + container_id = env['CM_DOCKER_CONTAINER_ID'] + CMD = f"docker kill {container_id}" + docker_out = subprocess.check_output(CMD, shell=True).decode("utf-8") + if state.get("cm-mlperf-inference-results"): #print(state["cm-mlperf-inference-results"]) for sut in state["cm-mlperf-inference-results"]:#only one sut will be there From 924fd3430f7ab8fa784306a3a2d9b8e057d2755c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 19:24:39 +0530 Subject: [PATCH 126/658] Invalidate old container ids --- script/run-docker-container/customize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 84114d7af5..a3971e4482 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -64,8 +64,10 @@ def preprocess(i): existing_container_id = out[0] env['CM_DOCKER_CONTAINER_ID'] = existing_container_id - else: + if env.get('CM_DOCKER_CONTAINER_ID', '') != '': + del(env['CM_DOCKER_CONTAINER_ID']) #not valid ID + CMD = "docker images -q " + DOCKER_CONTAINER if os_info['platform'] == 'windows': From 91955db3d5f7160249b5b7ca090086ea67927709 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 20:52:26 +0530 Subject: [PATCH 127/658] Support repo owner and branch in final inference report generation --- script/run-mlperf-inference-submission-checker/_cm.json | 4 +++- .../run-mlperf-inference-submission-checker/customize.py | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/script/run-mlperf-inference-submission-checker/_cm.json b/script/run-mlperf-inference-submission-checker/_cm.json index a8c7d5752d..ab57623799 100644 --- a/script/run-mlperf-inference-submission-checker/_cm.json +++ b/script/run-mlperf-inference-submission-checker/_cm.json @@ -77,7 +77,9 @@ "input": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", "submitter": "CM_MLPERF_SUBMITTER", "src_version": "CM_MLPERF_SUBMISSION_CHECKER_VERSION", - "repository": "CM_MLPERF_RESULTS_GIT_REPO_URL", + "repo_name": "CM_MLPERF_RESULTS_GIT_REPO_NAME", + "repo_owner": "CM_MLPERF_RESULTS_GIT_REPO_OWNER", + "repo_branch": "CM_MLPERF_RESULTS_GIT_REPO_BRANCH", "push_to_github": "CM_MLPERF_RESULT_PUSH_TO_GITHUB", "extra_model_benchmark_map": "CM_MLPERF_EXTRA_MODEL_MAPPING", "power": "CM_MLPERF_POWER", diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 5c863bfca3..586a58041e 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -61,7 +61,13 @@ def preprocess(i): skip_compliance + extra_map + power_check + extra_args x_version = ' --version ' + version[1:] +' ' if version!='' else '' - x_submission_repository = ' --repository ' + env.get('CM_MLPERF_RESULTS_GIT_REPO_URL', f'https://github.com/mlcommons/submissions_inference_results_{version}') + + if env.get('CM_MLPERF_RESULTS_GIT_REPO_NAME', '') != '': + x_submission_repo_name = f""" --repository {env.['CM_MLPERF_RESULTS_GIT_REPO_NAME']}""" + if env.get('CM_MLPERF_RESULTS_GIT_REPO_OWNER', '') != '': + x_submission_repo_owner = f""" --repository_owner {env['CM_MLPERF_RESULTS_GIT_REPO_OWNER']}""" + if env.get('CM_MLPERF_RESULTS_GIT_REPO_BRANCH', '') != '': + x_submission_repo_branch = f""" --repository_branch {env['CM_MLPERF_RESULTS_GIT_REPO_BRANCH']}""" report_generator_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission", "generate_final_report.py") From 3953aef25fafd3f1fe979b805c90fa9a81c2e8f4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 20:57:47 +0530 Subject: [PATCH 128/658] Support repo owner and branch in final inference report generation --- script/run-mlperf-inference-submission-checker/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 586a58041e..56c6de09c3 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -63,7 +63,7 @@ def preprocess(i): x_version = ' --version ' + version[1:] +' ' if version!='' else '' if env.get('CM_MLPERF_RESULTS_GIT_REPO_NAME', '') != '': - x_submission_repo_name = f""" --repository {env.['CM_MLPERF_RESULTS_GIT_REPO_NAME']}""" + x_submission_repo_name = f""" --repository {env['CM_MLPERF_RESULTS_GIT_REPO_NAME']}""" if env.get('CM_MLPERF_RESULTS_GIT_REPO_OWNER', '') != '': x_submission_repo_owner = f""" --repository_owner {env['CM_MLPERF_RESULTS_GIT_REPO_OWNER']}""" if env.get('CM_MLPERF_RESULTS_GIT_REPO_BRANCH', '') != '': From f11f1f689f2bb08d9b441c625c01526c370eeeb0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 21:02:32 +0530 Subject: [PATCH 129/658] Support repo owner and branch in final inference report generation --- .../run-mlperf-inference-submission-checker/customize.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 56c6de09c3..b0e85d79b0 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -62,6 +62,10 @@ def preprocess(i): x_version = ' --version ' + version[1:] +' ' if version!='' else '' + x_submission_repo_name = '' + x_submission_repo_owner = '' + x_submission_repo_branch = '' + if env.get('CM_MLPERF_RESULTS_GIT_REPO_NAME', '') != '': x_submission_repo_name = f""" --repository {env['CM_MLPERF_RESULTS_GIT_REPO_NAME']}""" if env.get('CM_MLPERF_RESULTS_GIT_REPO_OWNER', '') != '': @@ -75,7 +79,9 @@ def preprocess(i): print(CMD) env['CM_POST_RUN_CMD'] = env['CM_PYTHON_BIN_WITH_PATH'] +' ' + q + report_generator_file + q + ' --input summary.csv ' + \ x_version + \ - x_submission_repository + x_submission_repo_name + \ + x_submission_repo_owner + \ + x_submission_repo_branch return {'return':0} From 1766a88df58ec8c05dbe9ebca8e3ade67d63eb92 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 21:08:39 +0530 Subject: [PATCH 130/658] Support repo owner and branch in final inference report generation --- script/run-mlperf-inference-submission-checker/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index b0e85d79b0..1781385c59 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -69,9 +69,9 @@ def preprocess(i): if env.get('CM_MLPERF_RESULTS_GIT_REPO_NAME', '') != '': x_submission_repo_name = f""" --repository {env['CM_MLPERF_RESULTS_GIT_REPO_NAME']}""" if env.get('CM_MLPERF_RESULTS_GIT_REPO_OWNER', '') != '': - x_submission_repo_owner = f""" --repository_owner {env['CM_MLPERF_RESULTS_GIT_REPO_OWNER']}""" + x_submission_repo_owner = f""" --repository-owner {env['CM_MLPERF_RESULTS_GIT_REPO_OWNER']}""" if env.get('CM_MLPERF_RESULTS_GIT_REPO_BRANCH', '') != '': - x_submission_repo_branch = f""" --repository_branch {env['CM_MLPERF_RESULTS_GIT_REPO_BRANCH']}""" + x_submission_repo_branch = f""" --repository-branch {env['CM_MLPERF_RESULTS_GIT_REPO_BRANCH']}""" report_generator_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission", "generate_final_report.py") From bc8b552e5b927490dad73bd7f4b56b85ac7a165d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 17:54:34 +0100 Subject: [PATCH 131/658] Update test-mlperf-inference-gptj.yml --- .github/workflows/test-mlperf-inference-gptj.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml index 643b10115e..f2f9f41523 100644 --- a/.github/workflows/test-mlperf-inference-gptj.yml +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -26,6 +26,6 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --implementation=reference --clean + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --get_platform_details=yes --implementation=reference --clean cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 551bd0806a237ec66351ac82e3c1aa3d83979bbb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 22:25:06 +0530 Subject: [PATCH 132/658] Support get-platform-details for mlperf-inference --- script/app-mlperf-inference/_cm.yaml | 8 ++++++++ script/get-platform-details/customize.py | 5 ++++- script/run-mlperf-inference-app/_cm.yaml | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 92aac8c4ac..c65723b600 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -1052,6 +1052,14 @@ variations: - mlperf-accuracy-script - imagenet-accuracy-script tags: run,accuracy,mlperf,_imagenet + - tags: get,platform,details + enable_if_any_env: + CM_SUDO_USER: + - yes + CM_GET_PLATFORM_DETAILS: + - yes + env: + CM_PLATFORM_DETAILS_FILE_PATH: '<<>>/system_info.txt' onnxruntime: group: backend diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index d7affa2608..e3b33a244d 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -24,7 +24,10 @@ def preprocess(i): #if not check_installation("cpupower",os_info): env['CM_INSTALL_CPUPOWER'] = 'True' - return {'return':0} + if env.get('CM_PLATFORM_DETAILS_FILE_PATH', '') == '': + env['CM_GET_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") + + return {'return':0} def postprocess(i): diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index e08182194c..603d5733f2 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -57,6 +57,7 @@ input_mapping: find_performance: CM_MLPERF_FIND_PERFORMANCE_MODE framework: CM_MLPERF_BACKEND docker_keep_alive: CM_DOCKER_CONTAINER_KEEP_ALIVE + get_platform_details: CM_GET_PLATFORM_DETAILS gpu_name: CM_NVIDIA_GPU_NAME hw_name: CM_HW_NAME pip_loadgen: CM_MLPERF_INFERENCE_LOADGEN_INSTALL_FROM_PIP From a2d86dd6f2baf7a81bc927e18526667384783cdf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 22:58:07 +0530 Subject: [PATCH 133/658] Support system_info.txt in mlperf inference submission generation --- .../customize.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index b1b77ed4ba..9d1160b44b 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -239,8 +239,11 @@ def generate_submission(i): results = {} + model_platform_info_file = None + for model in models: results[model] = {} + platform_info_file = None result_model_path = os.path.join(result_path, model) submission_model_path = os.path.join(submission_path, model) measurement_model_path = os.path.join(measurement_path, model) @@ -386,8 +389,10 @@ def generate_submission(i): files.append(f) elif f == "spl.txt": files.append(f) - elif f in [ "README.md", "README-extra.md", "cm-version-info.json", "os_info.json", "cpu_info.json", "pip_freeze.json" ] and mode == "performance": + elif f in [ "README.md", "README-extra.md", "cm-version-info.json", "os_info.json", "cpu_info.json", "pip_freeze.json", "system_info.txt" ] and mode == "performance": shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, f)) + if f == "system_info.txt" and not platform_info_file: + platform_info_file = os.path.join(result_mode_path, f) elif f in [ "console.out" ]: shutil.copy(os.path.join(result_mode_path, f), os.path.join(submission_measurement_path, mode+"_"+f)) @@ -417,6 +422,26 @@ def generate_submission(i): with open(readme_file, mode='a') as f: f.write(result_string) + #Copy system_info.txt to the submission measurements model folder if any scenario performance run has it + sys_info_file = None + if os.path.exists(os.path.join(result_model_path, "system_info.txt")): + sys_info_file = os.path.join(result_model_path, "system_info.txt") + elif platform_info_file: + sys_info_file = platform_info_file + if sys_info_file: + model_platform_info_file = sys_info_file + shutil.copy(sys_info_file, os.path.join(measurement_model_path, "system_info.txt")) + + #Copy system_info.txt to the submission measurements folder if any model performance run has it + sys_info_file = None + if os.path.exists(os.path.join(result_path, "system_info.txt")): + sys_info_file = os.path.join(result_path, "system_info.txt") + elif model_platform_info_file: + sys_info_file = model_platform_info_file + if sys_info_file: + shutil.copy(sys_info_file, os.path.join(measurement_path, "system_info.txt")) + + with open(system_file, "w") as fp: json.dump(system_meta, fp, indent=2) From 766303016d46fd8b0165037c5312cb084b1bcc9a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 18:28:53 +0100 Subject: [PATCH 134/658] Update test-mlperf-inference-gptj.yml --- .github/workflows/test-mlperf-inference-gptj.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml index f2f9f41523..794c4b6d69 100644 --- a/.github/workflows/test-mlperf-inference-gptj.yml +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -5,7 +5,7 @@ name: MLPerf inference GPT-J on: schedule: - - cron: "1 2 * * *" + - cron: "35 17 * * *" jobs: build: From b983a6cf259475e58c06959d7c3a3bbb7ac812a6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Oct 2024 00:38:55 +0530 Subject: [PATCH 135/658] Cleanups for mlperf inference get-platform-details --- script/app-mlperf-inference/_cm.yaml | 19 +++++++++++-------- script/get-platform-details/customize.py | 8 +++----- script/get-platform-details/run.sh | 7 ++++--- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index c65723b600..5376f44ba5 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -111,6 +111,17 @@ deps: posthook_deps: - tags: get,mlperf,sut,description #populate system meta information like framework + - tags: get,platform,details + enable_if_any_env: + CM_SUDO_USER: + - yes + CM_GET_PLATFORM_DETAILS: + - yes + skip_if_env: + CM_MLPERF_LOADGEN_MODE: + - accuracy + env: + CM_PLATFORM_DETAILS_FILE_PATH: '<<>>/system_info.txt' # Order of variations for documentation variation_groups_order: @@ -1052,14 +1063,6 @@ variations: - mlperf-accuracy-script - imagenet-accuracy-script tags: run,accuracy,mlperf,_imagenet - - tags: get,platform,details - enable_if_any_env: - CM_SUDO_USER: - - yes - CM_GET_PLATFORM_DETAILS: - - yes - env: - CM_PLATFORM_DETAILS_FILE_PATH: '<<>>/system_info.txt' onnxruntime: group: backend diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index e3b33a244d..47f1bab8b8 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -16,18 +16,16 @@ def preprocess(i): if os_info['platform'] == "windows": return {'return':1, 'error':'get-platform-details script not yet supported in windows!'} - print(env['CM_HOST_OS_KERNEL_VERSION']) - if not check_installation("numactl",os_info): env['CM_INSTALL_NUMACTL'] = 'True' #if not check_installation("cpupower",os_info): env['CM_INSTALL_CPUPOWER'] = 'True' - if env.get('CM_PLATFORM_DETAILS_FILE_PATH', '') == '': - env['CM_GET_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") + if env.get('CM_PLATFORM_DETAILS_FILE_PATH', '') == '': + env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") - return {'return':0} + return {'return':0} def postprocess(i): diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 05fdb57de7..f9d81cb8e4 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -1,8 +1,9 @@ #!/bin/bash -OUTPUT_FILE="system_info.txt" - -echo "WARNING: sudo permission is needed to some packages for measuring the platform details" +OUTPUT_FILE="$CM_PLATFORM_DETAILS_FILE_PATH" +set -e +#echo $OUTPUT_FILE +echo "WARNING: sudo permission is needed for some of the below commands" if [[ ${CM_HOST_OS_FLAVOR} == "macos" ]]; then echo "WARNING: To be done for the mac os" From a9c19878d5829fa1161f9c4e1ce501ca11bbb2c2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 20:10:02 +0100 Subject: [PATCH 136/658] Update test-mlperf-inference-gptj.yml --- .github/workflows/test-mlperf-inference-gptj.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml index 794c4b6d69..0562b9176b 100644 --- a/.github/workflows/test-mlperf-inference-gptj.yml +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -5,7 +5,7 @@ name: MLPerf inference GPT-J on: schedule: - - cron: "35 17 * * *" + - cron: "15 19 * * *" jobs: build: From 052dd0de3e25a4f2918932bd6c6bca95388b08bd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Oct 2024 01:00:11 +0530 Subject: [PATCH 137/658] Don't fail if we can't get all platform details --- script/get-platform-details/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index f9d81cb8e4..481319f223 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -1,7 +1,7 @@ #!/bin/bash OUTPUT_FILE="$CM_PLATFORM_DETAILS_FILE_PATH" -set -e +#set -e #echo $OUTPUT_FILE echo "WARNING: sudo permission is needed for some of the below commands" @@ -47,7 +47,7 @@ else echo "8. numactl --hardware" >> $OUTPUT_FILE eval "numactl --hardware" >> $OUTPUT_FILE - test $? -eq 0 || exit $? + #test $? -eq 0 || exit $? echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "9. /proc/meminfo" >> $OUTPUT_FILE @@ -82,7 +82,7 @@ else echo "15. sysctl" >> $OUTPUT_FILE eval "sudo sysctl -a" >> $OUTPUT_FILE - test $? -eq 0 || exit $? + #test $? -eq 0 || exit $? echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE @@ -112,12 +112,12 @@ else echo "21. dmidecode" >> $OUTPUT_FILE eval "sudo dmidecode" >> $OUTPUT_FILE - test $? -eq 0 || exit $? + #test $? -eq 0 || exit $? echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "22. BIOS" >> $OUTPUT_FILE eval "sudo dmidecode -t bios" >> $OUTPUT_FILE - test $? -eq 0 || exit $? + #test $? -eq 0 || exit $? echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "System information has been saved to $PWD/$OUTPUT_FILE" From 3db5cd2c65f4ab2ce1d1466cbf6e610cc3174ea7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sun, 13 Oct 2024 21:49:18 +0530 Subject: [PATCH 138/658] used copy instead of softlink in retinanet --- script/app-mlperf-inference-nvidia/customize.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index f7c116b145..23472d6d04 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -157,12 +157,10 @@ def preprocess(i): cmds.append(f"ln -sf {dataset_path} {target_data_path}") calibration_dataset_path=env['CM_CALIBRATION_DATASET_PATH'] - target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf','calibration', 'train') + target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf','calibration', 'calibration') if not os.path.exists(target_data_path_dir): cmds.append(f"mkdir -p {target_data_path_dir}") - target_data_path = os.path.join(target_data_path_dir, 'data') - if not os.path.exists(target_data_path): - cmds.append(f"ln -sf {calibration_dataset_path} {target_data_path}") + cmds.append(f"cp -r {calibration_dataset_path} {target_data_path}") preprocessed_data_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'preprocessed_data') target_model_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'models', 'retinanet-resnext50-32x4d') From 93c45072caec9865eafe67dc6e8284241d53d56c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:43:11 +0530 Subject: [PATCH 139/658] enabled retinanet model download --- script/app-mlperf-inference-nvidia/_cm.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 09820e5683..f58e61139c 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -169,6 +169,15 @@ deps: - criteo-preprocessed tags: get,dataset,preprocessed,criteo + ######################################################################## + # install retinanet model + - enable_if_env: + CM_MODEL: + - retinanet + names: + -retinanet-model + tags: get,ml-model,retinanet,_pytorch,_fp32,_weights + ######################################################################## # Install dlrm model - enable_if_env: From e375b7e482f0f03d4f86d04fbdcf7b73cbba323a Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 14 Oct 2024 18:25:13 +0530 Subject: [PATCH 140/658] added gh actions - nvidia implementation --- .../test-all-nvidia-mlperf-implementation.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/test-all-nvidia-mlperf-implementation.yml diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml new file mode 100644 index 0000000000..b68ad48f45 --- /dev/null +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -0,0 +1,27 @@ +name: MLPerf inference Nvidia implementations + +on: + schedule: + - cron: "1 3 * * *" #to be adjusted + +build_nvidia: + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64, GO-spr] + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + model: [ "resnet50", "3d-unet-99", "3d-unet-99.9", "bert-99", "bert-99.9", "retinanet" ] + steps: + - name: Preprocessing step for the run + run: | + if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi + python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM + pip install --upgrade cm4mlops + cm pull repo + - name: Test MLPerf Inference NVIDIA ${{ matrix.model }} + run: | + cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev,_short --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=${{ matrix.model }} --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 6c2c8f4c62442ebef7163876d85aa1ec84f72435 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 14 Oct 2024 19:15:20 +0530 Subject: [PATCH 141/658] WIP - updated cron time and system tags --- .github/workflows/test-all-nvidia-mlperf-implementation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml index b68ad48f45..01641049d0 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -2,11 +2,11 @@ name: MLPerf inference Nvidia implementations on: schedule: - - cron: "1 3 * * *" #to be adjusted + - cron: "30 14 * * *" #to be adjusted build_nvidia: - if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64, GO-spr] + if: github.repository_owner == 'anandhu-eng' + runs-on: [ self-hosted, linux, x64, GO-i9] strategy: fail-fast: false matrix: From 5604405539bff788ed8a0aa9d698a8fc376f9fbb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 14 Oct 2024 15:09:35 +0100 Subject: [PATCH 142/658] Update test-all-nvidia-mlperf-implementation.yml --- .github/workflows/test-all-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml index 01641049d0..ccc5c51902 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -5,7 +5,7 @@ on: - cron: "30 14 * * *" #to be adjusted build_nvidia: - if: github.repository_owner == 'anandhu-eng' + if: github.repository_owner == 'gateoverflow' runs-on: [ self-hosted, linux, x64, GO-i9] strategy: fail-fast: false From 605c415760eff9b0be8d4ef6d7bc5d45c7c349ef Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 14 Oct 2024 15:19:08 +0100 Subject: [PATCH 143/658] Update test-all-nvidia-mlperf-implementation.yml --- .github/workflows/test-all-nvidia-mlperf-implementation.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml index ccc5c51902..58fd302919 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -6,14 +6,14 @@ on: build_nvidia: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64, GO-i9] + runs-on: [ self-hosted, linux, GO-i9] strategy: fail-fast: false matrix: python-version: [ "3.12" ] model: [ "resnet50", "3d-unet-99", "3d-unet-99.9", "bert-99", "bert-99.9", "retinanet" ] steps: - - name: Preprocessing step for the run + - name: Test MLPerf Inference NVIDIA ${{ matrix.model }} run: | if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi python3 -m venv gh_action @@ -21,7 +21,5 @@ build_nvidia: export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops cm pull repo - - name: Test MLPerf Inference NVIDIA ${{ matrix.model }} - run: | cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev,_short --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=${{ matrix.model }} --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 2eeae268114965f2d1779392824f1457450530f9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 14 Oct 2024 15:20:01 +0100 Subject: [PATCH 144/658] Update test-all-nvidia-mlperf-implementation.yml --- .github/workflows/test-all-nvidia-mlperf-implementation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml index 58fd302919..11ca88d010 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -4,7 +4,8 @@ on: schedule: - cron: "30 14 * * *" #to be adjusted -build_nvidia: +jobs: + build_nvidia: if: github.repository_owner == 'gateoverflow' runs-on: [ self-hosted, linux, GO-i9] strategy: From 714921b3076db4ed818c115ebbf169b77356f545 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 14 Oct 2024 15:22:52 +0100 Subject: [PATCH 145/658] Update test-all-nvidia-mlperf-implementation.yml --- .github/workflows/test-all-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml index 11ca88d010..ac4cabb629 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -1,4 +1,4 @@ -name: MLPerf inference Nvidia implementations +name: MLPerf Inference Nvidia implementations on: schedule: @@ -7,7 +7,7 @@ on: jobs: build_nvidia: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, GO-i9] + runs-on: [ self-hosted, linux, x64, GO-i9] strategy: fail-fast: false matrix: From f439d78e666a64fd93e09d538080d89afa22865a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:34:46 +0530 Subject: [PATCH 146/658] reverted dataset path changes --- script/app-mlperf-inference-nvidia/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 23472d6d04..17574e3b29 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -157,7 +157,7 @@ def preprocess(i): cmds.append(f"ln -sf {dataset_path} {target_data_path}") calibration_dataset_path=env['CM_CALIBRATION_DATASET_PATH'] - target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf','calibration', 'calibration') + target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf','calibration', 'train') if not os.path.exists(target_data_path_dir): cmds.append(f"mkdir -p {target_data_path_dir}") cmds.append(f"cp -r {calibration_dataset_path} {target_data_path}") From fe750b56bf43de93d910d0c9e60bf99c02302897 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:35:17 +0530 Subject: [PATCH 147/658] reverted retinanet model download --- script/app-mlperf-inference-nvidia/_cm.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index f58e61139c..09820e5683 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -169,15 +169,6 @@ deps: - criteo-preprocessed tags: get,dataset,preprocessed,criteo - ######################################################################## - # install retinanet model - - enable_if_env: - CM_MODEL: - - retinanet - names: - -retinanet-model - tags: get,ml-model,retinanet,_pytorch,_fp32,_weights - ######################################################################## # Install dlrm model - enable_if_env: From 13f2a031146b80fbddf7f5b9d1a12033e58d80a6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 10:04:56 +0100 Subject: [PATCH 148/658] Update test-all-nvidia-mlperf-implementation.yml --- .github/workflows/test-all-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-all-nvidia-mlperf-implementation.yml index ac4cabb629..a1be28ce93 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-all-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "30 14 * * *" #to be adjusted + - cron: "10 9 * * *" #to be adjusted jobs: build_nvidia: @@ -22,5 +22,5 @@ jobs: export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev,_short --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=${{ matrix.model }} --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From ff1d660cb02e98a05989630e5ceddf6c061cc812 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 14:39:21 +0530 Subject: [PATCH 149/658] Added detect command for git-lfs --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 94abbf371d..1bd8266511 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -156,7 +156,8 @@ }, "git-lfs": { "env": { - "CM_SYS_UTIL_NAME": "git-lfs" + "CM_SYS_UTIL_NAME": "git-lfs", + "CM_SYS_UTIL_CHECK_CMD": "git-lfs --version" }, "state": { "git-lfs": { From 499fb7d1d9c2e9815c52c94021e66d8323be3421 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 10:14:05 +0100 Subject: [PATCH 150/658] Update and rename test-all-nvidia-mlperf-implementation.yml to test-nvidia-mlperf-implementation.yml --- ...implementation.yml => test-nvidia-mlperf-implementation.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{test-all-nvidia-mlperf-implementation.yml => test-nvidia-mlperf-implementation.yml} (97%) diff --git a/.github/workflows/test-all-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml similarity index 97% rename from .github/workflows/test-all-nvidia-mlperf-implementation.yml rename to .github/workflows/test-nvidia-mlperf-implementation.yml index a1be28ce93..f5ca79a18c 100644 --- a/.github/workflows/test-all-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "10 9 * * *" #to be adjusted + - cron: "20 9 * * *" #to be adjusted jobs: build_nvidia: From 09aeb8fd62f5dcaf144acb9f8550cc2c73c194e6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 10:16:33 +0100 Subject: [PATCH 151/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index f5ca79a18c..aa85a30960 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -7,7 +7,7 @@ on: jobs: build_nvidia: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, linux, x64, GO-i9] + runs-on: [ self-hosted, linux, x64, GO-spr ] strategy: fail-fast: false matrix: From 27bfb50bb08976b3e1ad00d8549ec76608403795 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 10:27:38 +0100 Subject: [PATCH 152/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index aa85a30960..ca3dca615b 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "20 9 * * *" #to be adjusted + - cron: "40 9 * * *" #to be adjusted jobs: build_nvidia: From 900f4593d66cd943601c943465a6f7c80b586f9a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:53:12 +0530 Subject: [PATCH 153/658] added env key - CM_OPENIMAGES_CALIBRATION_DATASET_PATH --- script/get-dataset-openimages/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 6ccb558c59..8bc9569a01 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -77,6 +77,7 @@ def postprocess(i): env['CM_DATASET_OPENIMAGES_PATH_ROOT'] = env['CM_DATASET_PATH_ROOT'] else: env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') + env['CM_OPENIMAGES_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') env['CM_CALIBRATION_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-calibration-mlperf.json") env['CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path From 86ea53a6ab3f3fc3895a05ae38afe487aabc1be8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:53:48 +0530 Subject: [PATCH 154/658] Updated new env keys --- script/get-dataset-openimages/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-dataset-openimages/_cm.json b/script/get-dataset-openimages/_cm.json index 3f3904076b..a474d798f2 100644 --- a/script/get-dataset-openimages/_cm.json +++ b/script/get-dataset-openimages/_cm.json @@ -76,7 +76,8 @@ "CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH", "CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH", "CM_CALIBRATION_DATASET_PATH", - "CM_CALIBRATION_DATASET_PATH_ROOT" + "CM_CALIBRATION_DATASET_PATH_ROOT", + "CM_OPENIMAGES_CALIBRATION_DATASET_PATH" ], "tags": [ "get", From 210262f5d974fdbfd1146d2b9d47c5915815f35b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:55:35 +0530 Subject: [PATCH 155/658] added env key - CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH --- script/get-dataset-openimages/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 8bc9569a01..2d788b8534 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -69,6 +69,7 @@ def postprocess(i): env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data') annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-mlperf.json") env['CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path + env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = annotations_file_path if env.get("CM_DATASET_OPENIMAGES_CUSTOM_ANNOTATIONS",'') == "yes": annotations_file_src = env['CM_DATASET_OPENIMAGES_ANNOTATIONS_FILE_PATH'] From 90f75288b81d124f689084637df4bfc18ab78676 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:56:04 +0530 Subject: [PATCH 156/658] updated new env keys --- script/get-dataset-openimages/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-dataset-openimages/_cm.json b/script/get-dataset-openimages/_cm.json index a474d798f2..46c5da1bcb 100644 --- a/script/get-dataset-openimages/_cm.json +++ b/script/get-dataset-openimages/_cm.json @@ -77,7 +77,8 @@ "CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH", "CM_CALIBRATION_DATASET_PATH", "CM_CALIBRATION_DATASET_PATH_ROOT", - "CM_OPENIMAGES_CALIBRATION_DATASET_PATH" + "CM_OPENIMAGES_CALIBRATION_DATASET_PATH", + "CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH" ], "tags": [ "get", From 3acbcedb390f84b70cb8fe23f82d5fa810ee7d7f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:15:15 +0530 Subject: [PATCH 157/658] download retinanet dataset in host --- script/app-mlperf-inference/_cm.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 5376f44ba5..5e9b8db6cb 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -625,6 +625,16 @@ variations: - openimages-accuracy-script tags: run,accuracy,mlperf,_openimages + retinanet,nvidia-original: + docker: + deps: + - names: + - openimages-original + tags: get,dataset,original,openimages,_validation,_full,_custom-annotations + - names: + - openimages-calibration + tags: get,dataset,original,openimages,_calibration + 3d-unet-99: group: model @@ -1636,6 +1646,8 @@ docker: - cm pull repo mounts: - "${{ CM_DATASET_IMAGENET_PATH }}:${{ CM_DATASET_IMAGENET_PATH }}" + - "${{ CM_OPENIMAGES_CALIBRATION_DATASET_PATH }}:${{ CM_OPENIMAGES_CALIBRATION_DATASET_PATH }}" + - "${{ CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH }}:${{ CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH }}" - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}" - "${{ OUTPUT_BASE_DIR }}:${{ OUTPUT_BASE_DIR }}" - "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}" From ab6afca8575ea3423f1b41f2d7787e3b9af63263 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:16:21 +0530 Subject: [PATCH 158/658] disabled model download inside docker - nvidia --- script/app-mlperf-inference-nvidia/_cm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 09820e5683..473d78c1de 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -217,14 +217,14 @@ deps: - enable_if_env: CM_MODEL: - - retinanet + - retinanet-disabled names: - openimages-original tags: get,dataset,original,openimages,_validation,_full,_custom-annotations - enable_if_env: CM_MODEL: - - retinanet + - retinanet-disabled names: - openimages-calibration tags: get,dataset,original,openimages,_calibration From e3ee8a88b025c5c950c63a07de3f1a138d680a80 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:22:41 +0530 Subject: [PATCH 159/658] updated env variables for annotation and data --- script/app-mlperf-inference-nvidia/customize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 17574e3b29..17a5be734e 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -141,7 +141,7 @@ def preprocess(i): dataset_path = env['CM_DATASET_PATH'] #return {'return': 1, 'error': 'error'} - annotations_path = env['CM_DATASET_ANNOTATIONS_DIR_PATH'] + annotations_path = env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf') if not os.path.exists(target_data_path_dir): cmds.append(f"mkdir -p {target_data_path_dir}") @@ -156,11 +156,13 @@ def preprocess(i): if not os.path.exists(target_data_path): cmds.append(f"ln -sf {dataset_path} {target_data_path}") - calibration_dataset_path=env['CM_CALIBRATION_DATASET_PATH'] + calibration_dataset_path=env['CM_OPENIMAGES_CALIBRATION_DATASET_PATH'] target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf','calibration', 'train') if not os.path.exists(target_data_path_dir): cmds.append(f"mkdir -p {target_data_path_dir}") - cmds.append(f"cp -r {calibration_dataset_path} {target_data_path}") + target_data_path = os.path.join(target_data_path_dir, 'data') + if not os.path.exists(target_data_path): + cmds.append(f"ln -sf {calibration_dataset_path} {target_data_path}") preprocessed_data_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'preprocessed_data') target_model_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'models', 'retinanet-resnext50-32x4d') From 64c74378ba2a687f08080e147f0afd272dff36c6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 17:33:29 +0100 Subject: [PATCH 160/658] Update test_tutorial_retinanet.py --- tests/tutorials/test_tutorial_retinanet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tutorials/test_tutorial_retinanet.py b/tests/tutorials/test_tutorial_retinanet.py index 404b69fe93..9ecb2a3bef 100644 --- a/tests/tutorials/test_tutorial_retinanet.py +++ b/tests/tutorials/test_tutorial_retinanet.py @@ -21,7 +21,7 @@ r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' }) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr': \ {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \ 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \ 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) From 0275a829f33c93398c8c87a9c453340e10f43b91 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 17:33:49 +0100 Subject: [PATCH 161/658] Update test_tutorial_tvm.py --- tests/tutorials/test_tutorial_tvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tutorials/test_tutorial_tvm.py b/tests/tutorials/test_tutorial_tvm.py index 930e3622df..0d02b87a55 100644 --- a/tests/tutorials/test_tutorial_tvm.py +++ b/tests/tutorials/test_tutorial_tvm.py @@ -16,7 +16,7 @@ checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_dashboard', 'adr': \ +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission', 'adr': \ {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) From 320c850d6fcec94ae4429abf28a0495bc7646885 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 17:34:15 +0100 Subject: [PATCH 162/658] Update test_tutorial_tvm_pip_ge.py --- tests/tutorials/test_tutorial_tvm_pip_ge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tutorials/test_tutorial_tvm_pip_ge.py b/tests/tutorials/test_tutorial_tvm_pip_ge.py index 0c9a4b9c33..47180fa774 100644 --- a/tests/tutorials/test_tutorial_tvm_pip_ge.py +++ b/tests/tutorials/test_tutorial_tvm_pip_ge.py @@ -13,7 +13,7 @@ 'device': 'cpu', 'scenario': 'Offline', 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr': \ {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ 'device': 'cpu', 'scenario': 'Offline', 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) From 99f3482453a1ff772ac1507521a7bb187393f89c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Oct 2024 17:34:33 +0100 Subject: [PATCH 163/658] Update test_tutorial_tvm_pip_vm.py --- tests/tutorials/test_tutorial_tvm_pip_vm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tutorials/test_tutorial_tvm_pip_vm.py b/tests/tutorials/test_tutorial_tvm_pip_vm.py index 81069194d4..b9e47152af 100644 --- a/tests/tutorials/test_tutorial_tvm_pip_vm.py +++ b/tests/tutorials/test_tutorial_tvm_pip_vm.py @@ -15,7 +15,7 @@ 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) checks.check_return(r) -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short', 'adr': \ {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) From 0f9c48e076abb5c1a8946401229631086ac31bff Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:15:02 +0530 Subject: [PATCH 164/658] modifed env variable for openimage dataset --- script/app-mlperf-inference-nvidia/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 17a5be734e..49c2c876b5 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -138,7 +138,7 @@ def preprocess(i): elif env['CM_MODEL'] == "retinanet": #print(env) - dataset_path = env['CM_DATASET_PATH'] + dataset_path = env['CM_DATASET_OPENIMAGES_PATH'] #return {'return': 1, 'error': 'error'} annotations_path = env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] From 94aadf0fa5105301a1fc22a17b3895ed3b65e7a7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:16:05 +0530 Subject: [PATCH 165/658] added CM_DATASET_OPENIMAGES_PATH to mount --- script/app-mlperf-inference/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 5e9b8db6cb..fe0736106a 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -1646,6 +1646,7 @@ docker: - cm pull repo mounts: - "${{ CM_DATASET_IMAGENET_PATH }}:${{ CM_DATASET_IMAGENET_PATH }}" + - "${{ CM_DATASET_OPENIMAGES_PATH }}:${{ CM_DATASET_OPENIMAGES_PATH }}" - "${{ CM_OPENIMAGES_CALIBRATION_DATASET_PATH }}:${{ CM_OPENIMAGES_CALIBRATION_DATASET_PATH }}" - "${{ CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH }}:${{ CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH }}" - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}" From 926deae0872115e168592d41c293c365910bc375 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:59:29 +0530 Subject: [PATCH 166/658] updated new env keys --- script/get-dataset-openimages/_cm.json | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-dataset-openimages/_cm.json b/script/get-dataset-openimages/_cm.json index 46c5da1bcb..c57f5ffbc1 100644 --- a/script/get-dataset-openimages/_cm.json +++ b/script/get-dataset-openimages/_cm.json @@ -69,6 +69,7 @@ "new_env_keys": [ "CM_DATASET_PATH", "CM_DATASET_PATH_ROOT", + "CM_DATASET_OPENIMAGES_PATH", "CM_DATASET_OPENIMAGES_DATASET_PATH", "CM_DATASET_OPENIMAGES_DATASET_PATH_ROOT", "CM_DATASET_ANNOTATIONS_DIR_PATH", From 2c1aba3e5996c70863afcf158ccaaffbadd64caa Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:25:28 +0530 Subject: [PATCH 167/658] added skip if env condition for dataset download --- script/app-mlperf-inference-nvidia/_cm.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 473d78c1de..1045ab1ee8 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -119,6 +119,9 @@ deps: - enable_if_env: CM_MODEL: - resnet50 + skip_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes' names: - imagenet-original tags: get,dataset,original,imagenet,_full @@ -217,14 +220,20 @@ deps: - enable_if_env: CM_MODEL: - - retinanet-disabled + - retinanet + skip_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes' names: - openimages-original tags: get,dataset,original,openimages,_validation,_full,_custom-annotations - enable_if_env: CM_MODEL: - - retinanet-disabled + - retinanet + skip_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes` names: - openimages-calibration tags: get,dataset,original,openimages,_calibration From 46d62e85312d01a5af806d1ea57d93b393cffa74 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:35:15 +0530 Subject: [PATCH 168/658] added common variable for using dataset from host --- script/app-mlperf-inference/_cm.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index fe0736106a..68b08f4999 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -63,6 +63,7 @@ input_mapping: gpu_name: CM_NVIDIA_GPU_NAME nvidia_llama2_dataset_file_path: CM_NVIDIA_LLAMA_DATASET_FILE_PATH tp_size: CM_NVIDIA_TP_SIZE + use_dataset_from_host: CM_USE_DATASET_FROM_HOST # Duplicate CM environment variables to the ones used in native apps env_key_mappings: @@ -596,10 +597,12 @@ variations: - mlperf-accuracy-script - imagenet-accuracy-script tags: run,accuracy,mlperf,_imagenet - docker: deps: - tags: get,dataset,imagenet,validation,original,_full + enable_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes' names: - imagenet-original - dataset-original @@ -630,9 +633,15 @@ variations: deps: - names: - openimages-original + enable_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes' tags: get,dataset,original,openimages,_validation,_full,_custom-annotations - names: - openimages-calibration + enable_if_env: + CM_USE_DATASET_FROM_HOST: + - 'yes' tags: get,dataset,original,openimages,_calibration 3d-unet-99: From 4a3decb090c8366143eca831abd7217b48f7efeb Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:47:37 +0530 Subject: [PATCH 169/658] added use_dataset_from_host input mapping --- script/run-mlperf-inference-app/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 603d5733f2..ff4ed42292 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -110,6 +110,7 @@ input_mapping: max_test_duration: CM_MLPERF_MAX_DURATION_TEST all_models: CM_MLPERF_ALL_MODELS criteo_day23_raw_data_path: CM_CRITEO_DAY23_RAW_DATA_PATH + use_dataset_from_host: CM_USE_DATASET_FROM_HOST new_state_keys: - app_mlperf_inference_* From 49a8d7d02249b0a3d0af57431a3fbeaae22ebf1e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:57:40 +0530 Subject: [PATCH 170/658] fix typo --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 1045ab1ee8..0f4d1abbcf 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -233,7 +233,7 @@ deps: - retinanet skip_if_env: CM_USE_DATASET_FROM_HOST: - - 'yes` + - 'yes' names: - openimages-calibration tags: get,dataset,original,openimages,_calibration From b519cfd16178b834959e8188a984cd86db62e8a8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:28:53 +0530 Subject: [PATCH 171/658] skipped dataset download while building container --- script/app-mlperf-inference-nvidia/_cm.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 0f4d1abbcf..affb10611a 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -122,6 +122,8 @@ deps: skip_if_env: CM_USE_DATASET_FROM_HOST: - 'yes' + CM_RUN_STATE_DOCKER: + - 'yes' names: - imagenet-original tags: get,dataset,original,imagenet,_full @@ -224,6 +226,8 @@ deps: skip_if_env: CM_USE_DATASET_FROM_HOST: - 'yes' + CM_RUN_STATE_DOCKER: + - 'yes' names: - openimages-original tags: get,dataset,original,openimages,_validation,_full,_custom-annotations @@ -234,6 +238,8 @@ deps: skip_if_env: CM_USE_DATASET_FROM_HOST: - 'yes' + CM_RUN_STATE_DOCKER: + - 'yes' names: - openimages-calibration tags: get,dataset,original,openimages,_calibration From 79a911d574086768a02ff372ca480f93f509b653 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:47:08 +0530 Subject: [PATCH 172/658] changed inference_submission -> inference_result repo --- script/get-ml-model-llama2/_cm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index d64c6e004a..e4701cc060 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -174,8 +174,8 @@ "tags": "get,preprocessed,dataset,openorca,_calibration,_mlc" }, { - "tags": "get,git,repo,_repo.https://github.com/mlcommons/submissions_inference_v4.1", - "extra_cache_tags": "inference,submissions", + "tags": "get,git,repo,_repo.https://github.com/mlcommons/inference_results_v4.1", + "extra_cache_tags": "inference,results", "env": { "CM_GIT_CHECKOUT_PATH_ENV_NAME": "CM_MLPERF_INFERENCE_RESULTS_PATH" } From 5f368603945e553e1f085c1a2df57d0712b9037a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:49:42 +0530 Subject: [PATCH 173/658] Added 4.1 default version --- script/app-mlperf-inference-amd/_cm.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-amd/_cm.yaml b/script/app-mlperf-inference-amd/_cm.yaml index 78ecad9f88..7464c05e74 100644 --- a/script/app-mlperf-inference-amd/_cm.yaml +++ b/script/app-mlperf-inference-amd/_cm.yaml @@ -255,8 +255,8 @@ variations: - tags: get,generic-python-lib,_package.vllm names: - vllm - - tags: get,git,repo,_repo.https://github.com/mlcommons/submissions_inference_v4.1 - extra_cache_tags: inference,submissions + - tags: get,git,repo,_repo.https://github.com/mlcommons/inference_results_v4.1 + extra_cache_tags: inference,results env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_MLPERF_INFERENCE_RESULTS_PATH @@ -315,5 +315,10 @@ variations: env: CM_MLPERF_INFERENCE_RESULTS_REPO: https://github.com/mlcommons/inference_results_v4.0 + r4.1_default: + group: version + env: + CM_MLPERF_INFERENCE_RESULTS_REPO: https://github.com/mlcommons/inference_results_v4.1 + docker: real_run: False From fa2eacfdf632d6b16675eee5bb00c98a7ecd601f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:21:23 +0530 Subject: [PATCH 174/658] fix syntax error --- script/app-mlperf-inference-amd/run-llama2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index 4692bfcc46..7377c8955b 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -33,7 +33,7 @@ LOG_DIR=${CM_MLPERF_OUTPUT_DIR} cp $USER_CONF ${LOG_DIR}/user.conf -cmd ="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Offline.py \ +cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Offline.py \ --scenario ${CM_MLPERF_LOADGEN_SCENARIO \ --output-log-dir ${LOG_DIR} \ --model-path $MODEL_PATH \ From edbc0ed4c9a56e4fbc8de1ab1594e04cad7d6b40 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:23:58 +0530 Subject: [PATCH 175/658] fix typo --- script/app-mlperf-inference-amd/run-llama2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index 7377c8955b..f4aa32bf44 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -34,7 +34,7 @@ LOG_DIR=${CM_MLPERF_OUTPUT_DIR} cp $USER_CONF ${LOG_DIR}/user.conf cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Offline.py \ - --scenario ${CM_MLPERF_LOADGEN_SCENARIO \ + --scenario ${CM_MLPERF_LOADGEN_SCENARIO} \ --output-log-dir ${LOG_DIR} \ --model-path $MODEL_PATH \ --mlperf-conf $MLPERF_CONF \ From 55cd12172ff833251244b466e2290e10cc199b60 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:28:09 +0530 Subject: [PATCH 176/658] added echo and run commands --- script/app-mlperf-inference-amd/run-llama2.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index f4aa32bf44..92b0e9558d 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -54,3 +54,8 @@ cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Of --sorting ${SORTING} \ --enforce-eager True \ --gpu-memory-utilization 0.99" + +echo "${cmd}" +# uncomment the below lines for testing +#eval "${cmd}" +#test $? -eq 0 || exit $? From 5c7f766ee13ad768ab1af75ad95414fa66416e19 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 16 Oct 2024 16:16:45 +0100 Subject: [PATCH 177/658] Update and rename publish.yaml to publish-docs.yaml --- .github/workflows/{publish.yaml => publish-docs.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{publish.yaml => publish-docs.yaml} (96%) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish-docs.yaml similarity index 96% rename from .github/workflows/publish.yaml rename to .github/workflows/publish-docs.yaml index fa9ace5da0..958e5c8bb1 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish-docs.yaml @@ -1,6 +1,6 @@ # This is a basic workflow to help you get started with Actions -name: Publish site +name: Publish docs site on: From ca4bd668b617b1426086a603ede7b6ac8b58014e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 16 Oct 2024 16:17:54 +0100 Subject: [PATCH 178/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index ca3dca615b..37b8fa35a5 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "40 9 * * *" #to be adjusted + - cron: "24 15 * * *" #to be adjusted jobs: build_nvidia: @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.12" ] - model: [ "resnet50", "3d-unet-99", "3d-unet-99.9", "bert-99", "bert-99.9", "retinanet" ] + model: [ "resnet50" ] steps: - name: Test MLPerf Inference NVIDIA ${{ matrix.model }} run: | From 4668f6f927f94bf39dbcfb4c67fb9c9040d90f6d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 16 Oct 2024 22:01:28 +0530 Subject: [PATCH 179/658] Fix no submission generation for full MLPerf inference runs --- script/run-mlperf-inference-app/_cm.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index ff4ed42292..9a2c8f23af 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -227,7 +227,6 @@ variations: tags: _full env: CM_MLPERF_SUBMISSION_GENERATION_STYLE: full - CM_MLPERF_SKIP_SUBMISSION_GENERATION: 'yes' group: submission-generation-style performance-only: From 86331699717350beb455f2a5a5f10c0f4f37a5f1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 16 Oct 2024 17:33:14 +0100 Subject: [PATCH 180/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 37b8fa35a5..002e69b2c1 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "24 15 * * *" #to be adjusted + - cron: "40 16 * * *" #to be adjusted jobs: build_nvidia: @@ -22,5 +22,5 @@ jobs: export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=90000 --server_target_qps=72000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 2f09bf33679694bc83e9267e7641c81304acea37 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 16 Oct 2024 17:48:24 +0100 Subject: [PATCH 181/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 002e69b2c1..f0d8f45a1a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "40 16 * * *" #to be adjusted + - cron: "55 16 * * *" #to be adjusted jobs: build_nvidia: @@ -22,5 +22,5 @@ jobs: export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=90000 --server_target_qps=72000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --scenario=Offline --test_query_count=500 --target_qps=1 --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=90000 --server_target_qps=72000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From fcc1249788f7bb96948e064047f58f7d40553901 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 22:32:26 +0530 Subject: [PATCH 182/658] removed test command as -xeu is set --- script/app-mlperf-inference-amd/run-llama2.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index 92b0e9558d..9c4996c8c8 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -58,4 +58,3 @@ cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Of echo "${cmd}" # uncomment the below lines for testing #eval "${cmd}" -#test $? -eq 0 || exit $? From 898a564d337ab27cf37e83cddb2912b2a91c9c8e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 22:56:43 +0530 Subject: [PATCH 183/658] use cm-code-only as branch --- script/app-mlperf-inference-amd/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-amd/_cm.yaml b/script/app-mlperf-inference-amd/_cm.yaml index 7464c05e74..089a1be1eb 100644 --- a/script/app-mlperf-inference-amd/_cm.yaml +++ b/script/app-mlperf-inference-amd/_cm.yaml @@ -255,7 +255,7 @@ variations: - tags: get,generic-python-lib,_package.vllm names: - vllm - - tags: get,git,repo,_repo.https://github.com/mlcommons/inference_results_v4.1 + - tags: get,git,repo,_repo.https://github.com/mlcommons/inference_results_v4.1,_branch.cm-code-only extra_cache_tags: inference,results env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_MLPERF_INFERENCE_RESULTS_PATH From 5d8d11a4d66e04d3268bd367d209b148cda2e559 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 16 Oct 2024 22:57:46 +0530 Subject: [PATCH 184/658] clone cm-code-only branch --- script/get-ml-model-llama2/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index e4701cc060..8c80521130 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -174,7 +174,7 @@ "tags": "get,preprocessed,dataset,openorca,_calibration,_mlc" }, { - "tags": "get,git,repo,_repo.https://github.com/mlcommons/inference_results_v4.1", + "tags": "get,git,repo,_repo.https://github.com/mlcommons/inference_results_v4.1,_branch.cm-code-only", "extra_cache_tags": "inference,results", "env": { "CM_GIT_CHECKOUT_PATH_ENV_NAME": "CM_MLPERF_INFERENCE_RESULTS_PATH" From d1bdcc020a5177b61332441f6355f8252521ccc9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 02:54:56 +0530 Subject: [PATCH 185/658] Using torch whl files for nvidia mlperf inference --- .../_cm.yaml | 23 +++++++++++++++++++ script/get-generic-python-lib/_cm.json | 15 ++++++++++++ 2 files changed, 38 insertions(+) diff --git a/script/build-mlperf-inference-server-nvidia/_cm.yaml b/script/build-mlperf-inference-server-nvidia/_cm.yaml index 74ce30e2f1..0e4b3ccf1f 100644 --- a/script/build-mlperf-inference-server-nvidia/_cm.yaml +++ b/script/build-mlperf-inference-server-nvidia/_cm.yaml @@ -199,10 +199,33 @@ variations: names: - pytorch - torch + skip_if_any_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: install,torchvision,from.src,_for-nvidia-mlperf-inference-v4.0 names: - pytorchvision - torchvision + skip_if_any_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 versions: r2.1: diff --git a/script/get-generic-python-lib/_cm.json b/script/get-generic-python-lib/_cm.json index 19b6f41441..977a27fe59 100644 --- a/script/get-generic-python-lib/_cm.json +++ b/script/get-generic-python-lib/_cm.json @@ -567,6 +567,21 @@ "CM_PANDAS_VERSION" ] }, + "whl-url.#": { + "deps": [ + { + "tags": "download,file,_url.#", + "force_cache": "yes", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_GENERIC_PYTHON_PIP_URL", + "CM_GENERIC_PYTHON_PACKAGE_NAME": "from_whl" + } + } + ], + "env": { + "CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL": "yes" + } + }, "path.#": { "env": { "CM_GENERIC_PYTHON_PIP_URL": "#" From f4de46b09fbe0506910de0c9cadb71b32b1f8a52 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 02:58:35 +0530 Subject: [PATCH 186/658] Added code-only variation for inference-results --- script/get-mlperf-inference-results/_cm.json | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/script/get-mlperf-inference-results/_cm.json b/script/get-mlperf-inference-results/_cm.json index 45d1c2f519..01451d42b8 100644 --- a/script/get-mlperf-inference-results/_cm.json +++ b/script/get-mlperf-inference-results/_cm.json @@ -9,7 +9,7 @@ "CM_GIT_DEPTH": "--depth 1", "CM_GIT_PATCH": "no" }, - "default_version": "v3.1", + "default_version": "v4.0", "prehook_deps": [ { "tags": "get,git,repo", @@ -72,11 +72,6 @@ "env": { "CM_MLPERF_INFERENCE_RESULTS_VERSION_NAME": "v4.1", "CM_GIT_URL": "https://github.com/<<>>/inference_results_v4.1.git" - }, - "adr": { - "inference-results-repo": { - "tags": "_branch.cm-fixes" - } } } }, @@ -112,6 +107,14 @@ "env": { "GITHUB_REPO_OWNER": "GATEOverflow" } + }, + "code-only": { + "group": "repo-branch", + "adr": { + "inference-results-repo": { + "tags": "_branch.cm-code-only" + } + } } } } From 8993c2f04861850dc821ed2313db115a68f50030 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 03:00:40 +0530 Subject: [PATCH 187/658] Using code-only variation for nvidia-mlperf-inference --- script/get-mlperf-inference-nvidia-common-code/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-nvidia-common-code/_cm.json b/script/get-mlperf-inference-nvidia-common-code/_cm.json index 5785f93f80..df9a152a04 100644 --- a/script/get-mlperf-inference-nvidia-common-code/_cm.json +++ b/script/get-mlperf-inference-nvidia-common-code/_cm.json @@ -11,7 +11,7 @@ "names": [ "mlperf-inference-results" ], - "tags": "get,mlperf,inference,results,official", + "tags": "get,mlperf,inference,results,official,_code-only", "inherit_variation_tags": true } ], From acab6a3e59f8a1eb4fab2cdbbda67574dcd2095e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 03:28:34 +0530 Subject: [PATCH 188/658] Use mlcommons repo for nvidia code v4.0 --- automation/script/module_misc.py | 4 +++- script/app-mlperf-inference/_cm.yaml | 5 +++-- script/get-mlperf-inference-nvidia-common-code/_cm.json | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 91ce181af5..cedd3e8f50 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1976,6 +1976,8 @@ def docker(i): device = i.get('docker_device', docker_settings.get('device')) + image_name = i.get('docker_image_name', docker_settings.get('image_name')) + r = check_gh_token(i, docker_settings, quiet) if r['return'] >0 : return r gh_token = r['gh_token'] @@ -2047,7 +2049,7 @@ def docker(i): 'image_repo': image_repo, 'interactive': interactive, 'mounts': mounts, - 'image_name': i.get('docker_image_name', ''), + 'image_name': image_name, # 'image_tag': script_alias, 'image_tag_extra': image_tag_extra, 'detached': detached, diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 68b08f4999..c6b480efcd 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -310,6 +310,7 @@ variations: nvidia-original,r4.1-dev_default: docker: base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.0-cuda12.2-cudnn8.9-x86_64-ubuntu20.04-public + image_name: mlperf-inference-nvidia-v4.1-dev-common nvidia-original,r4.1-dev_default,gptj_: docker: @@ -1515,10 +1516,10 @@ variations: add_deps_recursive: nvidia-inference-common-code: version: r4.0 - tags: _go + tags: _mlcommons nvidia-inference-server: version: r4.0 - tags: _go + tags: _mlcommons intel-harness: tags: _v4.0 default_env: diff --git a/script/get-mlperf-inference-nvidia-common-code/_cm.json b/script/get-mlperf-inference-nvidia-common-code/_cm.json index df9a152a04..a0191d9e9e 100644 --- a/script/get-mlperf-inference-nvidia-common-code/_cm.json +++ b/script/get-mlperf-inference-nvidia-common-code/_cm.json @@ -29,7 +29,8 @@ "uid": "26b78bf3ffdc4926", "variations": { "mlcommons": { - "group": "repo-owner" + "group": "repo-owner", + "default": true }, "custom": { "group": "repo-owner" From 57e2187cece2cc8c10b01a2cb943419177d0ae53 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 03:45:17 +0530 Subject: [PATCH 189/658] Fix whl installs for nvidia 4.1-dev --- .../_cm.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/script/build-mlperf-inference-server-nvidia/_cm.yaml b/script/build-mlperf-inference-server-nvidia/_cm.yaml index 0e4b3ccf1f..c52e463164 100644 --- a/script/build-mlperf-inference-server-nvidia/_cm.yaml +++ b/script/build-mlperf-inference-server-nvidia/_cm.yaml @@ -293,10 +293,32 @@ versions: names: - pytorch - torch + skip_if_any_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 - tags: install,torchvision,from.src,_for-nvidia-mlperf-inference-v4.0 names: - pytorchvision - torchvision + skip_if_any_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 r4.1: add_deps_recursive: From b5dcccf9ac5f471efebaace727d7342665eda2a6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 03:55:13 +0530 Subject: [PATCH 190/658] Fix whl installs for nvidia 4.1-dev --- .../_cm.yaml | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/script/build-mlperf-inference-server-nvidia/_cm.yaml b/script/build-mlperf-inference-server-nvidia/_cm.yaml index c52e463164..128d9732b1 100644 --- a/script/build-mlperf-inference-server-nvidia/_cm.yaml +++ b/script/build-mlperf-inference-server-nvidia/_cm.yaml @@ -199,7 +199,7 @@ variations: names: - pytorch - torch - skip_if_any_env: + skip_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 CM_PYTHON_MINOR_VERSION: @@ -215,7 +215,7 @@ variations: names: - pytorchvision - torchvision - skip_if_any_env: + skip_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 CM_PYTHON_MINOR_VERSION: @@ -273,10 +273,32 @@ versions: names: - pytorch - torch + skip_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 - tags: install,torchvision,from.src,_for-nvidia-mlperf-inference-v4.0 names: - pytorchvision - torchvision + skip_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + enable_if_env: + CM_HOST_PLATFORM_FLAVOR: + - x86_64 + CM_PYTHON_MINOR_VERSION: + - 8 r4.1-dev: add_deps_recursive: @@ -293,7 +315,7 @@ versions: names: - pytorch - torch - skip_if_any_env: + skip_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 CM_PYTHON_MINOR_VERSION: @@ -308,7 +330,7 @@ versions: names: - pytorchvision - torchvision - skip_if_any_env: + skip_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 CM_PYTHON_MINOR_VERSION: From ae85da064083e2684f0c82fab2f876dcc2b607e7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 14:15:09 +0530 Subject: [PATCH 191/658] Fix for pip install from whl --- script/get-generic-python-lib/_cm.json | 3 +-- script/get-generic-python-lib/customize.py | 2 +- script/get-generic-python-lib/install.sh | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-generic-python-lib/_cm.json b/script/get-generic-python-lib/_cm.json index 977a27fe59..38ea8f9e09 100644 --- a/script/get-generic-python-lib/_cm.json +++ b/script/get-generic-python-lib/_cm.json @@ -573,8 +573,7 @@ "tags": "download,file,_url.#", "force_cache": "yes", "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_GENERIC_PYTHON_PIP_URL", - "CM_GENERIC_PYTHON_PACKAGE_NAME": "from_whl" + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_GENERIC_PYTHON_PIP_URL" } } ], diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index e0e30c7f67..20c22e2ba4 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -97,7 +97,7 @@ def preprocess(i): env['CM_GENERIC_PYTHON_PIP_EXTRA'] = extra package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() - if package_name == '': + if package_name == '' and env.get('CM_GENERIC_PYTHON_PIP_URL', '') == '': return automation._available_variations({'meta':meta}) r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'}) diff --git a/script/get-generic-python-lib/install.sh b/script/get-generic-python-lib/install.sh index 04243d9f4c..b79aa81463 100644 --- a/script/get-generic-python-lib/install.sh +++ b/script/get-generic-python-lib/install.sh @@ -35,6 +35,7 @@ if [[ ${CM_GENERIC_PYTHON_PACKAGE_NAME} == "tensorflow_old" ]]; then exit 0 fi fi + if [[ -n ${CM_GENERIC_PYTHON_PIP_URL} ]]; then cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip install \"${CM_GENERIC_PYTHON_PIP_URL}\" ${CM_GENERIC_PYTHON_PIP_EXTRA}" echo $cmd From 534d33bc7e59f1ce10a11702281576e373b8e1b1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 15:08:07 +0530 Subject: [PATCH 192/658] Fix whl install in generic-python-lib --- script/get-generic-python-lib/customize.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index 20c22e2ba4..32b049bf61 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -11,7 +11,7 @@ def preprocess(i): run_script_input = i['run_script_input'] pip_version = env.get('CM_PIP_VERSION', '').strip().split('.') package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() - if package_name == '': + if package_name == '' and env.get('CM_GENERIC_PYTHON_PIP_URL', '') == '': return automation._available_variations({'meta':meta}) if package_name == "onnxruntime_gpu": @@ -40,7 +40,7 @@ def preprocess(i): r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'uninstall_deps'}) if r['return']>0: return r - prepare_env_key = env['CM_GENERIC_PYTHON_PACKAGE_NAME'] + prepare_env_key = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '') for x in ["-", "[", "]"]: prepare_env_key = prepare_env_key.replace(x,"_") @@ -96,10 +96,6 @@ def preprocess(i): env['CM_GENERIC_PYTHON_PIP_EXTRA'] = extra - package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() - if package_name == '' and env.get('CM_GENERIC_PYTHON_PIP_URL', '') == '': - return automation._available_variations({'meta':meta}) - r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'}) if r['return']>0: return r From dbe3e6eb6f8adb19332610dcc9bd1f932041b39b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 15:28:34 +0530 Subject: [PATCH 193/658] Fix whl installs for nvidia-mlperf-inference --- script/build-mlperf-inference-server-nvidia/_cm.yaml | 8 ++++---- script/get-generic-python-lib/customize.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/script/build-mlperf-inference-server-nvidia/_cm.yaml b/script/build-mlperf-inference-server-nvidia/_cm.yaml index 128d9732b1..13737bc159 100644 --- a/script/build-mlperf-inference-server-nvidia/_cm.yaml +++ b/script/build-mlperf-inference-server-nvidia/_cm.yaml @@ -278,7 +278,7 @@ versions: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_package.torch,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 @@ -293,7 +293,7 @@ versions: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_package.torchvision,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 @@ -320,7 +320,7 @@ versions: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_package.torch,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torch-2.1.0a0+git32f93b1-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 @@ -335,7 +335,7 @@ versions: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_package.torchvision,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index 32b049bf61..ea27ec6603 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -10,8 +10,9 @@ def preprocess(i): automation = i['automation'] run_script_input = i['run_script_input'] pip_version = env.get('CM_PIP_VERSION', '').strip().split('.') + package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() - if package_name == '' and env.get('CM_GENERIC_PYTHON_PIP_URL', '') == '': + if package_name == '': return automation._available_variations({'meta':meta}) if package_name == "onnxruntime_gpu": From 355355ba58751985294dd3dfac85c43bd0174029 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:31:20 +0530 Subject: [PATCH 194/658] command generation for server scenario - WIP --- script/app-mlperf-inference-amd/run-llama2.sh | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index 9c4996c8c8..11a810a979 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -5,8 +5,6 @@ set -xeu N_SAMPLES=${N_SAMPLES:-24576} #24576 #3072 #2457 #6 TP=1 DP=${DP:-8} -WD=${WD:-0} -SORTING=${SORTING:-descending} #ascending #descending #lexicographic #skip export HIP_FORCE_DEV_KERNARG=1 export VLLM_USE_TRITON_FLASH_ATTN=0 @@ -14,7 +12,6 @@ export VLLM_FP8_PADDING=1 export VLLM_FP8_ACT_PADDING=1 export VLLM_FP8_WEIGHT_PADDING=1 export VLLM_FP8_REDUCE_CONV=1 -export VLLM_SCHED_PREFILL_KVC_FREEPCT=31.0 export HARNESS_DISABLE_VLLM_LOGS=1 export VLLM_LOGGING_LEVEL=ERROR @@ -33,7 +30,7 @@ LOG_DIR=${CM_MLPERF_OUTPUT_DIR} cp $USER_CONF ${LOG_DIR}/user.conf -cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Offline.py \ +COMMON_CMD_OPTIONS="\ --scenario ${CM_MLPERF_LOADGEN_SCENARIO} \ --output-log-dir ${LOG_DIR} \ --model-path $MODEL_PATH \ @@ -49,11 +46,31 @@ cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Of -dp ${DP} \ --quantization fp8 \ --quantized-weights-path ${QUANTIZED_WEIGHTS_PATH} \ - --quantization-param-path ${QUANTIZATION_PARAM_PATH} \ + --quantization-param-path ${QUANTIZATION_PARAM_PATH}" + +if [ "${CM_MLPERF_LOADGEN_MODE:-}" == "accuracy" ]; then + COMMON_CMD_OPTIONS+=" --accuracy" +fi + + +if [ "$CM_MLPERF_LOADGEN_SCENARIO" == "Offline" ]; then + WD=${WD:-0} + SORTING=${SORTING:-descending} #ascending #descending #lexicographic #skip + export VLLM_SCHED_PREFILL_KVC_FREEPCT=31.0 + # generate run command + cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_Offline.py \ + ${COMMON_CMD_OPTIONS} \ --warmup-duration ${WD} \ --sorting ${SORTING} \ --enforce-eager True \ --gpu-memory-utilization 0.99" +else + # generate run command + cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_MLPERF_AMD_LLAMA2_CODE_PATH}/mainVllmFp8_SyncServer.py \ + ${COMMON_CMD_OPTIONS} \ + --enable-warm-up \ + --enable-batcher" +fi echo "${cmd}" # uncomment the below lines for testing From bc7fb9bc869a6d77fec16db2197f27df8a222f02 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:42:33 +0530 Subject: [PATCH 195/658] code clean --- script/app-mlperf-inference-amd/run-llama2.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index 11a810a979..4cf9f4f2be 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -48,12 +48,11 @@ COMMON_CMD_OPTIONS="\ --quantized-weights-path ${QUANTIZED_WEIGHTS_PATH} \ --quantization-param-path ${QUANTIZATION_PARAM_PATH}" -if [ "${CM_MLPERF_LOADGEN_MODE:-}" == "accuracy" ]; then +if [ "${CM_MLPERF_LOADGEN_MODE}" == "accuracy" ]; then COMMON_CMD_OPTIONS+=" --accuracy" fi - -if [ "$CM_MLPERF_LOADGEN_SCENARIO" == "Offline" ]; then +if [ "${CM_MLPERF_LOADGEN_SCENARIO}" == "Offline" ]; then WD=${WD:-0} SORTING=${SORTING:-descending} #ascending #descending #lexicographic #skip export VLLM_SCHED_PREFILL_KVC_FREEPCT=31.0 From c387b61eb25218e15b63099adafce6aad8c82144 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 15:47:25 +0530 Subject: [PATCH 196/658] Fix torchvision whl url --- script/build-mlperf-inference-server-nvidia/_cm.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/build-mlperf-inference-server-nvidia/_cm.yaml b/script/build-mlperf-inference-server-nvidia/_cm.yaml index 13737bc159..c5003f67cc 100644 --- a/script/build-mlperf-inference-server-nvidia/_cm.yaml +++ b/script/build-mlperf-inference-server-nvidia/_cm.yaml @@ -220,7 +220,7 @@ variations: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 @@ -293,7 +293,7 @@ versions: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_package.torchvision,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_package.torchvision,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 @@ -335,7 +335,7 @@ versions: - x86_64 CM_PYTHON_MINOR_VERSION: - 8 - - tags: get,generic-python-lib,_package.torchvision,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl + - tags: get,generic-python-lib,_package.torchvision,_whl-url.https://github.com/mlcommons/cm4mlperf-inference/releases/download/mlperf-inference-v4.0/torchvision-0.16.0a0+657027f-cp38-cp38-linux_x86_64.whl enable_if_env: CM_HOST_PLATFORM_FLAVOR: - x86_64 From c253ad34e9e7bd606caa4bfacc955ecb73989c78 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 11:40:29 +0100 Subject: [PATCH 197/658] Improve the pypi whl --- pyproject.toml | 2 +- setup.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3eea978b4f..c05abc8ab1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools>=60", "wheel", "cmind @ git+https://git@github.com/mlcommons/ck.git@a4c6a7b477af5f1e7099c55f5468a47854adaa6c#egg=cmind&subdirectory=cm"] +requires = ["setuptools>=60", "wheel", "cmind"] diff --git a/setup.py b/setup.py index 2ff4112866..5d7a093d35 100644 --- a/setup.py +++ b/setup.py @@ -127,11 +127,15 @@ def custom_function(self): def get_sys_platform(self): self.system = platform.system() +with open("README.md", "r") as f: + long_description = f.read() + setup( name='cm4mlops', - version='0.1', - long_description='CM automations and scripts for MLOps', - long_description_content_type='text/x-rst', + version='0.3', + long_description=long_description, + long_description_content_type='text/markdown', + url="https://github.com/mlcommons/cm4mlops", packages=[], install_requires=[ "setuptools>=60", From de8399add95ee1d1e9a30e4c55dfd01ba2f98195 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:49:11 +0530 Subject: [PATCH 198/658] pass annotation dir instead of file --- script/get-dataset-openimages/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-openimages/_cm.json b/script/get-dataset-openimages/_cm.json index c57f5ffbc1..a142d7b0f4 100644 --- a/script/get-dataset-openimages/_cm.json +++ b/script/get-dataset-openimages/_cm.json @@ -79,7 +79,7 @@ "CM_CALIBRATION_DATASET_PATH", "CM_CALIBRATION_DATASET_PATH_ROOT", "CM_OPENIMAGES_CALIBRATION_DATASET_PATH", - "CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH" + "CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH" ], "tags": [ "get", From 2348845fa3ac401313e8ec0ed43523480ccfa9fc Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:50:05 +0530 Subject: [PATCH 199/658] removed assigning of unused env variable --- script/get-dataset-openimages/customize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 2d788b8534..8bc9569a01 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -69,7 +69,6 @@ def postprocess(i): env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data') annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-mlperf.json") env['CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path - env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = annotations_file_path if env.get("CM_DATASET_OPENIMAGES_CUSTOM_ANNOTATIONS",'') == "yes": annotations_file_src = env['CM_DATASET_OPENIMAGES_ANNOTATIONS_FILE_PATH'] From 8232182f9fc3da04528650e7b3e558e05202cb96 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:55:27 +0530 Subject: [PATCH 200/658] Changed the mount from annotation file to dir --- script/app-mlperf-inference/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index c6b480efcd..54d7e3073a 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -1658,7 +1658,7 @@ docker: - "${{ CM_DATASET_IMAGENET_PATH }}:${{ CM_DATASET_IMAGENET_PATH }}" - "${{ CM_DATASET_OPENIMAGES_PATH }}:${{ CM_DATASET_OPENIMAGES_PATH }}" - "${{ CM_OPENIMAGES_CALIBRATION_DATASET_PATH }}:${{ CM_OPENIMAGES_CALIBRATION_DATASET_PATH }}" - - "${{ CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH }}:${{ CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH }}" + - "${{ CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH }}:${{ CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH }}" - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}" - "${{ OUTPUT_BASE_DIR }}:${{ OUTPUT_BASE_DIR }}" - "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}" From 372c63322d5e7b2d63dea69d76025cf8e47cabb9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:14:45 +0530 Subject: [PATCH 201/658] modified annotations env variable --- script/app-mlperf-inference-nvidia/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 49c2c876b5..72f6594559 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -141,7 +141,7 @@ def preprocess(i): dataset_path = env['CM_DATASET_OPENIMAGES_PATH'] #return {'return': 1, 'error': 'error'} - annotations_path = env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] + annotations_path = env['CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH'] target_data_path_dir = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'open-images-v6-mlperf') if not os.path.exists(target_data_path_dir): cmds.append(f"mkdir -p {target_data_path_dir}") From 4d3d163a87a133d2c7580a6026ff4f7ae27e03c4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 16:06:13 +0100 Subject: [PATCH 202/658] Create build_wheels.yml --- .github/workflows/build_wheels.yml | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/build_wheels.yml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml new file mode 100644 index 0000000000..80e3fd3238 --- /dev/null +++ b/.github/workflows/build_wheels.yml @@ -0,0 +1,86 @@ +name: Build wheels and release them into PYPI + +on: + release: + types: [published] + push: + branches: + - main + - mlperf_inference + paths: + - VERSION + - setup.py + +jobs: + build_wheels: + name: Build wheel + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + strategy: + fail-fast: false + steps: + # Step 1: Checkout the code + - uses: actions/checkout@v3 + + # Step 2: Set up Python + - uses: actions/setup-python@v3 + + # Step 3: Check if VERSION file has changed in this push + - name: Check if VERSION file has changed + id: version_changed + run: | + if git diff --name-only HEAD~1 | grep -q "VERSION"; then + echo "VERSION file has been modified" + echo "::set-output name=version_changed::true" + new_version=$(cat VERSION) + else + echo "VERSION file has NOT been modified" + echo "::set-output name=version_changed::false" + fi + echo "::set-output name=new_version::$new_version" + + # Step 4: Increment version if VERSION was not changed + - name: Increment version if necessary + if: steps.version_changed.outputs.version_changed == 'false' + run: | + # Check if VERSION file exists, else initialize it + if [ ! -f VERSION ]; then + echo "0.0.0" > VERSION + fi + + version=$(cat VERSION) + IFS='.' read -r major minor patch <<< "$version" + patch=$((patch + 1)) + new_version="$major.$minor.$patch" + echo $new_version > VERSION + echo "New version: $new_version" + echo "::set-output name=new_version::$new_version" + + # Step 5: Commit the updated version to the repository + - name: Commit updated version + run: | + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" + git add VERSION + git commit -m "Increment version to ${{ steps.version_changed.outputs.new_version }}" + git push + + # Step 6: Install required dependencies + - name: Install requirements + run: python3 -m pip install setuptools wheel + + # Step 7: Build the Python wheel + - name: Build wheels + run: python3 setup.py bdist_wheel + + # Step 8: Publish to PyPI + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verify-metadata: true + skip-existing: true + packages-dir: dist + repository-url: https://upload.pypi.org/legacy/ + verbose: true From d3b6a8e193d6247a20986dc2741878b66148988c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 16:07:25 +0100 Subject: [PATCH 203/658] Create VERSION --- VERSION | 1 + 1 file changed, 1 insertion(+) create mode 100644 VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000..9e11b32fca --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.1 From a97d60a3ba0828c6a99297590100d495b2ecfac4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 16:10:09 +0100 Subject: [PATCH 204/658] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9e11b32fca..0d91a54c7d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +0.3.0 From 3e86da0048a334ed4044d07594a20fc805e6b3e1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 20:47:27 +0530 Subject: [PATCH 205/658] Added nvidia mlperf inference server config for R50 --- script/app-mlperf-inference-nvidia/_cm.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index affb10611a..c7155c7296 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -89,7 +89,6 @@ input_mapping: embedding_weights_on_gpu_part: CM_MLPERF_NVIDIA_HARNESS_EMBEDDING_WEIGHTS_ON_GPU_PART sdxl_batcher_time_limit: CM_MLPERF_NVIDIA_HARNESS_SDXL_SERVER_BATCHER_TIME_LIMIT - # Dependencies on other CM scripts deps: @@ -1107,6 +1106,14 @@ variations: env: SKIP_POLICIES: '1' + server,resnet50: + env: + CM_MLPERF_NVIDIA_HARNESS_DEQUE_TIMEOUT_USEC: 2000 + CM_MLPERF_NVIDIA_HARNESS_USE_DEQUE_LIMIT: True + CM_MLPERF_NVIDIA_HARNESS_USE_CUDA_THREAD_PER_DEVICE: True + CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: 9 + CM_MLPERF_NVIDIA_HARNESS_GPU_INFERENCE_STREAMS: 2 + multistream,resnet50: env: SKIP_POLICIES: '1' From ce11cd156ee905d0ce8634470d1864db932479e2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 20:51:42 +0530 Subject: [PATCH 206/658] Support bool input for nvidia mlperf inference --- .../app-mlperf-inference-nvidia/customize.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 49c2c876b5..d15298f859 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -378,7 +378,7 @@ def preprocess(i): if audio_batch_size: run_config += f" --audio_batch_size={audio_batch_size}" - disable_encoder_plugin = env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_ENCODER_PLUGIN') + disable_encoder_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_ENCODER_PLUGIN')) if disable_encoder_plugin and disable_encoder_plugin.lower() not in [ "no", "false" ]: run_config += " --disable_encoder_plugin" @@ -393,11 +393,11 @@ def preprocess(i): if log_dir: run_config += f" --log_dir={log_dir}" - use_graphs = env.get('CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS') + use_graphs = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS')) if use_graphs and use_graphs.lower() not in [ "no", "false" ]: run_config += " --use_graphs" - use_deque_limit = env.get('CM_MLPERF_NVIDIA_HARNESS_USE_DEQUE_LIMIT') + use_deque_limit = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_DEQUE_LIMIT')) if use_deque_limit and use_deque_limit.lower() not in [ "no", "false" ]: run_config += " --use_deque_limit" @@ -405,19 +405,19 @@ def preprocess(i): if deque_timeout_usec: run_config += f" --deque_timeout_usec={deque_timeout_usec}" - use_cuda_thread_per_device = env.get('CM_MLPERF_NVIDIA_HARNESS_USE_CUDA_THREAD_PER_DEVICE') + use_cuda_thread_per_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_CUDA_THREAD_PER_DEVICE')) if use_cuda_thread_per_device and use_cuda_thread_per_device.lower() not in [ "no", "false" ]: run_config += " --use_cuda_thread_per_device" - run_infer_on_copy_streams = env.get('CM_MLPERF_NVIDIA_HARNESS_RUN_INFER_ON_COPY_STREAMS') + run_infer_on_copy_streams = str(env.get('CM_MLPERF_NVIDIA_HARNESS_RUN_INFER_ON_COPY_STREAMS')) if run_infer_on_copy_streams and run_infer_on_copy_streams.lower() not in [ "no", "false" ]: run_config += " --run_infer_on_copy_streams" - start_from_device = env.get('CM_MLPERF_NVIDIA_HARNESS_START_FROM_DEVICE') + start_from_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_START_FROM_DEVICE')) if start_from_device and start_from_device.lower() not in [ "no", "false" ]: run_config += " --start_from_device" - end_on_device = env.get('CM_MLPERF_NVIDIA_HARNESS_END_ON_DEVICE') + end_on_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_END_ON_DEVICE')) if end_on_device and end_on_device.lower() not in [ "no", "false" ]: run_config += " --end_on_device" @@ -437,7 +437,7 @@ def preprocess(i): if soft_drop: run_config += f" --soft_drop={soft_drop}" - use_small_tile_gemm_plugin = env.get('CM_MLPERF_NVIDIA_HARNESS_USE_SMALL_TILE_GEMM_PLUGIN') + use_small_tile_gemm_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_SMALL_TILE_GEMM_PLUGIN')) if use_small_tile_gemm_plugin and use_small_tile_gemm_plugin.lower() not in [ "no", "false" ]: run_config += f" --use_small_tile_gemm_plugin" @@ -445,7 +445,7 @@ def preprocess(i): if audio_buffer_num_lines: run_config += f" --audio_buffer_num_lines={audio_buffer_num_lines}" - use_fp8 = env.get('CM_MLPERF_NVIDIA_HARNESS_USE_FP8') + use_fp8 = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_FP8')) if use_fp8 and use_fp8.lower() not in [ "no", "false" ]: run_config += f" --use_fp8" @@ -473,7 +473,7 @@ def preprocess(i): if num_warmups != '': run_config += f" --num_warmups={num_warmups}" - skip_postprocess = env.get('CM_MLPERF_NVIDIA_HARNESS_SKIP_POSTPROCESS') + skip_postprocess = str(env.get('CM_MLPERF_NVIDIA_HARNESS_SKIP_POSTPROCESS')) if skip_postprocess and skip_postprocess.lower() not in [ "no", "false" ]: run_config += f" --skip_postprocess" From 85e7e6ff637cbc5631ea05e59af02c77494fb787 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 16:06:13 +0100 Subject: [PATCH 207/658] Create build_wheels.yml --- .github/workflows/build_wheels.yml | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/build_wheels.yml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml new file mode 100644 index 0000000000..80e3fd3238 --- /dev/null +++ b/.github/workflows/build_wheels.yml @@ -0,0 +1,86 @@ +name: Build wheels and release them into PYPI + +on: + release: + types: [published] + push: + branches: + - main + - mlperf_inference + paths: + - VERSION + - setup.py + +jobs: + build_wheels: + name: Build wheel + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + strategy: + fail-fast: false + steps: + # Step 1: Checkout the code + - uses: actions/checkout@v3 + + # Step 2: Set up Python + - uses: actions/setup-python@v3 + + # Step 3: Check if VERSION file has changed in this push + - name: Check if VERSION file has changed + id: version_changed + run: | + if git diff --name-only HEAD~1 | grep -q "VERSION"; then + echo "VERSION file has been modified" + echo "::set-output name=version_changed::true" + new_version=$(cat VERSION) + else + echo "VERSION file has NOT been modified" + echo "::set-output name=version_changed::false" + fi + echo "::set-output name=new_version::$new_version" + + # Step 4: Increment version if VERSION was not changed + - name: Increment version if necessary + if: steps.version_changed.outputs.version_changed == 'false' + run: | + # Check if VERSION file exists, else initialize it + if [ ! -f VERSION ]; then + echo "0.0.0" > VERSION + fi + + version=$(cat VERSION) + IFS='.' read -r major minor patch <<< "$version" + patch=$((patch + 1)) + new_version="$major.$minor.$patch" + echo $new_version > VERSION + echo "New version: $new_version" + echo "::set-output name=new_version::$new_version" + + # Step 5: Commit the updated version to the repository + - name: Commit updated version + run: | + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" + git add VERSION + git commit -m "Increment version to ${{ steps.version_changed.outputs.new_version }}" + git push + + # Step 6: Install required dependencies + - name: Install requirements + run: python3 -m pip install setuptools wheel + + # Step 7: Build the Python wheel + - name: Build wheels + run: python3 setup.py bdist_wheel + + # Step 8: Publish to PyPI + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verify-metadata: true + skip-existing: true + packages-dir: dist + repository-url: https://upload.pypi.org/legacy/ + verbose: true From fb0db128069c9ead016a1467ac1bf96cad9ef946 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 16:07:25 +0100 Subject: [PATCH 208/658] Create VERSION --- VERSION | 1 + 1 file changed, 1 insertion(+) create mode 100644 VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000..9e11b32fca --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.1 From b1248921265ac1857a9a792047a687e55fe7ffd8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 16:10:09 +0100 Subject: [PATCH 209/658] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9e11b32fca..0d91a54c7d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +0.3.0 From 93c563de2ea6ccbadf8ba2778e5caaf719703cfd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:16:59 +0100 Subject: [PATCH 210/658] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0d91a54c7d..9e11b32fca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.3.1 From 005815ff41ff1bd453afe95f0f83a10d5a55a34c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:18:36 +0100 Subject: [PATCH 211/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 80e3fd3238..cca0a6fb47 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -6,7 +6,7 @@ on: push: branches: - main - - mlperf_inference + - mlperf-inference paths: - VERSION - setup.py From 5ff71e9551287798784938f6802c18970a6cd999 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:21:11 +0100 Subject: [PATCH 212/658] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 5d7a093d35..9139b84334 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,6 @@ PackageNotFoundError = pkg_resources.DistributionNotFound - class CustomInstallCommand(install): def run(self): self.get_sys_platform() From 6cacfc71ebf62248428f365d3cf9794d5b9db26d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:26:41 +0100 Subject: [PATCH 213/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index cca0a6fb47..baf65d3a86 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -66,6 +66,8 @@ jobs: git add VERSION git commit -m "Increment version to ${{ steps.version_changed.outputs.new_version }}" git push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Step 6: Install required dependencies - name: Install requirements From 95bf9d7314936fa01f8c5d154974987ddb3ec007 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:30:22 +0100 Subject: [PATCH 214/658] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9139b84334..274ba5a5e8 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def is_package_installed(self, package_name): except PackageNotFoundError: return False + def install_system_packages(self): # List of packages to install via system package manager packages = [] From 2180eb5cad8c95a1998410970fe3a0faf32dc64b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:38:22 +0100 Subject: [PATCH 215/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index baf65d3a86..f240af3af4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -62,12 +62,11 @@ jobs: - name: Commit updated version run: | git config --global user.name "${{ github.actor }}" - git config --global user.email "${{ github.actor }}@users.noreply.github.com" + # git config --global user.email "${{ github.actor }}@users.noreply.github.com" + git config --global user.email "admin@gateoverflow.com" git add VERSION git commit -m "Increment version to ${{ steps.version_changed.outputs.new_version }}" - git push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + git push origin # Step 6: Install required dependencies - name: Install requirements From f67dd488f9ce9000b551a88e57670ab1afc10c78 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:39:01 +0100 Subject: [PATCH 216/658] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 274ba5a5e8..8f12c3f48d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ # setup.py + from setuptools import setup from setuptools._distutils.dist import Distribution from setuptools.command.install import install From 30ccde31079fa659d2b3b302ac4dfdceb71aef6a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:48:07 +0100 Subject: [PATCH 217/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f240af3af4..40f88a0023 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -43,6 +43,7 @@ jobs: # Step 4: Increment version if VERSION was not changed - name: Increment version if necessary + id: do_version_increment if: steps.version_changed.outputs.version_changed == 'false' run: | # Check if VERSION file exists, else initialize it @@ -60,12 +61,14 @@ jobs: # Step 5: Commit the updated version to the repository - name: Commit updated version + if: steps.version_changed.outputs.version_changed == 'false' run: | - git config --global user.name "${{ github.actor }}" + # git config --global user.name "${{ github.actor }}" # git config --global user.email "${{ github.actor }}@users.noreply.github.com" + git config --global user.name "gateoverflow-admin" git config --global user.email "admin@gateoverflow.com" git add VERSION - git commit -m "Increment version to ${{ steps.version_changed.outputs.new_version }}" + git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}" git push origin # Step 6: Install required dependencies From 33020f2e570d3f997b5728e6ea64d98be5387b79 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:49:17 +0100 Subject: [PATCH 218/658] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8f12c3f48d..a9d41e43eb 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# setup.py +# Build a whl file for cm4mlperf-inference from setuptools import setup from setuptools._distutils.dist import Distribution From 3d36bd1942f4bd2a8aadeb5fc880e6e3fa3a3c9c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:59:06 +0100 Subject: [PATCH 219/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 40f88a0023..af40f29f7e 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -67,6 +67,7 @@ jobs: # git config --global user.email "${{ github.actor }}@users.noreply.github.com" git config --global user.name "gateoverflow-admin" git config --global user.email "admin@gateoverflow.com" + #git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN_TOKEN }}@github.com/${{ github.repository }} git add VERSION git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}" git push origin From 28b178a60ebd4c2d02209855e9e354b0122d7cee Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 17:59:29 +0100 Subject: [PATCH 220/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index af40f29f7e..7ea5843469 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -10,6 +10,8 @@ on: paths: - VERSION - setup.py + - .github/workflows/build_wheels.yml + jobs: build_wheels: From 88cb4730086211dfa8765c1e08f05225004abd21 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:18:19 +0100 Subject: [PATCH 221/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7ea5843469..949425d264 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -20,6 +20,7 @@ jobs: environment: release permissions: id-token: write + contents: write strategy: fail-fast: false steps: @@ -65,11 +66,8 @@ jobs: - name: Commit updated version if: steps.version_changed.outputs.version_changed == 'false' run: | - # git config --global user.name "${{ github.actor }}" - # git config --global user.email "${{ github.actor }}@users.noreply.github.com" - git config --global user.name "gateoverflow-admin" - git config --global user.email "admin@gateoverflow.com" - #git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN_TOKEN }}@github.com/${{ github.repository }} + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" git add VERSION git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}" git push origin From 105dc7ffc8f4c004563c3aae77690638e77ce4a9 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:19:04 +0000 Subject: [PATCH 222/658] Increment version to 0.3.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9e11b32fca..d15723fbe8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +0.3.2 From 345e9012ae1d41aef07d018d8a635d82984ad8e1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:24:48 +0100 Subject: [PATCH 223/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 949425d264..e7f68ca911 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -78,7 +78,7 @@ jobs: # Step 7: Build the Python wheel - name: Build wheels - run: python3 setup.py bdist_wheel + run: python3 -m build # Step 8: Publish to PyPI - name: Publish to PyPI From ec34ab2e74f99e1ee8da81377f001ef87d7885f2 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:25:25 +0000 Subject: [PATCH 224/658] Increment version to 0.3.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d15723fbe8..1c09c74e22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.2 +0.3.3 From 78888f4d7ab43a70ba282ea244523e57888023bb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:29:49 +0100 Subject: [PATCH 225/658] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e7f68ca911..8d41a00ff7 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -74,7 +74,7 @@ jobs: # Step 6: Install required dependencies - name: Install requirements - run: python3 -m pip install setuptools wheel + run: python3 -m pip install setuptools wheel build # Step 7: Build the Python wheel - name: Build wheels From 9e2d1ca453bb61e3debafbf301c6b480d58736af Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:30:24 +0000 Subject: [PATCH 226/658] Increment version to 0.3.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1c09c74e22..42045acae2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.3 +0.3.4 From 569d6a01a3ce4375394d0b2116b4b0fbdd0e7c95 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:41:37 +0100 Subject: [PATCH 227/658] Update and rename build_wheels.yml to build_wheel.yml --- .github/workflows/{build_wheels.yml => build_wheel.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{build_wheels.yml => build_wheel.yml} (96%) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheel.yml similarity index 96% rename from .github/workflows/build_wheels.yml rename to .github/workflows/build_wheel.yml index 8d41a00ff7..0b4176f4ee 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheel.yml @@ -1,4 +1,4 @@ -name: Build wheels and release them into PYPI +name: Build wheel and release into PYPI on: release: @@ -10,7 +10,7 @@ on: paths: - VERSION - setup.py - - .github/workflows/build_wheels.yml + - .github/workflows/build_wheel.yml jobs: From bfc7569b396fc6d0a39eeddbc1d363b8d84d0335 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:42:15 +0000 Subject: [PATCH 228/658] Increment version to 0.3.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 42045acae2..c2c0004f0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.4 +0.3.5 From b3864abe53b041dae55c0de5aa1cdea240aaf84a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:45:24 +0100 Subject: [PATCH 229/658] Update setup.py --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a9d41e43eb..e343906b89 100644 --- a/setup.py +++ b/setup.py @@ -130,10 +130,12 @@ def get_sys_platform(self): with open("README.md", "r") as f: long_description = f.read() +with open("VERSION", "r") as f: + version = f.read() setup( name='cm4mlops', - version='0.3', + version=version, long_description=long_description, long_description_content_type='text/markdown', url="https://github.com/mlcommons/cm4mlops", From 4504a30c7f50adec994f529e02073d5534ccc932 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:46:01 +0000 Subject: [PATCH 230/658] Increment version to 0.3.6 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c2c0004f0e..449d7e73a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.5 +0.3.6 From 249b0c2a3a99dead737836330406f6547a07d28c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:49:36 +0100 Subject: [PATCH 231/658] Update VERSION From 96ffb931b39f5c28a86b85bad8ca90dc72ca4d22 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:51:38 +0100 Subject: [PATCH 232/658] Update build_wheel.yml --- .github/workflows/build_wheel.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 0b4176f4ee..f1f40d8034 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -18,9 +18,11 @@ jobs: name: Build wheel runs-on: ubuntu-latest environment: release + permissions: id-token: write contents: write + strategy: fail-fast: false steps: From 5682af17ea2cb3d5d03287b9d5e4fecb459de01b Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:52:13 +0000 Subject: [PATCH 233/658] Increment version to 0.3.7 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 449d7e73a9..0f82685331 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.6 +0.3.7 From 468bdf270eb187e77ff4e7c3885c34fa15f8d1ce Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 18:57:30 +0100 Subject: [PATCH 234/658] Update build_wheel.yml --- .github/workflows/build_wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index f1f40d8034..4dfcb5fc52 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -80,6 +80,7 @@ jobs: # Step 7: Build the Python wheel - name: Build wheels + working-directory: ./ run: python3 -m build # Step 8: Publish to PyPI From cca53eeb30e3ae647847241c95af9d721caa0a7f Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 17:58:05 +0000 Subject: [PATCH 235/658] Increment version to 0.3.8 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0f82685331..6678432209 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.7 +0.3.8 From 7b535bfabfef9653fc4b1314622a95428073f5d0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 19:01:49 +0100 Subject: [PATCH 236/658] Update setup.py --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index e343906b89..3a43d4fa00 100644 --- a/setup.py +++ b/setup.py @@ -128,10 +128,15 @@ def custom_function(self): def get_sys_platform(self): self.system = platform.system() -with open("README.md", "r") as f: - long_description = f.read() -with open("VERSION", "r") as f: - version = f.read() +# Read long description and version +def read_file(file_name, default=""): + if os.path.isfile(file_name): + with open(file_name, "r") as f: + return f.read().strip() + return default + +long_description = read_file("README.md", "No description available.") +version = read_file("VERSION", "0.3.0") setup( name='cm4mlops', From 04afa8c4ffc3c8b10091b3cb542cd98751166ead Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 18:02:22 +0000 Subject: [PATCH 237/658] Increment version to 0.3.9 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6678432209..940ac09aa6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.8 +0.3.9 From 37d4cd81c540394abe82a155769e3c3beffc1f6d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 19:04:40 +0100 Subject: [PATCH 238/658] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3a43d4fa00..93b2fb926b 100644 --- a/setup.py +++ b/setup.py @@ -136,11 +136,11 @@ def read_file(file_name, default=""): return default long_description = read_file("README.md", "No description available.") -version = read_file("VERSION", "0.3.0") +version_ = read_file("VERSION", "0.3.0") setup( name='cm4mlops', - version=version, + version=version_, long_description=long_description, long_description_content_type='text/markdown', url="https://github.com/mlcommons/cm4mlops", From 50b3d3a1bc7a6f7a0b574b8df922aaafece7d128 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 18:05:18 +0000 Subject: [PATCH 239/658] Increment version to 0.3.10 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 940ac09aa6..5503126d59 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.9 +0.3.10 From 6482e6cb866db96e459cd04d572bce1494e2f613 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 19:18:56 +0100 Subject: [PATCH 240/658] Update build_wheel.yml --- .github/workflows/build_wheel.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 4dfcb5fc52..6bde1c81c0 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -10,7 +10,6 @@ on: paths: - VERSION - setup.py - - .github/workflows/build_wheel.yml jobs: @@ -72,7 +71,7 @@ jobs: git config --global user.email "${{ github.actor }}@users.noreply.github.com" git add VERSION git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}" - git push origin + git push # Step 6: Install required dependencies - name: Install requirements From 488ceec0966a9b5bf79824f55063ffdc3440e981 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 19:29:19 +0100 Subject: [PATCH 241/658] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b1c1fb58cb..31ffd862e2 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ [![Test QAIC Software kit Compilation](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml) +* [Upcoming updates](https://github.com/mlcommons/cm4mlops/discussions/categories/announcements) +* [Ongoing Discussions](https://github.com/mlcommons/cm4mlops/discussions/categories/ideas) + The `mlperf-branch` of the **cm4mlops** repository contains updated CM scripts specifically for MLPerf Inference, including support for Automotive. Please note that the general CM scripts in this branch may not be compatible with other projects. For more information on using CM for MLPerf Inference, visit the [MLPerf Inference Documentation site](https://docs.mlcommons.org/inference/). [![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) From 38a907d319ed24a61e2e111cc170ccd94d6a661b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 19:29:47 +0100 Subject: [PATCH 242/658] Update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 31ffd862e2..da3177b25a 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,12 @@ [![Test QAIC Software kit Compilation](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml) -* [Upcoming updates](https://github.com/mlcommons/cm4mlops/discussions/categories/announcements) -* [Ongoing Discussions](https://github.com/mlcommons/cm4mlops/discussions/categories/ideas) - The `mlperf-branch` of the **cm4mlops** repository contains updated CM scripts specifically for MLPerf Inference, including support for Automotive. Please note that the general CM scripts in this branch may not be compatible with other projects. For more information on using CM for MLPerf Inference, visit the [MLPerf Inference Documentation site](https://docs.mlcommons.org/inference/). [![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) - +* [Upcoming updates](https://github.com/mlcommons/cm4mlops/discussions/categories/announcements) +* [Ongoing Discussions](https://github.com/mlcommons/cm4mlops/discussions/categories/ideas) ## License From 77f8b2b42fc416479c3a4009f5798ba1cacf7f18 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 19:30:31 +0100 Subject: [PATCH 243/658] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da3177b25a..be8fe1bf8c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ The `mlperf-branch` of the **cm4mlops** repository contains updated CM scripts s [![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) -* [Upcoming updates](https://github.com/mlcommons/cm4mlops/discussions/categories/announcements) +## News + +* [Upcoming Changes](https://github.com/mlcommons/cm4mlops/discussions/categories/announcements) * [Ongoing Discussions](https://github.com/mlcommons/cm4mlops/discussions/categories/ideas) ## License From 9fff82626a503187a21e4676392ce2255bd00f78 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 17 Oct 2024 19:11:24 +0000 Subject: [PATCH 244/658] Increment version to 0.3.11 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5503126d59..208059121d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.10 +0.3.11 From ac0a7ce23adaa79fc347ce1197b50ac383a0343a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 20:11:50 +0100 Subject: [PATCH 245/658] Update build_wheel.yml --- .github/workflows/build_wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 6bde1c81c0..5777c3a71d 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -14,6 +14,7 @@ on: jobs: build_wheels: + if: github.repository_owner == 'mlcommons' name: Build wheel runs-on: ubuntu-latest environment: release From 72638178f6c0db070653ced712a81d833d559df7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 01:18:06 +0530 Subject: [PATCH 246/658] Fix env corruption when scripts are skipped inside docker --- automation/script/module.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index b2b527b218..183f7c48c0 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -971,7 +971,19 @@ def _run(self, i): if state.get('docker'): if str(state['docker'].get('run', True)).lower() in ['false', '0', 'no']: logging.info(recursion_spaces+' - Skipping script::{} run as we are inside docker'.format(found_script_artifact)) - return {'return': 0} + + # restore env and state + for k in list(env.keys()): + del(env[k]) + for k in list(state.keys()): + del(state[k]) + + env.update(saved_env) + state.update(saved_state) + + rr = {'return':0, 'env':env, 'new_env':{}, 'state':state, 'new_state':{}, 'deps': []} + return rr + elif str(state['docker'].get('real_run', True)).lower() in ['false', '0', 'no']: logging.info(recursion_spaces+' - Doing fake run for script::{} as we are inside docker'.format(found_script_artifact)) fake_run = True From 18cb6e0d1ba64127c41566855925416fdf1068a2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Oct 2024 22:57:22 +0100 Subject: [PATCH 247/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index f0d8f45a1a..b46b9bd5ca 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "55 16 * * *" #to be adjusted + - cron: "55 1 * * *" #to be adjusted jobs: build_nvidia: From 077b930155ceffd16f2304f20c74f26fe34aa02a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:06:45 +0530 Subject: [PATCH 248/658] added annotation file papth to env key --- script/get-dataset-openimages/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-dataset-openimages/_cm.json b/script/get-dataset-openimages/_cm.json index a142d7b0f4..b93cf8aa0f 100644 --- a/script/get-dataset-openimages/_cm.json +++ b/script/get-dataset-openimages/_cm.json @@ -79,7 +79,8 @@ "CM_CALIBRATION_DATASET_PATH", "CM_CALIBRATION_DATASET_PATH_ROOT", "CM_OPENIMAGES_CALIBRATION_DATASET_PATH", - "CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH" + "CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH", + "CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH" ], "tags": [ "get", From c9bbd2a814e78b7ee93215752e2d7f099a056486 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:07:41 +0530 Subject: [PATCH 249/658] set validation annotation path env key --- script/get-dataset-openimages/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 8bc9569a01..84d1f52312 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -70,6 +70,7 @@ def postprocess(i): annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-mlperf.json") env['CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = annotations_file_path + env['CM_DATASET_OPENIMAGES_VALIDATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path if env.get("CM_DATASET_OPENIMAGES_CUSTOM_ANNOTATIONS",'') == "yes": annotations_file_src = env['CM_DATASET_OPENIMAGES_ANNOTATIONS_FILE_PATH'] shutil.copy(annotations_file_src, env['CM_DATASET_ANNOTATIONS_DIR_PATH']) From 16daf01cbe34ed6dd6a00f1a1c90759dbc57c225 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:27:07 +0530 Subject: [PATCH 250/658] utilise command generated in preprocess in batch file --- script/get-ml-model-huggingface-zoo/run.bat | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-huggingface-zoo/run.bat b/script/get-ml-model-huggingface-zoo/run.bat index 27155cb427..a87b0e6e99 100644 --- a/script/get-ml-model-huggingface-zoo/run.bat +++ b/script/get-ml-model-huggingface-zoo/run.bat @@ -1,2 +1,3 @@ -%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\download_model.py -IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% +echo %CM_RUN_CMD% +call %CM_RUN_CMD% +if errorlevel 1 exit /b %errorlevel% From 9103ace6092405a2a81d2774d82c588c6940b616 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:36:04 +0530 Subject: [PATCH 251/658] Update exit condition --- script/get-ml-model-huggingface-zoo/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-huggingface-zoo/run.bat b/script/get-ml-model-huggingface-zoo/run.bat index a87b0e6e99..6a4faa929d 100644 --- a/script/get-ml-model-huggingface-zoo/run.bat +++ b/script/get-ml-model-huggingface-zoo/run.bat @@ -1,3 +1,3 @@ echo %CM_RUN_CMD% call %CM_RUN_CMD% -if errorlevel 1 exit /b %errorlevel% +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% From 7194938e92bb30e0a0936839bfec710d22e88616 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 09:22:22 +0100 Subject: [PATCH 252/658] Update build_wheel.yml --- .github/workflows/build_wheel.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 6bde1c81c0..f589b00cc5 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -80,7 +80,8 @@ jobs: # Step 7: Build the Python wheel - name: Build wheels working-directory: ./ - run: python3 -m build + # run: python3 -m build + run: python3 setup.py bdist_wheel # Step 8: Publish to PyPI - name: Publish to PyPI From 29e648a5d1e46dc583bc71da2f30e028cdc054c7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 09:24:48 +0100 Subject: [PATCH 253/658] Update and rename publish.yaml to publish-docs.yaml --- .github/workflows/{publish.yaml => publish-docs.yaml} | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename .github/workflows/{publish.yaml => publish-docs.yaml} (89%) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish-docs.yaml similarity index 89% rename from .github/workflows/publish.yaml rename to .github/workflows/publish-docs.yaml index fa9ace5da0..e720258ecd 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish-docs.yaml @@ -1,6 +1,6 @@ # This is a basic workflow to help you get started with Actions -name: Publish site +name: Publish docs site on: @@ -11,6 +11,9 @@ on: - main - docs - mlperf-inference + paths: + - docs/** + - mkdocs.yml jobs: From f1d918bfb3d5e8479bc97912799d7cdd6863543a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 13:13:58 +0100 Subject: [PATCH 254/658] Fix warmup count R50 --- script/app-mlperf-inference-nvidia/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index c7155c7296..e0a35043dc 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -413,6 +413,7 @@ variations: CM_ML_MODEL_WEIGHT_TRANSFORMATIONS: quantization, affine fusion CM_ML_MODEL_INPUTS_DATA_TYPE: int8 CM_ML_MODEL_WEIGHTS_DATA_TYPE: int8 + CM_MLPERF_NVIDIA_HARNESS_NUM_WARMUPS: 10 deps: - tags: get,generic-python-lib,_onnx-graphsurgeon version: 0.3.27 From 7cdb7ecdd3d10d9774336d1d197e449c88aea49e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 18:35:22 +0530 Subject: [PATCH 255/658] Fix docker image name passing --- automation/script/module_misc.py | 2 +- script/run-docker-container/customize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index cedd3e8f50..78d0115d19 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1976,7 +1976,7 @@ def docker(i): device = i.get('docker_device', docker_settings.get('device')) - image_name = i.get('docker_image_name', docker_settings.get('image_name')) + image_name = i.get('docker_image_name', docker_settings.get('image_name', '')) r = check_gh_token(i, docker_settings, quiet) if r['return'] >0 : return r diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index a3971e4482..dc9510bff6 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -299,7 +299,7 @@ def update_docker_info(env): env['CM_DOCKER_IMAGE_BASE'] = docker_image_base - if env.get('CM_DOCKER_IMAGE_NAME', '') != '': + if env.get('CM_DOCKER_IMAGE_NAME', '') != '' and env['CM_DOCKER_IMAGE_NAME']: docker_image_name = env['CM_DOCKER_IMAGE_NAME'] else: docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-') From f7de50507a328e127f410e7167ca76b43240ab8e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 14:23:12 +0100 Subject: [PATCH 256/658] Fix pypi dist --- .github/workflows/build_wheel.yml | 3 +-- MANIFEST.in | 2 ++ VERSION | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 MANIFEST.in diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index f589b00cc5..bf471d4dd6 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -80,8 +80,7 @@ jobs: # Step 7: Build the Python wheel - name: Build wheels working-directory: ./ - # run: python3 -m build - run: python3 setup.py bdist_wheel + run: python3 -m build && rm dist/*.whl # Step 8: Publish to PyPI - name: Publish to PyPI diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..a17c59a62a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include VERSION diff --git a/VERSION b/VERSION index 5503126d59..e5a9958c32 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.10 +0.3.17 From 4f220ff11e206e1b70316f37ad64fa0a693baf26 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Fri, 18 Oct 2024 13:24:31 +0000 Subject: [PATCH 257/658] Increment version to 0.3.18 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e5a9958c32..8355eafc6f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.17 +0.3.18 From 7ec0d8e9ffa1281bfe22465b0ff0f7c12eb862b5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 19:06:05 +0530 Subject: [PATCH 258/658] Fix submissio generation --- script/run-mlperf-inference-app/_cm.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 9a2c8f23af..e92ab4253b 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -34,6 +34,7 @@ default_env: CM_MLPERF_IMPLEMENTATION: reference CM_MLPERF_MODEL: resnet50 CM_MLPERF_RUN_STYLE: test + CM_MLPERF_SKIP_SUBMISSION_GENERATION: no input_mapping: api_server: CM_MLPERF_INFERENCE_API_SERVER @@ -381,9 +382,9 @@ variations: post_deps: - names: - submission-generator - enable_if_env: + skip_if_env: CM_MLPERF_SKIP_SUBMISSION_GENERATION: - - 'no' + - 'yes' tags: generate,mlperf,inference,submission versions: From 59cc626955aee9bb24650411347831c8e2eecec1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 19:07:14 +0530 Subject: [PATCH 259/658] Enable closed-datacenter run for nvidia --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index b46b9bd5ca..ff0ffa297d 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -22,5 +22,5 @@ jobs: export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=90000 --server_target_qps=72000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 09d3239e273256085b22ff6a7454cc799c9e027e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 14:38:11 +0100 Subject: [PATCH 260/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index ff0ffa297d..97fb9edd97 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "55 1 * * *" #to be adjusted + - cron: "45 13 * * *" #to be adjusted jobs: build_nvidia: From f5024e1ba2b74de9d80656e12c015b60334cbd7a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 16:02:08 +0100 Subject: [PATCH 261/658] Handle venv detect in setup.py --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 93b2fb926b..17fd539655 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,12 @@ def run(self): return self.custom_function() def is_package_installed(self, package_name): + if package_name == "venv": + try: + import venv + return True + except ImportError:: + return False try: if sys.version_info >= (3, 8): version(package_name) # Tries to get the version of the package From 3e214f99691482f0ed14509bfbe5bfa72dd4c161 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Fri, 18 Oct 2024 15:02:53 +0000 Subject: [PATCH 262/658] Increment version to 0.3.19 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8355eafc6f..9ec8136c25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.18 +0.3.19 From b12cc987067b87e542dba77b4260aec170a236d2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 16:03:15 +0100 Subject: [PATCH 263/658] Handle venv detect in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17fd539655..c5e096ca6b 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ def is_package_installed(self, package_name): try: import venv return True - except ImportError:: + except ImportError: return False try: if sys.version_info >= (3, 8): From df8bd61e134152b07bfcad3aed2a7784533cdbee Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Fri, 18 Oct 2024 15:04:41 +0000 Subject: [PATCH 264/658] Increment version to 0.3.20 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9ec8136c25..f9a4b5f993 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.19 +0.3.20 From bf538f9db929c6e13a187f164b0aa79b2e383298 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 16:26:44 +0100 Subject: [PATCH 265/658] Fix package detect in setup.py --- setup.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index c5e096ca6b..dd69c63eb0 100644 --- a/setup.py +++ b/setup.py @@ -35,15 +35,12 @@ def run(self): return self.custom_function() def is_package_installed(self, package_name): - if package_name == "venv": - try: - import venv - return True - except ImportError: - return False try: if sys.version_info >= (3, 8): - version(package_name) # Tries to get the version of the package + spec = importlib.util.find_spec(package_name)) + module = importlib.util.module_from_spec(spec) + sys.modules[package_name] = module + spec.loader.exec_module(module) else: pkg_resources.get_distribution(package_name) # Fallback for < 3.8 return True From af0478fe8c89f9227a41468516c2421dbb3b4cc1 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Fri, 18 Oct 2024 15:27:35 +0000 Subject: [PATCH 266/658] Increment version to 0.3.21 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f9a4b5f993..dfdc368868 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.20 +0.3.21 From d1fec40620922cb64957cd92c3c8d33365b43d11 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 16:29:33 +0100 Subject: [PATCH 267/658] Fix package detect in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dd69c63eb0..48e844e593 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def run(self): def is_package_installed(self, package_name): try: if sys.version_info >= (3, 8): - spec = importlib.util.find_spec(package_name)) + spec = importlib.util.find_spec(package_name) module = importlib.util.module_from_spec(spec) sys.modules[package_name] = module spec.loader.exec_module(module) From 5aeaffdca72142871dcde95ebf8a37e65fe3e06e Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Fri, 18 Oct 2024 15:30:23 +0000 Subject: [PATCH 268/658] Increment version to 0.3.22 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dfdc368868..0c4b454928 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.21 +0.3.22 From 948c9f886072e04245eb7c285518e285ec7c5589 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 21:21:48 +0530 Subject: [PATCH 269/658] Fix for compliance run --- script/app-mlperf-inference/customize.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index c1fe196828..d0587b80db 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -122,6 +122,18 @@ def postprocess(i): pass # Not giving an error now. But accuracy paths need to be done for other benchmarks which may need the non-determinism test #return {'return': 1, 'error': f'Accuracy paths not done for model {model}'} scenario = env['CM_MLPERF_LOADGEN_SCENARIO'] + + if not state.get('cm-mlperf-inference-results'): + state['cm-mlperf-inference-results'] = {} + if not state.get('cm-mlperf-inference-results-last'): + state['cm-mlperf-inference-results-last'] = {} + if not state['cm-mlperf-inference-results'].get(state['CM_SUT_CONFIG_NAME']): + state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']] = {} + if not state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']].get(model): + state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model] = {} + if not state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model].get(scenario): + state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario] = {} + #if env.get("CM_MLPERF_FIND_PERFORMANCE_MODE", '') == "yes" and mode == "performance" and scenario != "Server": if mode == "performance" and scenario != "Server": @@ -239,17 +251,6 @@ def postprocess(i): power = power_result_split[0] power_efficiency = power_result_split[1] - if not state.get('cm-mlperf-inference-results'): - state['cm-mlperf-inference-results'] = {} - if not state.get('cm-mlperf-inference-results-last'): - state['cm-mlperf-inference-results-last'] = {} - if not state['cm-mlperf-inference-results'].get(state['CM_SUT_CONFIG_NAME']): - state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']] = {} - if not state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']].get(model): - state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model] = {} - if not state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model].get(scenario): - state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario] = {} - state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario][mode] = result state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario][mode+'_valid'] = valid.get(mode, False) From 01322c466c04cf130ff475ee8d25707eadadc636 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 22:21:55 +0530 Subject: [PATCH 270/658] Fix for docker image name --- script/run-docker-container/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index dc9510bff6..a3971e4482 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -299,7 +299,7 @@ def update_docker_info(env): env['CM_DOCKER_IMAGE_BASE'] = docker_image_base - if env.get('CM_DOCKER_IMAGE_NAME', '') != '' and env['CM_DOCKER_IMAGE_NAME']: + if env.get('CM_DOCKER_IMAGE_NAME', '') != '': docker_image_name = env['CM_DOCKER_IMAGE_NAME'] else: docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-') From ccfadef582c22091d2c45070a17fce57830ca5eb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 23:28:42 +0530 Subject: [PATCH 271/658] Use fork for Nvidia self hosted run --- .github/workflows/test-nvidia-mlperf-implementation.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 97fb9edd97..d76763f70b 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -20,7 +20,8 @@ jobs: python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - pip install --upgrade cm4mlops - cm pull repo + pip install --upgrade cmind + cm rm repo mlcommons@cm4mlops + cm pull repo gateoverflow@cm4mlops cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From e73069e9bfec5b55c9892cbc8dab7c03a8d908b3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 19:01:22 +0100 Subject: [PATCH 272/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index d76763f70b..de393ec327 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "45 13 * * *" #to be adjusted + - cron: "08 18 * * *" #to be adjusted jobs: build_nvidia: From 8e472d7af0fc357cff0c8538091fd70a5deb036e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 19:32:28 +0100 Subject: [PATCH 273/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index de393ec327..ec35c971d4 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "08 18 * * *" #to be adjusted + - cron: "41 18 * * *" #to be adjusted jobs: build_nvidia: From 6ef938843966b465d63a8faa4988824d9f6dbd5b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 19:49:17 +0100 Subject: [PATCH 274/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index ec35c971d4..5e3d4ed620 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "41 18 * * *" #to be adjusted + - cron: "55 18 * * *" #to be adjusted jobs: build_nvidia: From 1497712794fc1e854a968a3bf6d35bb0bcabd3b5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 20:01:27 +0100 Subject: [PATCH 275/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 5e3d4ed620..8ae58f357a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -21,7 +21,7 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cmind - cm rm repo mlcommons@cm4mlops + cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From b9785e94cf7578be9a1bf8ad07cb06e15c94449e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 20:04:32 +0100 Subject: [PATCH 276/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 8ae58f357a..5e6e166c98 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "55 18 * * *" #to be adjusted + - cron: "07 19 * * *" #to be adjusted jobs: build_nvidia: From 1f93c11be3fd4cd5ec55126c04b8228257f46cad Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 20:04:49 +0100 Subject: [PATCH 277/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 5e6e166c98..4f536708ae 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "07 19 * * *" #to be adjusted + - cron: "10 19 * * *" #to be adjusted jobs: build_nvidia: From bcda8eac3b8a0ecdd96b54a88777cff12853371c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 20:22:09 +0100 Subject: [PATCH 278/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 4f536708ae..19483699e3 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "10 19 * * *" #to be adjusted + - cron: "28 19 * * *" #to be adjusted jobs: build_nvidia: From bc2c8fb25b9c488bd97b02ff0953e52fbff0cd27 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 23:14:39 +0100 Subject: [PATCH 279/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 19483699e3..b70704b8ff 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -23,5 +23,5 @@ jobs: pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 0179dec7444249383219e6fcd3c85910554a0b19 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Oct 2024 23:40:04 +0100 Subject: [PATCH 280/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index b70704b8ff..ae6b6d4c20 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "28 19 * * *" #to be adjusted + - cron: "2 1 * * *" #to be adjusted jobs: build_nvidia: @@ -23,5 +23,5 @@ jobs: pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=open --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From c53de39ff54a5b57fd9c84b6c3ee2c74f408fd75 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 19 Oct 2024 13:46:01 +0530 Subject: [PATCH 281/658] Improve R50 rtx4090 config --- script/app-mlperf-inference-nvidia/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index e0a35043dc..7c94008fe2 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -1426,6 +1426,7 @@ variations: rtx_4090,resnet50,server,run_harness: default_variations: batch-size: batch_size.32 + use_graphs: "True" rtx_4090,retinanet,offline,run_harness: default_variations: From 3ac9f6221c1d3922df7fe0ca86a750101941cb38 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 19 Oct 2024 10:18:09 +0100 Subject: [PATCH 282/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index ae6b6d4c20..3049076588 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -23,5 +23,5 @@ jobs: pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=open --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=open --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 46d0108852f6fcc0781fc94dbb8552c7e647c4eb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 19 Oct 2024 11:02:42 +0100 Subject: [PATCH 283/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 5b73cda70b..5527be3fac 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -1,7 +1,7 @@ name: MLPerf inference SDXL on: schedule: - - cron: "30 9 * * *" + - cron: "30 2 * * *" jobs: build_reference: From 994a96a9640c7396d9c8f7a72b308c536d0f2abf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 19 Oct 2024 13:11:10 +0100 Subject: [PATCH 284/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 3049076588..aa782a8a4a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -23,5 +23,5 @@ jobs: pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=75000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=open --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From ccd444261d5ab9c380ac4cbbd26345a4419fbf0e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 20 Oct 2024 15:56:31 +0100 Subject: [PATCH 285/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index aa782a8a4a..2aff3b3595 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -23,5 +23,5 @@ jobs: pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 08e58af594e48e28b362bca670cb400d2f3d19fc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 15:09:14 +0530 Subject: [PATCH 286/658] Support for dc,edge category --- script/run-mlperf-inference-app/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index a60060edfb..1c80ee0b82 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -317,7 +317,7 @@ def get_valid_scenarios(model, category, mlperf_version, mlperf_path): internal_model_name = config[mlperf_version]["model_mapping"].get(model, model) - valid_scenarios = config[mlperf_version]["required-scenarios-"+category][internal_model_name] + valid_scenarios = config[mlperf_version]["required-scenarios-"+category.replace(",", "-")][internal_model_name] print("Valid Scenarios for " + model + " in " + category + " category are :" + str(valid_scenarios)) From 6f042b07b9e0c8ce455097dd1f5d4ef883949f10 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 10:40:13 +0100 Subject: [PATCH 287/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 2aff3b3595..c25db52e90 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "2 1 * * *" #to be adjusted + - cron: "47 9 * * *" #to be adjusted jobs: build_nvidia: From 3252b96c9bcd2a358f725b5e77f0d83031beaa34 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:32:04 +0530 Subject: [PATCH 288/658] Added base image for amd r4.1 --- script/app-mlperf-inference/_cm.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 54d7e3073a..255b351d72 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -456,6 +456,10 @@ variations: cnndm-accuracy-script: tags: _int32 + amd,r4.1_default: + docker: + base_image: rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging + amd: group: implementation From 1e2162501fc647bc1195755cc8ee9e6161a76c3c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 12:18:51 +0100 Subject: [PATCH 289/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index c25db52e90..5d1d80b92d 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -24,4 +24,4 @@ jobs: cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 547f3b444973fcad6155fa0228fff90bba8b4652 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 13:10:37 +0100 Subject: [PATCH 290/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 5d1d80b92d..274b283692 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "47 9 * * *" #to be adjusted + - cron: "17 12 * * *" #to be adjusted jobs: build_nvidia: @@ -23,5 +23,5 @@ jobs: pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f cm pull repo gateoverflow@cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From b726a0096d4a94997784e6bac1852a937b648a5b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 13:33:30 +0100 Subject: [PATCH 291/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 274b283692..3942b5c36d 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "17 12 * * *" #to be adjusted + - cron: "40 12 * * *" #to be adjusted jobs: build_nvidia: From 114709c8f6dbefa9ce5f8a599d55b349b5464bca Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 14:03:53 +0100 Subject: [PATCH 292/658] Update _cm.yaml --- script/get-mlperf-inference-loadgen/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-mlperf-inference-loadgen/_cm.yaml b/script/get-mlperf-inference-loadgen/_cm.yaml index a097a1edde..5035a07e6a 100644 --- a/script/get-mlperf-inference-loadgen/_cm.yaml +++ b/script/get-mlperf-inference-loadgen/_cm.yaml @@ -24,6 +24,7 @@ deps: - CM_GIT_CHECKOUT names: - inference-src-loadgen + - inference-src skip_if_env: CM_MLPERF_INFERENCE_LOADGEN_DOWNLOAD: - 'YES' From 8690324a50267a122ae2e1b6a92a86a555e8d35c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:00:57 +0530 Subject: [PATCH 293/658] added devices as extra run args --- script/app-mlperf-inference/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 255b351d72..45094cacbc 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -459,6 +459,7 @@ variations: amd,r4.1_default: docker: base_image: rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging + extra_run_args: ' --device=/dev/kfd --device=/dev/dri --device=/dev/mem' amd: group: From c9a9dc30b49a061d164c62660ec45d99e27eba83 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 23:22:38 +0530 Subject: [PATCH 294/658] Fix deprecation warning in detect-version --- script/get-generic-python-lib/detect-version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-python-lib/detect-version.py b/script/get-generic-python-lib/detect-version.py index 8cd53515ae..86ab8adf0b 100644 --- a/script/get-generic-python-lib/detect-version.py +++ b/script/get-generic-python-lib/detect-version.py @@ -1,4 +1,5 @@ import os +import sys package_name = os.environ.get('CM_GENERIC_PYTHON_PACKAGE_NAME','') @@ -18,7 +19,7 @@ except Exception as e: error = format(e) - if error != '': + if error != '' and sys.version_info < (3, 9): try: import pkg_resources version = pkg_resources.get_distribution(package_name).version From 4e954c3d500c8dbdeef9a7c05eb7700fbc37e4d7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 01:37:26 +0530 Subject: [PATCH 295/658] Fix inference vision path --- script/get-mlperf-inference-src/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py index 7ba1b62be3..63b2a59895 100644 --- a/script/get-mlperf-inference-src/customize.py +++ b/script/get-mlperf-inference-src/customize.py @@ -85,7 +85,7 @@ def postprocess(i): state = i['state'] inference_root = env['CM_MLPERF_INFERENCE_SOURCE'] - env['CM_MLPERF_INFERENCE_VISION_PATH'] = os.path.join(inference_root, 'inference', 'vision') + env['CM_MLPERF_INFERENCE_VISION_PATH'] = os.path.join(inference_root, 'vision') env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'] = os.path.join(inference_root, 'vision', 'classification_and_detection') env['CM_MLPERF_INFERENCE_BERT_PATH'] = os.path.join(inference_root, 'language', 'bert') env['CM_MLPERF_INFERENCE_GPTJ_PATH'] = os.path.join(inference_root, 'language', 'gpt-j') From 2ba815e1d5d974215d4e76e09250e9ba92331ffb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 02:20:29 +0530 Subject: [PATCH 296/658] Export VERSION for mlperf inference src --- script/get-mlperf-inference-src/_cm.json | 1 + script/get-mlperf-inference-src/customize.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/script/get-mlperf-inference-src/_cm.json b/script/get-mlperf-inference-src/_cm.json index 9bd2f4c3aa..2b62b7f5a7 100644 --- a/script/get-mlperf-inference-src/_cm.json +++ b/script/get-mlperf-inference-src/_cm.json @@ -32,6 +32,7 @@ "CM_MLPERF_INFERENCE_GPTJ_PATH", "CM_MLPERF_INFERENCE_RNNT_PATH", "CM_MLPERF_INFERENCE_SOURCE", + "CM_MLPERF_INFERENCE_SOURCE_VERSION", "CM_MLPERF_INFERENCE_VERSION", "CM_MLPERF_INFERENCE_VISION_PATH", "CM_MLPERF_LAST_RELEASE", diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py index 63b2a59895..de2b83b9cc 100644 --- a/script/get-mlperf-inference-src/customize.py +++ b/script/get-mlperf-inference-src/customize.py @@ -101,6 +101,11 @@ def postprocess(i): env['+PYTHONPATH']=[] env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], 'python')) + if os.path.exists(os.path.join(inference_root, "loadgen", "VERSION")): + with open(os.path.join(inference_root, "loadgen", "VERSION") as f: + version_info = f.read() + env['CM_MLPERF_INFERENCE_SOURCE_VERSION'] = version_info + if env.get('CM_GET_MLPERF_IMPLEMENTATION_ONLY', '') == "yes": return {'return':0} From 05e8156975ffbff2947427e1b5f697ebe2c0803a Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Mon, 21 Oct 2024 20:53:52 +0000 Subject: [PATCH 297/658] Increment version to 0.3.23 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7ae1b8574c..a1dad2aa79 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.22 \ No newline at end of file +0.3.23 From dde159653c08a20cac7278525f35c557d4ec72f4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 02:26:44 +0530 Subject: [PATCH 298/658] Do not pass mlperf_conf for the reference implementation if version >=4.1.1 --- .../customize.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 0376293484..76b5352f6a 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -70,10 +70,15 @@ def preprocess(i): x="" if os_info['platform'] == 'windows' else "'" - if "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"]: - env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf " + x+ env['CM_MLPERF_CONF'] + x + + inference_src_version = env.get('CM_MLPERF_INFERENCE_SOURCE_VERSION', '') + if inference_src_version and inference_src_version >= (4,1,1): + pass # mlperf_conf is automatically loaded by the loadgen else: - env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf "+ x + env['CM_MLPERF_CONF'] + x + if "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"]: + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf " + x+ env['CM_MLPERF_CONF'] + x + else: + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf "+ x + env['CM_MLPERF_CONF'] + x if env.get('CM_NETWORK_LOADGEN', '') != "lon" and env.get('CM_MLPERF_INFERENCE_API_SERVER','')=='' and "llama2-70b" not in env['CM_MODEL']: env['MODEL_DIR'] = env.get('CM_ML_MODEL_PATH') From 6d0f66371334be66f448e87e4958031c9b68f018 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 02:33:34 +0530 Subject: [PATCH 299/658] Fix syntax --- script/get-mlperf-inference-src/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py index de2b83b9cc..e8e94ae0a2 100644 --- a/script/get-mlperf-inference-src/customize.py +++ b/script/get-mlperf-inference-src/customize.py @@ -102,7 +102,7 @@ def postprocess(i): env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], 'python')) if os.path.exists(os.path.join(inference_root, "loadgen", "VERSION")): - with open(os.path.join(inference_root, "loadgen", "VERSION") as f: + with open(os.path.join(inference_root, "loadgen", "VERSION")) as f: version_info = f.read() env['CM_MLPERF_INFERENCE_SOURCE_VERSION'] = version_info From ae6b5e59cab67afe94292c0e55ecf643b3aef713 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 02:40:06 +0530 Subject: [PATCH 300/658] Fix version check for inference-src --- script/app-mlperf-inference-mlcommons-python/customize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 76b5352f6a..fc13755cfa 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -72,7 +72,11 @@ def preprocess(i): x="" if os_info['platform'] == 'windows' else "'" inference_src_version = env.get('CM_MLPERF_INFERENCE_SOURCE_VERSION', '') - if inference_src_version and inference_src_version >= (4,1,1): + inference_tuple = None + if inference_src_version: + version_tuple = tuple(map(int, inference_src_version.split('.'))) + + if version_tuple and version_tuple >= (4,1,1): pass # mlperf_conf is automatically loaded by the loadgen else: if "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"]: From 1dafcaed44bb0bd3267688be55659857be10ebe1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 23:10:34 +0100 Subject: [PATCH 301/658] Added no-compilation-warning option for loadgen --- script/get-mlperf-inference-loadgen/_cm.yaml | 5 +++++ script/get-mlperf-inference-loadgen/run.sh | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/script/get-mlperf-inference-loadgen/_cm.yaml b/script/get-mlperf-inference-loadgen/_cm.yaml index 5035a07e6a..952f72efa0 100644 --- a/script/get-mlperf-inference-loadgen/_cm.yaml +++ b/script/get-mlperf-inference-loadgen/_cm.yaml @@ -151,6 +151,11 @@ variations: CM_MLPERF_INFERENCE_LOADGEN_DOWNLOAD_URL: https://www.dropbox.com/scl/fi/gk5e9kziju5t56umxyzyx/loadgen.zip?rlkey=vsie4xnzml1inpjplm5cg7t54&dl=0 CM_MLPERF_INFERENCE_LOADGEN_VERSION: v4.0 CM_VERIFY_SSL: false + no-compilation-warning: + env: + '+ CXXFLAGS': + - '-Werror' + - '-Wno-unused-parameter' versions: custom: diff --git a/script/get-mlperf-inference-loadgen/run.sh b/script/get-mlperf-inference-loadgen/run.sh index d9acb121b8..3ee6196663 100644 --- a/script/get-mlperf-inference-loadgen/run.sh +++ b/script/get-mlperf-inference-loadgen/run.sh @@ -40,14 +40,8 @@ if [[ $CM_MLPERF_INFERENCE_LOADGEN_BUILD_CLEAN == "yes" ]]; then rm -rf build fi -PYTHON_VERSION=`${CM_PYTHON_BIN_WITH_PATH} -V |cut -d' ' -f2` -PYTHON_SHORT_VERSION=${PYTHON_VERSION%.*} -PYTHON_MINOR_VERSION=${PYTHON_SHORT_VERSION#*.} -MLPERF_INFERENCE_PYTHON_SITE_BASE=${INSTALL_DIR}"/python" cd "${CM_MLPERF_INFERENCE_SOURCE}/loadgen" -#CFLAGS="-std=c++14 -O3" ${CM_PYTHON_BIN_WITH_PATH} setup.py bdist_wheel -#${CM_PYTHON_BIN_WITH_PATH} -m pip install --force-reinstall `ls dist/mlperf_loadgen-*cp3${PYTHON_MINOR_VERSION}*.whl` --target="${MLPERF_INFERENCE_PYTHON_SITE_BASE}" ${CM_PYTHON_BIN_WITH_PATH} -m pip install . --target="${MLPERF_INFERENCE_PYTHON_SITE_BASE}" if [ "${?}" != "0" ]; then exit 1; fi From a362d2041ebc4958a4dc3b227121679cd29f3f60 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Oct 2024 23:22:33 +0100 Subject: [PATCH 302/658] Fix typo --- script/get-mlperf-inference-loadgen/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-loadgen/_cm.yaml b/script/get-mlperf-inference-loadgen/_cm.yaml index 952f72efa0..3ce1f83992 100644 --- a/script/get-mlperf-inference-loadgen/_cm.yaml +++ b/script/get-mlperf-inference-loadgen/_cm.yaml @@ -151,7 +151,7 @@ variations: CM_MLPERF_INFERENCE_LOADGEN_DOWNLOAD_URL: https://www.dropbox.com/scl/fi/gk5e9kziju5t56umxyzyx/loadgen.zip?rlkey=vsie4xnzml1inpjplm5cg7t54&dl=0 CM_MLPERF_INFERENCE_LOADGEN_VERSION: v4.0 CM_VERIFY_SSL: false - no-compilation-warning: + no-compilation-warnings: env: '+ CXXFLAGS': - '-Werror' From d81793fc48e3dac74369389d7dc5b7a19ead0737 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 05:01:16 +0530 Subject: [PATCH 303/658] VERSION -> VERSION.txt for mlperf inference loadgen --- script/get-mlperf-inference-src/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py index e8e94ae0a2..2397d25206 100644 --- a/script/get-mlperf-inference-src/customize.py +++ b/script/get-mlperf-inference-src/customize.py @@ -101,8 +101,8 @@ def postprocess(i): env['+PYTHONPATH']=[] env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], 'python')) - if os.path.exists(os.path.join(inference_root, "loadgen", "VERSION")): - with open(os.path.join(inference_root, "loadgen", "VERSION")) as f: + if os.path.exists(os.path.join(inference_root, "loadgen", "VERSION.txt")): + with open(os.path.join(inference_root, "loadgen", "VERSION.txt")) as f: version_info = f.read() env['CM_MLPERF_INFERENCE_SOURCE_VERSION'] = version_info From 352f37d7fd700fc812aaea2c0eefcfe817cec6f6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 05:04:12 +0530 Subject: [PATCH 304/658] Fix typo --- script/app-mlperf-inference-mlcommons-python/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index fc13755cfa..efff6513d9 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -72,7 +72,7 @@ def preprocess(i): x="" if os_info['platform'] == 'windows' else "'" inference_src_version = env.get('CM_MLPERF_INFERENCE_SOURCE_VERSION', '') - inference_tuple = None + version_tuple = None if inference_src_version: version_tuple = tuple(map(int, inference_src_version.split('.'))) From de33a78c1c93e8fb6e4307a85df648cfcd109b9a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 10:54:31 +0100 Subject: [PATCH 305/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 40f563f460..da9595d285 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -24,6 +24,7 @@ jobs: python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM + cm rm repo mlcommons@cm4mlops -f pip install --upgrade cm4mlops pip install tabulate cm pull repo From 35530aaf756053bdf9924e0fddfee62231e6e357 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 10:55:41 +0100 Subject: [PATCH 306/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 3942b5c36d..3237c359f5 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "40 12 * * *" #to be adjusted + - cron: "00 10 * * *" #to be adjusted jobs: build_nvidia: @@ -20,8 +20,7 @@ jobs: python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - pip install --upgrade cmind cm rm repo mlcommons@cm4mlops -f - cm pull repo gateoverflow@cm4mlops + pip install --upgrade cm4mlops cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions From 78d334367e6e6118c252c4bae33e33a2ed0fc69a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 10:56:06 +0100 Subject: [PATCH 307/658] Update test-mlperf-inference-sdxl.yaml --- .github/workflows/test-mlperf-inference-sdxl.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 5527be3fac..580988e0ae 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -19,6 +19,7 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM + cm rm repo mlcommons@cm4mlops -f python3 -m pip install cm4mlops cm pull repo cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean From 933a3256d72e59f476d93889910a14e99123446b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 17:13:23 +0530 Subject: [PATCH 308/658] Support 8G Nvidia GPUs --- script/app-mlperf-inference-nvidia/_cm.yaml | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 7c94008fe2..ca06f7cfb3 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -939,6 +939,7 @@ variations: - bert-99.9 - dlrm-v2-99 - dlrm-v2-99.9 + - tags: reproduce,mlperf,inference,nvidia,harness,_calibrate inherit_variation_tags: true enable_if_env: @@ -1200,6 +1201,10 @@ variations: default_variations: batch-size: batch_size.8 + gpu_memory.8,bert_,offline,run_harness: + default_variations: + batch-size: batch_size.256 + gpu_memory.16,bert_,offline,run_harness: default_variations: batch-size: batch_size.256 @@ -1224,6 +1229,12 @@ variations: default_variations: batch-size: batch_size.64 + gpu_memory.8,resnet50,offline,run_harness: + default_variations: + batch-size: batch_size.64 + env: + CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "4" + gpu_memory.16,resnet50,offline,run_harness: default_variations: batch-size: batch_size.1024 @@ -1277,6 +1288,10 @@ variations: default_variations: batch-size: batch_size.2 + gpu_memory.8,retinanet,offline,run_harness: + default_variations: + batch-size: batch_size.2 + gpu_memory.16,retinanet,offline,run_harness: default_variations: batch-size: batch_size.2 @@ -1311,6 +1326,10 @@ variations: CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "2" CM_MLPERF_NVIDIA_HARNESS_GPU_INFERENCE_STREAMS: "2" + gpu_memory.8,rnnt,offline,run_harness: + default_variations: + batch-size: batch_size.1024 + gpu_memory.16,rnnt,offline,run_harness: default_variations: batch-size: batch_size.1024 @@ -1335,6 +1354,10 @@ variations: default_variations: batch-size: batch_size.2048 + gpu_memory.8,3d-unet_,offline,run_harness: + default_variations: + batch-size: batch_size.4 + gpu_memory.16,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size.4 From 9cfa143c11fdc6456d62720b5a17baf0ed800a50 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 19:01:51 +0530 Subject: [PATCH 309/658] Fix benchmark-program to exit on error --- script/benchmark-program/run.sh | 46 ++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/script/benchmark-program/run.sh b/script/benchmark-program/run.sh index cb4eb92046..162c281d60 100755 --- a/script/benchmark-program/run.sh +++ b/script/benchmark-program/run.sh @@ -57,19 +57,41 @@ fi echo $CM_PRE_RUN_CMD eval ${CM_PRE_RUN_CMD} -# Check CM_RUN_CMD0 -if [[ "${CM_RUN_CMD0}" != "" ]]; then - eval ${CM_RUN_CMD0} - exitstatus=$? -else - echo "${CM_RUN_CMD}" - eval ${CM_RUN_CMD} - exitstatus=$? -fi +# Function to run command and check exit status +run_command() { + local cmd="$1" + + if [[ -n "$cmd" ]]; then + echo "$cmd" + eval "$cmd" + exitstatus=$? + + # If 'exitstatus' file exists, overwrite the exit status with its content + if [[ -e exitstatus ]]; then + exitstatus=$(cat exitstatus) + fi -eval ${CM_POST_RUN_CMD} -test $? -eq 0 || exit $? + # If exitstatus is non-zero, exit with that status + if [[ $exitstatus -ne 0 ]]; then + exit $exitstatus + fi + fi +} -test $exitstatus -eq 0 || $exitstatus +# Run CM_RUN_CMD0 if it exists, otherwise run CM_RUN_CMD +run_command "$CM_RUN_CMD0" || run_command "$CM_RUN_CMD" +# Run post-run command if it exists +if [[ -n "$CM_POST_RUN_CMD" ]]; then + eval "$CM_POST_RUN_CMD" + post_exitstatus=$? + # Exit if post-run command fails + if [[ $post_exitstatus -ne 0 ]]; then + exit $post_exitstatus + fi +fi +# Final check for exitstatus and exit with the appropriate code +if [[ $exitstatus -ne 0 ]]; then + exit $exitstatus +fi From df85d0ce8198391c45f62a99e24981939f2045f2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 19:31:13 +0530 Subject: [PATCH 310/658] Fix benchmark-program to exit on error --- script/benchmark-program/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/script/benchmark-program/run.sh b/script/benchmark-program/run.sh index 162c281d60..6eb39d3337 100755 --- a/script/benchmark-program/run.sh +++ b/script/benchmark-program/run.sh @@ -79,7 +79,12 @@ run_command() { } # Run CM_RUN_CMD0 if it exists, otherwise run CM_RUN_CMD -run_command "$CM_RUN_CMD0" || run_command "$CM_RUN_CMD" +if [[ -n "$CM_RUN_CMD0" ]]; then + run_command "$CM_RUN_CMD0" +fi + +run_command "$CM_RUN_CMD" + # Run post-run command if it exists if [[ -n "$CM_POST_RUN_CMD" ]]; then From 182f4ca1489a3e247c3272be00cbce063bdd601d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 19:57:34 +0100 Subject: [PATCH 311/658] Cleanups for loadgen build --- script/get-mlperf-inference-loadgen/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/get-mlperf-inference-loadgen/run.sh b/script/get-mlperf-inference-loadgen/run.sh index 3ee6196663..799a472209 100644 --- a/script/get-mlperf-inference-loadgen/run.sh +++ b/script/get-mlperf-inference-loadgen/run.sh @@ -24,15 +24,15 @@ fi cmake \ -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ "${CM_MLPERF_INFERENCE_SOURCE}/loadgen" \ - -DPYTHON_EXECUTABLE:FILEPATH=${CM_PYTHON_BIN_WITH_PATH} -if [ "${?}" != "0" ]; then exit 1; fi + -DPYTHON_EXECUTABLE:FILEPATH="${CM_PYTHON_BIN_WITH_PATH}" +if [ ${?} -ne 0 ]; then exit $?; fi echo "******************************************************" CM_MAKE_CORES=${CM_MAKE_CORES:-${CM_HOST_CPU_TOTAL_CORES}} CM_MAKE_CORES=${CM_MAKE_CORES:-2} -cmake --build . --target install -j ${CM_MAKE_CORES} -if [ "${?}" != "0" ]; then exit 1; fi +cmake --build . --target install -j "${CM_MAKE_CORES}" +if [ ${?} -ne 0 ]; then exit $?; fi # Clean build directory (too large) cd "${CUR_DIR}" @@ -44,7 +44,7 @@ fi cd "${CM_MLPERF_INFERENCE_SOURCE}/loadgen" ${CM_PYTHON_BIN_WITH_PATH} -m pip install . --target="${MLPERF_INFERENCE_PYTHON_SITE_BASE}" -if [ "${?}" != "0" ]; then exit 1; fi +if [ ${?} -ne 0 ]; then exit $?; fi # Clean the built wheel #find . -name 'mlcommons_loadgen*.whl' | xargs rm From 1e86429738f830bd2d391dbb93383c11466a4bda Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 20:38:54 +0100 Subject: [PATCH 312/658] Cleanups for loadgen build --- script/get-mlperf-inference-loadgen/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-loadgen/run.sh b/script/get-mlperf-inference-loadgen/run.sh index 799a472209..ac61ad329b 100644 --- a/script/get-mlperf-inference-loadgen/run.sh +++ b/script/get-mlperf-inference-loadgen/run.sh @@ -24,7 +24,7 @@ fi cmake \ -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ "${CM_MLPERF_INFERENCE_SOURCE}/loadgen" \ - -DPYTHON_EXECUTABLE:FILEPATH="${CM_PYTHON_BIN_WITH_PATH}" + -DPYTHON_EXECUTABLE:FILEPATH="${CM_PYTHON_BIN_WITH_PATH}" -B . if [ ${?} -ne 0 ]; then exit $?; fi echo "******************************************************" From 18588b6e65c7b54a7807000e38efee85e7446d8d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 21:21:26 +0100 Subject: [PATCH 313/658] Added vim-common deps for loadgen build --- script/get-generic-sys-util/_cm.json | 15 +++++++++++++++ script/get-mlperf-inference-loadgen/_cm.yaml | 1 + 2 files changed, 16 insertions(+) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 1bd8266511..a1f9d09918 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -668,6 +668,21 @@ } } }, + "vim-common": { + "env": { + "CM_SYS_UTIL_NAME": "vim_common", + "CM_SYS_UTIL_CHECK_CMD": "xxd --version" + }, + "state": { + "vim_common": { + "apt": "vim-common", + "brew": "vim", + "dnf": "vim-common", + "yum": "vim-common", + "choco": "vim" + } + } + }, "xz": { "env": { "CM_SYS_UTIL_NAME": "xz" diff --git a/script/get-mlperf-inference-loadgen/_cm.yaml b/script/get-mlperf-inference-loadgen/_cm.yaml index 3ce1f83992..49e3b186fe 100644 --- a/script/get-mlperf-inference-loadgen/_cm.yaml +++ b/script/get-mlperf-inference-loadgen/_cm.yaml @@ -76,6 +76,7 @@ deps: - pip-package - setuputils tags: get,generic-python-lib,_package.setuptools +- tags: get,generic-sys-util,_vim-common extra_cache_tags_from_env: - env: CM_PYTHON_CACHE_TAGS From 6c6af3546ec8c38cfa026cdf1e48733b2c61ec44 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 22:28:27 +0100 Subject: [PATCH 314/658] Fixed deps for loadgen build --- script/get-mlperf-inference-loadgen/_cm.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/get-mlperf-inference-loadgen/_cm.yaml b/script/get-mlperf-inference-loadgen/_cm.yaml index 49e3b186fe..3ce1f83992 100644 --- a/script/get-mlperf-inference-loadgen/_cm.yaml +++ b/script/get-mlperf-inference-loadgen/_cm.yaml @@ -76,7 +76,6 @@ deps: - pip-package - setuputils tags: get,generic-python-lib,_package.setuptools -- tags: get,generic-sys-util,_vim-common extra_cache_tags_from_env: - env: CM_PYTHON_CACHE_TAGS From a2821dd257695a9f8ba67c4ff96bcab89192701c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Oct 2024 23:30:20 +0100 Subject: [PATCH 315/658] Fix benchmark-program-mlperf command for power runs --- script/benchmark-program-mlperf/customize.py | 49 +++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/script/benchmark-program-mlperf/customize.py b/script/benchmark-program-mlperf/customize.py index 7d109051d9..4ca4f70df5 100644 --- a/script/benchmark-program-mlperf/customize.py +++ b/script/benchmark-program-mlperf/customize.py @@ -25,29 +25,34 @@ def postprocess(i): if os_info['platform'] != 'windows': # Construct the shell command with proper escaping - env['CM_MLPERF_RUN_CMD'] = ( - "CM_MLPERF_RUN_COUNT=$(cat ${CM_RUN_DIR}/count.txt); " - "echo ${CM_MLPERF_RUN_COUNT}; " - "CM_MLPERF_RUN_COUNT=$((CM_MLPERF_RUN_COUNT+1)); " - "echo ${CM_MLPERF_RUN_COUNT} > ${CM_RUN_DIR}/count.txt && " - "if [ ${CM_MLPERF_RUN_COUNT} -eq 1 ]; then " - "export CM_MLPERF_USER_CONF=${CM_MLPERF_RANGING_USER_CONF}; " - "else " - "export CM_MLPERF_USER_CONF=${CM_MLPERF_TESTING_USER_CONF}; " - "fi && " + env.get('CM_RUN_CMD', '').strip() - ) + env['CM_MLPERF_RUN_CMD'] = r""" +CM_MLPERF_RUN_COUNT=\$(cat \${CM_RUN_DIR}/count.txt); +echo \${CM_MLPERF_RUN_COUNT}; +CM_MLPERF_RUN_COUNT=\$((CM_MLPERF_RUN_COUNT+1)); +echo \${CM_MLPERF_RUN_COUNT} > \${CM_RUN_DIR}/count.txt && +if [ \${CM_MLPERF_RUN_COUNT} -eq 1 ]; then +export CM_MLPERF_USER_CONF="${CM_MLPERF_RANGING_USER_CONF}"; +else +export CM_MLPERF_USER_CONF="${CM_MLPERF_TESTING_USER_CONF}"; +fi && + """ + env.get('CM_RUN_CMD', '').strip() else: - env['CM_MLPERF_RUN_CMD'] = ( - "set /P CM_MLPERF_RUN_COUNT=<\"%CM_RUN_DIR%\\count.txt\" & " - "echo %CM_MLPERF_RUN_COUNT% & " - "set /A CM_MLPERF_RUN_COUNT=!CM_MLPERF_RUN_COUNT!+1 & " - "echo !CM_MLPERF_RUN_COUNT! > \"%CM_RUN_DIR%\\count.txt\" & " - "if !CM_MLPERF_RUN_COUNT! EQU 1 ( " - "set CM_MLPERF_USER_CONF=%CM_MLPERF_RANGING_USER_CONF% " - ") else ( " - "set CM_MLPERF_USER_CONF=%CM_MLPERF_TESTING_USER_CONF% " - ") & " + env.get('CM_RUN_CMD', '').strip() - ) + env['CM_MLPERF_RUN_CMD'] = r""" +:: Read the current count from the file +set /p CM_MLPERF_RUN_COUNT=<%CM_RUN_DIR%\count.txt +echo !CM_MLPERF_RUN_COUNT! + +:: Increment the count +set /a CM_MLPERF_RUN_COUNT=!CM_MLPERF_RUN_COUNT! + 1 +echo !CM_MLPERF_RUN_COUNT! > %CM_RUN_DIR%\count.txt + +:: Check the value and set the environment variable accordingly +if !CM_MLPERF_RUN_COUNT! EQU 1 ( + set CM_MLPERF_USER_CONF=%CM_MLPERF_RANGING_USER_CONF% +) else ( + set CM_MLPERF_USER_CONF=%CM_MLPERF_TESTING_USER_CONF% +) + """ + env.get('CM_RUN_CMD', '').strip() else: # Just use the existing CM_RUN_CMD if no ranging run is needed env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD', '').strip() From 4294218c68227f070e3dabb9f61e587e353ae707 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:12:07 +0530 Subject: [PATCH 316/658] Update customize.py --- .../app-mlperf-inference-mlcommons-python/customize.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index efff6513d9..efe3323360 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -264,7 +264,11 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio cmd = env['CM_PYTHON_BIN_WITH_PATH']+ " run.py --backend=" + env['CM_MLPERF_BACKEND'] + " --scenario="+env['CM_MLPERF_LOADGEN_SCENARIO'] + \ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + scenario_extra_options + mode_extra_options + dataset_options + quantization_options if env['CM_MLPERF_BACKEND'] == "deepsparse": - cmd += " --batch_size=" + env.get('CM_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') + " --model_path=" + env['MODEL_FILE'] + if "--batch-size" in cmd: + cmd.replace("--batch-size","--batch_size") + elif "--batch_size" not in cmd: + cmd += " --batch_size=" + env.get('CM_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') + cmd += " --model_path=" + env['MODEL_FILE'] if env.get('CM_MLPERF_CUSTOM_MODEL_PATH', '') != '': env['CM_ML_MODEL_FILE_WITH_PATH'] = env['MODEL_FILE'] @@ -300,11 +304,12 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio " --dataset-path " + env['CM_DATASET_PATH_ROOT'] + \ ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'].replace("bfloat", "bf").replace("float", "fp") + \ " --device " + device + \ - " --max-batchsize " + max_batchsize + \ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ scenario_extra_options + mode_extra_options + \ " --output " + env['CM_MLPERF_OUTPUT_DIR'] + \ " --model-path " + env['CM_ML_MODEL_PATH'] + if "--max-batchsize" not in cmd: + cmd += " --max-batchsize " + max_batchsize if env.get('CM_COCO2014_SAMPLE_ID_PATH','') != '': cmd += " --ids-path " + env['CM_COCO2014_SAMPLE_ID_PATH'] From c7cdc80a939c0c672f17a304a26c706f4a09cfd9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:38:03 +0530 Subject: [PATCH 317/658] reverted the batch size arg change done for bert --- script/app-mlperf-inference-mlcommons-python/customize.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index efe3323360..f227c07644 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -264,11 +264,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio cmd = env['CM_PYTHON_BIN_WITH_PATH']+ " run.py --backend=" + env['CM_MLPERF_BACKEND'] + " --scenario="+env['CM_MLPERF_LOADGEN_SCENARIO'] + \ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + scenario_extra_options + mode_extra_options + dataset_options + quantization_options if env['CM_MLPERF_BACKEND'] == "deepsparse": - if "--batch-size" in cmd: - cmd.replace("--batch-size","--batch_size") - elif "--batch_size" not in cmd: - cmd += " --batch_size=" + env.get('CM_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') - cmd += " --model_path=" + env['MODEL_FILE'] + cmd += " --batch_size=" + env.get('CM_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') + " --model_path=" + env['MODEL_FILE'] if env.get('CM_MLPERF_CUSTOM_MODEL_PATH', '') != '': env['CM_ML_MODEL_FILE_WITH_PATH'] = env['MODEL_FILE'] From 2f75c3aeacfa354baaf1834aa9225e5ca91cecb8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 15:46:46 +0100 Subject: [PATCH 318/658] Fix R50 SS run for Nvidia mlperf inference --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 +- .../app-mlperf-inference-nvidia/customize.py | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index ca06f7cfb3..9acfe0b256 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -1106,7 +1106,7 @@ variations: singlestream,resnet50: env: - SKIP_POLICIES: '1' + SKIP_POLICIES: '0' server,resnet50: env: diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index e2fd387c84..21f9ff626d 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -378,8 +378,8 @@ def preprocess(i): if audio_batch_size: run_config += f" --audio_batch_size={audio_batch_size}" - disable_encoder_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_ENCODER_PLUGIN')) - if disable_encoder_plugin and disable_encoder_plugin.lower() not in [ "no", "false" ]: + disable_encoder_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_ENCODER_PLUGIN', '')) + if disable_encoder_plugin and disable_encoder_plugin.lower() not in [ "no", "false", "" ]: run_config += " --disable_encoder_plugin" workspace_size = env.get('CM_MLPERF_NVIDIA_HARNESS_WORKSPACE_SIZE') @@ -393,8 +393,8 @@ def preprocess(i): if log_dir: run_config += f" --log_dir={log_dir}" - use_graphs = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS')) - if use_graphs and use_graphs.lower() not in [ "no", "false" ]: + use_graphs = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS', '')) + if use_graphs and use_graphs.lower() not in [ "no", "false", "" ]: run_config += " --use_graphs" use_deque_limit = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_DEQUE_LIMIT')) @@ -405,20 +405,20 @@ def preprocess(i): if deque_timeout_usec: run_config += f" --deque_timeout_usec={deque_timeout_usec}" - use_cuda_thread_per_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_CUDA_THREAD_PER_DEVICE')) - if use_cuda_thread_per_device and use_cuda_thread_per_device.lower() not in [ "no", "false" ]: + use_cuda_thread_per_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_CUDA_THREAD_PER_DEVICE', '')) + if use_cuda_thread_per_device and use_cuda_thread_per_device.lower() not in [ "no", "false", "" ]: run_config += " --use_cuda_thread_per_device" - run_infer_on_copy_streams = str(env.get('CM_MLPERF_NVIDIA_HARNESS_RUN_INFER_ON_COPY_STREAMS')) - if run_infer_on_copy_streams and run_infer_on_copy_streams.lower() not in [ "no", "false" ]: + run_infer_on_copy_streams = str(env.get('CM_MLPERF_NVIDIA_HARNESS_RUN_INFER_ON_COPY_STREAMS', '')) + if run_infer_on_copy_streams and run_infer_on_copy_streams.lower() not in [ "no", "false", "" ]: run_config += " --run_infer_on_copy_streams" - start_from_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_START_FROM_DEVICE')) - if start_from_device and start_from_device.lower() not in [ "no", "false" ]: + start_from_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_START_FROM_DEVICE', '')) + if start_from_device and start_from_device.lower() not in [ "no", "false", "" ]: run_config += " --start_from_device" - end_on_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_END_ON_DEVICE')) - if end_on_device and end_on_device.lower() not in [ "no", "false" ]: + end_on_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_END_ON_DEVICE', '')) + if end_on_device and end_on_device.lower() not in [ "no", "false", "" ]: run_config += " --end_on_device" max_dlas = env.get('CM_MLPERF_NVIDIA_HARNESS_MAX_DLAS') @@ -437,16 +437,16 @@ def preprocess(i): if soft_drop: run_config += f" --soft_drop={soft_drop}" - use_small_tile_gemm_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_SMALL_TILE_GEMM_PLUGIN')) - if use_small_tile_gemm_plugin and use_small_tile_gemm_plugin.lower() not in [ "no", "false" ]: + use_small_tile_gemm_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_SMALL_TILE_GEMM_PLUGIN', '')) + if use_small_tile_gemm_plugin and use_small_tile_gemm_plugin.lower() not in [ "no", "false", "" ]: run_config += f" --use_small_tile_gemm_plugin" audio_buffer_num_lines = env.get('CM_MLPERF_NVIDIA_HARNESS_AUDIO_BUFFER_NUM_LINES') if audio_buffer_num_lines: run_config += f" --audio_buffer_num_lines={audio_buffer_num_lines}" - use_fp8 = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_FP8')) - if use_fp8 and use_fp8.lower() not in [ "no", "false" ]: + use_fp8 = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_FP8', '')) + if use_fp8 and use_fp8.lower() not in [ "no", "false", "" ]: run_config += f" --use_fp8" if "llama2" in env["CM_MODEL"]: @@ -473,8 +473,8 @@ def preprocess(i): if num_warmups != '': run_config += f" --num_warmups={num_warmups}" - skip_postprocess = str(env.get('CM_MLPERF_NVIDIA_HARNESS_SKIP_POSTPROCESS')) - if skip_postprocess and skip_postprocess.lower() not in [ "no", "false" ]: + skip_postprocess = str(env.get('CM_MLPERF_NVIDIA_HARNESS_SKIP_POSTPROCESS', '')) + if skip_postprocess and skip_postprocess.lower() not in [ "no", "false", "" ]: run_config += f" --skip_postprocess" if test_mode: From fa48d2c0bc459587853bb4db9ec26954c273e46b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 15:57:13 +0100 Subject: [PATCH 319/658] Improve nvidia mlperf tests --- .github/workflows/test-mlperf-inference-sdxl.yaml | 1 - .github/workflows/test-nvidia-mlperf-implementation.yml | 1 - .github/workflows/test-scc24-sdxl.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-sdxl.yaml b/.github/workflows/test-mlperf-inference-sdxl.yaml index 580988e0ae..5527be3fac 100644 --- a/.github/workflows/test-mlperf-inference-sdxl.yaml +++ b/.github/workflows/test-mlperf-inference-sdxl.yaml @@ -19,7 +19,6 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - cm rm repo mlcommons@cm4mlops -f python3 -m pip install cm4mlops cm pull repo cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 3237c359f5..17baa42707 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -20,7 +20,6 @@ jobs: python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - cm rm repo mlcommons@cm4mlops -f pip install --upgrade cm4mlops cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index da9595d285..40f563f460 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -24,7 +24,6 @@ jobs: python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - cm rm repo mlcommons@cm4mlops -f pip install --upgrade cm4mlops pip install tabulate cm pull repo From d89e591d10355f62b5f94efdd1f8c23cb5b5abd0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Oct 2024 15:57:29 +0100 Subject: [PATCH 320/658] Improve nvidia mlperf tests --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 17baa42707..d140ca3a19 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "00 10 * * *" #to be adjusted + - cron: "04 15 * * *" #to be adjusted jobs: build_nvidia: From 60e4413ec9fba549d3a676d43595ec56e0bf1237 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 00:23:05 +0530 Subject: [PATCH 321/658] Disable skip_policies for Nvidia MLPerf inference R50 MS --- script/app-mlperf-inference-nvidia/_cm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 9acfe0b256..e85028585b 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -1106,7 +1106,7 @@ variations: singlestream,resnet50: env: - SKIP_POLICIES: '0' + SKIP_POLICIES: '0' # skip_policies used to give better latency but is not working with 4.0 and later Nvidia codes server,resnet50: env: @@ -1118,7 +1118,7 @@ variations: multistream,resnet50: env: - SKIP_POLICIES: '1' + SKIP_POLICIES: '0' singlestream,run_harness: default_variations: From 89dcc3af9382551c183da690f89874fa3d94bc3d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:29:34 +0530 Subject: [PATCH 322/658] instruction for standalone execution of script --- script/get-platform-details/README-EXTRA.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 script/get-platform-details/README-EXTRA.md diff --git a/script/get-platform-details/README-EXTRA.md b/script/get-platform-details/README-EXTRA.md new file mode 100644 index 0000000000..22b4875e86 --- /dev/null +++ b/script/get-platform-details/README-EXTRA.md @@ -0,0 +1,10 @@ +Please execute the following CM command to obtain the platform details of the System Under Test (SUT): + +``` +cm run script --tags=get,platform-details --platform_details_dir= +``` + + +The generated details will be saved as a text file in the specified directory. If no directory is specified, the generated text file will be saved in the CM cache + +A sample of the generated text file can be found [here](https://github.com/GATEOverflow/mlperf_inference_test_submissions_v5.0/blob/main/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.5.0-cu124/system_info.txt) From 35117a72491a2a2c5faefcb9418b94d0dba75e7e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:34:44 +0530 Subject: [PATCH 323/658] Added custom dir input var --- script/get-platform-details/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 3c6c5b4bd6..7168f1ce27 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -4,6 +4,9 @@ "automation_uid": "5b4e0237da074764", "cache": false, "category": "Platform information", + "input_mapping": { + "platform_details_dir": "CM_PLATFORM_DETAILS_DIR_PATH" + }, "deps": [ { "tags": "detect,os" From d9df89eb3c919ac49a3a4cdd07e3aef747f2a427 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:37:29 +0530 Subject: [PATCH 324/658] save system_info.txt to specific folder --- script/get-platform-details/customize.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index 47f1bab8b8..d23642fe86 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -22,8 +22,11 @@ def preprocess(i): #if not check_installation("cpupower",os_info): env['CM_INSTALL_CPUPOWER'] = 'True' - if env.get('CM_PLATFORM_DETAILS_FILE_PATH', '') == '': - env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") + if env.get('CM_PLATFORM_DETAILS_DIR_PATH', '') == '': + env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") + else: + env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(env['CM_PLATFORM_DETAILS_DIR_PATH'], "system_info.txt") + return {'return':0} From 2ba7bece1b40898808b85b86fd168c27ddb0a25c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:43:57 +0530 Subject: [PATCH 325/658] Handle automatic creation of directory if not present --- script/get-platform-details/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index d23642fe86..1a5cb6ff16 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -25,6 +25,8 @@ def preprocess(i): if env.get('CM_PLATFORM_DETAILS_DIR_PATH', '') == '': env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") else: + if not os.path.exists(env['CM_PLATFORM_DETAILS_DIR_PATH']): + os.makedirs(env['CM_PLATFORM_DETAILS_DIR_PATH']) env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(env['CM_PLATFORM_DETAILS_DIR_PATH'], "system_info.txt") From fb4daf0d442225e353cb5d7f4b5318240b00a0e5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 13:40:36 +0530 Subject: [PATCH 326/658] Fix use_graphs variation in Nvidia implementation --- script/app-mlperf-inference-nvidia/_cm.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index e85028585b..1033496a0b 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -762,6 +762,11 @@ variations: CM_MLPERF_NVIDIA_HARNESS_USE_TRITON: "yes" CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX3: "using_triton" + use_graphs: + group: graphs + env: + CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS: "yes" + prebuild: group: run-mode env: @@ -1435,12 +1440,12 @@ variations: rtx_4090,sdxl,offline,run_harness: default_variations: batch-size: batch_size.2 - use_graphs: "True" + graphs: use-graphs rtx_4090,sdxl,server,run_harness: default_variations: batch-size: batch_size.2 - use_graphs: "True" + graphs: use-graphs rtx_4090,resnet50,offline,run_harness: default_variations: @@ -1449,7 +1454,7 @@ variations: rtx_4090,resnet50,server,run_harness: default_variations: batch-size: batch_size.32 - use_graphs: "True" + graphs: use-graphs rtx_4090,retinanet,offline,run_harness: default_variations: From 806cab6fe70da7221f7230899d44ab2660eaf0d8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 13:49:00 +0530 Subject: [PATCH 327/658] Improve nvidia mlperf inference R50 SS/MS config --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 ++ .../app-mlperf-inference-nvidia/customize.py | 26 +++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 1033496a0b..63f80694b7 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -1111,6 +1111,7 @@ variations: singlestream,resnet50: env: + CM_MLPERF_NVIDIA_HARNESS_DISABLE_BETA1_SMALLK: yes SKIP_POLICIES: '0' # skip_policies used to give better latency but is not working with 4.0 and later Nvidia codes server,resnet50: @@ -1123,6 +1124,7 @@ variations: multistream,resnet50: env: + CM_MLPERF_NVIDIA_HARNESS_DISABLE_BETA1_SMALLK: yes SKIP_POLICIES: '0' singlestream,run_harness: diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index 21f9ff626d..c39227b894 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -379,9 +379,13 @@ def preprocess(i): run_config += f" --audio_batch_size={audio_batch_size}" disable_encoder_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_ENCODER_PLUGIN', '')) - if disable_encoder_plugin and disable_encoder_plugin.lower() not in [ "no", "false", "" ]: + if disable_encoder_plugin and disable_encoder_plugin.lower() not in [ "no", "false", "0", "" ]: run_config += " --disable_encoder_plugin" + disable_beta1_smallk = True = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_BETA1_SMALLK', '')) + if disable_beta1_smallk and disable_beta1_smallk.lower() in [ "yes", "true", "1" ]: + run_config += " --disable_beta1_smallk" + workspace_size = env.get('CM_MLPERF_NVIDIA_HARNESS_WORKSPACE_SIZE') if workspace_size: run_config += f" --workspace_size={workspace_size}" @@ -394,11 +398,11 @@ def preprocess(i): run_config += f" --log_dir={log_dir}" use_graphs = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS', '')) - if use_graphs and use_graphs.lower() not in [ "no", "false", "" ]: + if use_graphs and use_graphs.lower() not in [ "no", "false", "0", "" ]: run_config += " --use_graphs" use_deque_limit = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_DEQUE_LIMIT')) - if use_deque_limit and use_deque_limit.lower() not in [ "no", "false" ]: + if use_deque_limit and use_deque_limit.lower() not in [ "no", "false", "0" ]: run_config += " --use_deque_limit" deque_timeout_usec = env.get('CM_MLPERF_NVIDIA_HARNESS_DEQUE_TIMEOUT_USEC') @@ -406,19 +410,19 @@ def preprocess(i): run_config += f" --deque_timeout_usec={deque_timeout_usec}" use_cuda_thread_per_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_CUDA_THREAD_PER_DEVICE', '')) - if use_cuda_thread_per_device and use_cuda_thread_per_device.lower() not in [ "no", "false", "" ]: + if use_cuda_thread_per_device and use_cuda_thread_per_device.lower() not in [ "no", "false", "0", "" ]: run_config += " --use_cuda_thread_per_device" run_infer_on_copy_streams = str(env.get('CM_MLPERF_NVIDIA_HARNESS_RUN_INFER_ON_COPY_STREAMS', '')) - if run_infer_on_copy_streams and run_infer_on_copy_streams.lower() not in [ "no", "false", "" ]: + if run_infer_on_copy_streams and run_infer_on_copy_streams.lower() not in [ "no", "false", "0", "" ]: run_config += " --run_infer_on_copy_streams" start_from_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_START_FROM_DEVICE', '')) - if start_from_device and start_from_device.lower() not in [ "no", "false", "" ]: + if start_from_device and start_from_device.lower() not in [ "no", "false", "0", "" ]: run_config += " --start_from_device" end_on_device = str(env.get('CM_MLPERF_NVIDIA_HARNESS_END_ON_DEVICE', '')) - if end_on_device and end_on_device.lower() not in [ "no", "false", "" ]: + if end_on_device and end_on_device.lower() not in [ "no", "false", "0", "" ]: run_config += " --end_on_device" max_dlas = env.get('CM_MLPERF_NVIDIA_HARNESS_MAX_DLAS') @@ -438,7 +442,7 @@ def preprocess(i): run_config += f" --soft_drop={soft_drop}" use_small_tile_gemm_plugin = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_SMALL_TILE_GEMM_PLUGIN', '')) - if use_small_tile_gemm_plugin and use_small_tile_gemm_plugin.lower() not in [ "no", "false", "" ]: + if use_small_tile_gemm_plugin and use_small_tile_gemm_plugin.lower() not in [ "no", "false", "0", "" ]: run_config += f" --use_small_tile_gemm_plugin" audio_buffer_num_lines = env.get('CM_MLPERF_NVIDIA_HARNESS_AUDIO_BUFFER_NUM_LINES') @@ -446,7 +450,7 @@ def preprocess(i): run_config += f" --audio_buffer_num_lines={audio_buffer_num_lines}" use_fp8 = str(env.get('CM_MLPERF_NVIDIA_HARNESS_USE_FP8', '')) - if use_fp8 and use_fp8.lower() not in [ "no", "false", "" ]: + if use_fp8 and use_fp8.lower() not in [ "no", "false", "0", "" ]: run_config += f" --use_fp8" if "llama2" in env["CM_MODEL"]: @@ -454,7 +458,7 @@ def preprocess(i): run_config += f" --tensor_parallelism={tmp_tp_size}" enable_sort = env.get('CM_MLPERF_NVIDIA_HARNESS_ENABLE_SORT') - if enable_sort and enable_sort.lower() not in [ "no", "false" ]: + if enable_sort and enable_sort.lower() not in [ "no", "false", "0" ]: run_config += f" --enable_sort" sdxl_server_batcher_time_limit = env.get('CM_MLPERF_NVIDIA_HARNESS_ENABLE_SORT') @@ -474,7 +478,7 @@ def preprocess(i): run_config += f" --num_warmups={num_warmups}" skip_postprocess = str(env.get('CM_MLPERF_NVIDIA_HARNESS_SKIP_POSTPROCESS', '')) - if skip_postprocess and skip_postprocess.lower() not in [ "no", "false", "" ]: + if skip_postprocess and skip_postprocess.lower() not in [ "no", "false", "0", "" ]: run_config += f" --skip_postprocess" if test_mode: From b2ff3d8d732f19642bf0b85c79c352ce2394ced6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 09:19:48 +0100 Subject: [PATCH 328/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index d140ca3a19..281c635bce 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "04 15 * * *" #to be adjusted + - cron: "08 25 * * *" #to be adjusted jobs: build_nvidia: From 26bb27fe883bee3300c0a448f679aea5890fa4db Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 09:30:34 +0100 Subject: [PATCH 329/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 281c635bce..fe7819743b 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "08 25 * * *" #to be adjusted + - cron: "36 08 * * *" #to be adjusted jobs: build_nvidia: From 2c969e593149ba469bf341453a773ec10aafc7aa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 10:29:47 +0100 Subject: [PATCH 330/658] Fix typo --- script/app-mlperf-inference-nvidia/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index c39227b894..789ef6f9dc 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -382,7 +382,7 @@ def preprocess(i): if disable_encoder_plugin and disable_encoder_plugin.lower() not in [ "no", "false", "0", "" ]: run_config += " --disable_encoder_plugin" - disable_beta1_smallk = True = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_BETA1_SMALLK', '')) + disable_beta1_smallk = str(env.get('CM_MLPERF_NVIDIA_HARNESS_DISABLE_BETA1_SMALLK', '')) if disable_beta1_smallk and disable_beta1_smallk.lower() in [ "yes", "true", "1" ]: run_config += " --disable_beta1_smallk" From 506ac466ad63830d14284b9c9aa1435850f1a562 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:56:05 +0530 Subject: [PATCH 331/658] Update cm repo branch - docker --- script/app-image-classification-onnx-py/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-image-classification-onnx-py/_cm.yaml b/script/app-image-classification-onnx-py/_cm.yaml index 2e2241a07e..740a8a18ab 100644 --- a/script/app-image-classification-onnx-py/_cm.yaml +++ b/script/app-image-classification-onnx-py/_cm.yaml @@ -106,7 +106,7 @@ input_description: docker: skip_run_cmd: 'no' skip_cm_sys_upgrade: 'yes' - cm_repo_flags: '--checkout=dev' + cm_repo_flags: '--branch=dev' use_host_group_id: 'yes' image_tag_extra: '-cm-dev' input_paths: From 4306e658279de0804872964f9b2892378d6f2f66 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:02:35 +0530 Subject: [PATCH 332/658] input mapping name change --- script/get-platform-details/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 7168f1ce27..506d865f62 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -5,7 +5,7 @@ "cache": false, "category": "Platform information", "input_mapping": { - "platform_details_dir": "CM_PLATFORM_DETAILS_DIR_PATH" + "out_dir_path": "CM_PLATFORM_DETAILS_DIR_PATH" }, "deps": [ { From 71655a91431d7da88cbe44d8e1b85f7ee7ab1f31 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:13:53 +0530 Subject: [PATCH 333/658] added input mapping for file name --- script/get-platform-details/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 506d865f62..5a99bcddfd 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -5,7 +5,8 @@ "cache": false, "category": "Platform information", "input_mapping": { - "out_dir_path": "CM_PLATFORM_DETAILS_DIR_PATH" + "out_dir_path": "CM_PLATFORM_DETAILS_DIR_PATH", + "out_file_name": "CM_PLATFORM_DETAILS_FILE_NAME" }, "deps": [ { From 5f72cf5bac1c05cfaddb35b524f4a1bace266966 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 12:43:45 +0100 Subject: [PATCH 334/658] Fix typo --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 63f80694b7..1c12abbfe0 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -762,7 +762,7 @@ variations: CM_MLPERF_NVIDIA_HARNESS_USE_TRITON: "yes" CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX3: "using_triton" - use_graphs: + use-graphs: group: graphs env: CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS: "yes" From 71a57ed2c405d2b14eea1ec543ae9851a61ab3b3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:14:22 +0530 Subject: [PATCH 335/658] Updated file path --- script/get-platform-details/customize.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index 1a5cb6ff16..62fb2821f4 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -21,14 +21,13 @@ def preprocess(i): #if not check_installation("cpupower",os_info): env['CM_INSTALL_CPUPOWER'] = 'True' - - if env.get('CM_PLATFORM_DETAILS_DIR_PATH', '') == '': - env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(os.getcwd(), "system_info.txt") - else: - if not os.path.exists(env['CM_PLATFORM_DETAILS_DIR_PATH']): - os.makedirs(env['CM_PLATFORM_DETAILS_DIR_PATH']) - env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(env['CM_PLATFORM_DETAILS_DIR_PATH'], "system_info.txt") - + + if env.get('CM_PLATFORM_DETAILS_FILE_PATH', '') == '': + if env.get('CM_PLATFORM_DETAILS_DIR_PATH', '') == '': + env['CM_PLATFORM_DETAILS_DIR_PATH'] = os.getcwd() + if env.get('CM_PLATFORM_DETAILS_FILE_NAME', '') == '': + env['CM_PLATFORM_DETAILS_FILE_NAME'] = "system_info.txt" + env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(env['CM_PLATFORM_DETAILS_DIR_PATH'], env['CM_PLATFORM_DETAILS_FILE_NAME']) return {'return':0} From c81e33155b3db5dceafafed3beec310961406c17 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 25 Oct 2024 19:08:56 +0530 Subject: [PATCH 336/658] handle commands requiring sudo --- script/get-platform-details/_cm.json | 3 +++ script/get-platform-details/customize.py | 4 +-- script/get-platform-details/run.sh | 32 ++++++++++++++++++------ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 5a99bcddfd..384837a28e 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -12,6 +12,9 @@ { "tags": "detect,os" }, + { + "tags": "detect,sudo" + }, { "tags": "get,sys-util,generic,_psmisc" }, diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index 62fb2821f4..b3e72aaf55 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -13,8 +13,8 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - if os_info['platform'] == "windows": - return {'return':1, 'error':'get-platform-details script not yet supported in windows!'} + #if os_info['platform'] == "windows": + # return {'return':1, 'error':'get-platform-details script not yet supported in windows!'} if not check_installation("numactl",os_info): env['CM_INSTALL_NUMACTL'] = 'True' diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 481319f223..36bfc74db9 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -46,8 +46,12 @@ else echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "8. numactl --hardware" >> $OUTPUT_FILE - eval "numactl --hardware" >> $OUTPUT_FILE - #test $? -eq 0 || exit $? + if [[ ${CM_SUDO_USER} == "yes" ]]; then + eval "${CM_SUDO} numactl --hardware" >> $OUTPUT_FILE + test $? -eq 0 || exit $? + else + echo "Requires SUDO permission" >> $OUTPUT_FILE + fi echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "9. /proc/meminfo" >> $OUTPUT_FILE @@ -81,8 +85,12 @@ else echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "15. sysctl" >> $OUTPUT_FILE - eval "sudo sysctl -a" >> $OUTPUT_FILE - #test $? -eq 0 || exit $? + if [[ ${CM_SUDO_USER} == "yes" ]]; then + eval "${CM_SUDO} sysctl -a" >> $OUTPUT_FILE + test $? -eq 0 || exit $? + else + echo "Requires SUDO permission" >> $OUTPUT_FILE + fi echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE @@ -111,13 +119,21 @@ else echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "21. dmidecode" >> $OUTPUT_FILE - eval "sudo dmidecode" >> $OUTPUT_FILE - #test $? -eq 0 || exit $? + if [[ ${CM_SUDO_USER} == "yes" ]]; then + eval "${CM_SUDO} dmidecode" >> $OUTPUT_FILE + test $? -eq 0 || exit $? + else + echo "Requires SUDO permission" >> $OUTPUT_FILE + fi echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "22. BIOS" >> $OUTPUT_FILE - eval "sudo dmidecode -t bios" >> $OUTPUT_FILE - #test $? -eq 0 || exit $? + if [[ ${CM_SUDO_USER} == "yes" ]]; then + eval "${CM_SUDO} dmidecode -t bios" >> $OUTPUT_FILE + test $? -eq 0 || exit $? + else + echo "Requires SUDO permission" >> $OUTPUT_FILE + fi echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "System information has been saved to $PWD/$OUTPUT_FILE" From 2404047cbdfa33a02510cd885e00962980fee468 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 14:45:36 +0100 Subject: [PATCH 337/658] Support run configs from cm4mlops repo, adds more models to Nvidia mlperf GH action --- .../test-nvidia-mlperf-implementation.yml | 6 +-- .../customize.py | 4 +- .../tensorrt-framework/default-config.yaml | 38 +++++++++++++++++++ .../default-config.yaml | 38 +++++++++++++++++++ .../customize.py | 23 ++++++++--- 5 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/default-config.yaml create mode 100644 script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index fe7819743b..390151f3a5 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.12" ] - model: [ "resnet50" ] + model: [ "resnet50", "retinanet", "bert-99", "bert-99.9", "gptj-99.9", "3d-unet-99.9" ] steps: - name: Test MLPerf Inference NVIDIA ${{ matrix.model }} run: | @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --offline_target_qps=85000 --server_target_qps=73000 --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet - cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from ${{ matrix.model }} GH action on NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet --hw_name=RTX4090x2 + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 9d1160b44b..74e0ba2c45 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -359,8 +359,10 @@ def generate_submission(i): with open(measurements_json_path, "r") as f: measurements_json = json.load(f) model_precision = measurements_json.get("weight_data_types", "fp32") - if os.path.exists(user_conf_path): + if os.path.exists(measurements_json_path): + # This line can be removed once the PR in the inference repo is merged. shutil.copy(measurements_json_path, os.path.join(submission_measurement_path, sub_res+'.json')) + shutil.copy(measurements_json_path, os.path.join(submission_measurement_path, 'model-info.json')) files = [] readme = False diff --git a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/default-config.yaml b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/default-config.yaml new file mode 100644 index 0000000000..7b24138acb --- /dev/null +++ b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/default-config.yaml @@ -0,0 +1,38 @@ +--- + resnet50: + Offline: + target_qps: 43000.0 + Server: + target_qps: 37000.0 + retinanet: + Offline: + target_qps: 650.0 + Server: + target_qps: 600 + bert-99: + Offline: + target_qps: 4000 + bert-99.9: + Offline: + target_qps: 4000 + 3d-unet-99: + Offline: + target_qps: 2.0 + 3d-unet-99.9: + Offline: + target_qps: 2.0 + gptj-99.9: + Offline: + target_qps: 4 + Server: + target_qps: 3.5 + gptj-99: + Offline: + target_qps: 4 + Server: + target_qps: 3.5 + sdxl: + Offline: + target_qps: 2 + Server: + target_qps: 1 diff --git a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml new file mode 100644 index 0000000000..7b24138acb --- /dev/null +++ b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml @@ -0,0 +1,38 @@ +--- + resnet50: + Offline: + target_qps: 43000.0 + Server: + target_qps: 37000.0 + retinanet: + Offline: + target_qps: 650.0 + Server: + target_qps: 600 + bert-99: + Offline: + target_qps: 4000 + bert-99.9: + Offline: + target_qps: 4000 + 3d-unet-99: + Offline: + target_qps: 2.0 + 3d-unet-99.9: + Offline: + target_qps: 2.0 + gptj-99.9: + Offline: + target_qps: 4 + Server: + target_qps: 3.5 + gptj-99: + Offline: + target_qps: 4 + Server: + target_qps: 3.5 + sdxl: + Offline: + target_qps: 2 + Server: + target_qps: 1 diff --git a/script/get-mlperf-inference-sut-configs/customize.py b/script/get-mlperf-inference-sut-configs/customize.py index 75ef95b94a..27461d7b24 100644 --- a/script/get-mlperf-inference-sut-configs/customize.py +++ b/script/get-mlperf-inference-sut-configs/customize.py @@ -51,11 +51,24 @@ def postprocess(i): if os.path.exists(config_path_default): shutil.copy(config_path_default, config_path) else: - print(f"Config file missing for given hw_name: '{env['CM_HW_NAME']}', implementation: '{implementation_string}', device: '{device}, backend: '{backend}', copying from default") - src_config = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", "default", "config.yaml") - shutil.copy(src_config, config_path) - os.makedirs(os.path.dirname(config_path_default), exist_ok=True) - shutil.copy(src_config, config_path_default) + src_config_full = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-"+backend_version, run_config_string + "-config.yaml") + src_config_partial1 = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-"+backend_version, "default-config.yaml") + src_config_partial2 = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "framework-version-default", "default-config.yaml") + src_config_partial3 = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", env['CM_HW_NAME'], implementation_string+"-implementation", device+"-device", backend+"-framework", "default-config.yaml") + if os.path.exists(src_config_full): + shutil.copy(src_config_full, config_path) + elif os.path.exists(src_config_partial1): + shutil.copy(src_config_partial1, config_path) + elif os.path.exists(src_config_partial2): + shutil.copy(src_config_partial2, config_path) + elif os.path.exists(src_config_partial3): + shutil.copy(src_config_partial3, config_path) + else: + print(f"Config file missing for given hw_name: '{env['CM_HW_NAME']}', implementation: '{implementation_string}', device: '{device}, backend: '{backend}', copying from default") + src_config = os.path.join(env['CM_TMP_CURRENT_SCRIPT_PATH'], "configs", "default", "config.yaml") + shutil.copy(src_config, config_path) + os.makedirs(os.path.dirname(config_path_default), exist_ok=True) + shutil.copy(src_config, config_path_default) state['CM_SUT_CONFIG'][env['CM_SUT_NAME']] = yaml.load(open(config_path), Loader=yaml.SafeLoader) state['CM_SUT_CONFIG_NAME'] = env['CM_SUT_NAME'] From 0738add636c89904313ddce63fabd6df35363176 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 25 Oct 2024 19:17:08 +0530 Subject: [PATCH 338/658] changes for platform details --- script/app-mlperf-inference/_cm.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 45094cacbc..928709a8f4 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -27,6 +27,7 @@ default_env: CM_MLPERF_RUN_STYLE: test CM_TEST_QUERY_COUNT: '10' CM_MLPERF_QUANTIZATION: off + CM_GET_PLATFORM_DETAILS: yes env: CM_MLPERF_PRINT_SUMMARY: "no" @@ -114,8 +115,6 @@ posthook_deps: - tags: get,mlperf,sut,description #populate system meta information like framework - tags: get,platform,details enable_if_any_env: - CM_SUDO_USER: - - yes CM_GET_PLATFORM_DETAILS: - yes skip_if_env: From 32d9239e77835b21078cc2a4b0882f63b007397a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Oct 2024 19:54:12 +0530 Subject: [PATCH 339/658] add echo for debug - WIP --- script/get-platform-details/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 36bfc74db9..fb124524b6 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -86,6 +86,7 @@ else echo "15. sysctl" >> $OUTPUT_FILE if [[ ${CM_SUDO_USER} == "yes" ]]; then + echo "${CM_SUDO} sysctl -a" eval "${CM_SUDO} sysctl -a" >> $OUTPUT_FILE test $? -eq 0 || exit $? else From b09ff7a52faf7d24388c9a5f32c32f16e802d674 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 20:50:26 +0100 Subject: [PATCH 340/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 390151f3a5..564c999b41 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "36 08 * * *" #to be adjusted + - cron: "36 00 * * *" #to be adjusted jobs: build_nvidia: From 77f7583144781a46f01683826fcfb98993bf76fa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 25 Oct 2024 20:57:03 +0100 Subject: [PATCH 341/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 564c999b41..0ff8c20cdb 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "36 00 * * *" #to be adjusted + - cron: "03 20 * * *" #to be adjusted jobs: build_nvidia: From 92727c6254e7a06ef8d0abbac4806d9588826168 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 26 Oct 2024 08:59:59 +0100 Subject: [PATCH 342/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 0ff8c20cdb..95f79f4c09 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "03 20 * * *" #to be adjusted + - cron: "05 08 * * *" #to be adjusted jobs: build_nvidia: @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=gh_ubuntu_x86 --model=resnet50 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet --hw_name=RTX4090x2 + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 From 4cbbd65d53fe5d0e56aff5974697e00bfa99bb0c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 26 Oct 2024 09:22:11 +0100 Subject: [PATCH 343/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 95f79f4c09..fcdffc9d28 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "05 08 * * *" #to be adjusted + - cron: "28 08 * * *" #to be adjusted jobs: build_nvidia: From 4b914c3ca0ab6fa41f99c2663d298feee2f73ff0 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 17:40:55 +0530 Subject: [PATCH 344/658] typo + test commit to fix gh action error --- script/detect-sudo/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index dbc9b89705..c82dd7a979 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -99,10 +99,11 @@ def prompt_sudo(): try: r = subprocess.check_output(["sudo", "-p", msg, "echo", "Check sudo"], stderr=subprocess.STDOUT, timeout=20) + print(r) print(r.decode('utf-8')) # Decode bytes to string return 0 except subprocess.TimeoutExpired: - reset_terminal() # Reset terminal to sane state + reset_terminal() # Reset terminal to same state if not prompt_retry(): # If the user chooses not to retry or times out return -1 except subprocess.CalledProcessError as e: From 5fb3ea302ec59e7ae6ecdf1a79956ee6d9e695e1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 18:38:04 +0530 Subject: [PATCH 345/658] test commit for gh error - WIP --- script/get-platform-details/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index fb124524b6..94a607af94 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -47,6 +47,7 @@ else echo "8. numactl --hardware" >> $OUTPUT_FILE if [[ ${CM_SUDO_USER} == "yes" ]]; then + echo "${CM_SUDO} numactl --hardware" eval "${CM_SUDO} numactl --hardware" >> $OUTPUT_FILE test $? -eq 0 || exit $? else @@ -87,7 +88,7 @@ else echo "15. sysctl" >> $OUTPUT_FILE if [[ ${CM_SUDO_USER} == "yes" ]]; then echo "${CM_SUDO} sysctl -a" - eval "${CM_SUDO} sysctl -a" >> $OUTPUT_FILE + eval "${CM_SUDO} sudo sysctl -a" >> $OUTPUT_FILE test $? -eq 0 || exit $? else echo "Requires SUDO permission" >> $OUTPUT_FILE From d5ac90c875273df474e8cbbe03b9280ec8c3660a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 19:03:22 +0530 Subject: [PATCH 346/658] bug fix - WIP --- script/detect-sudo/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index c82dd7a979..23c2f9ad0c 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -20,6 +20,8 @@ def preprocess(i): env['CM_SUDO_USER'] = "yes" if os.geteuid() == 0: env['CM_SUDO'] = '' #root user does not need sudo + else: + env['CM_SUDO'] = 'sudo' else: if can_execute_sudo_without_password(): env['CM_SUDO_USER'] = "yes" From 24d0dda027876070a474b422e7d024260a1767c1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 19:04:04 +0530 Subject: [PATCH 347/658] test commit for gh error - WIP --- script/get-platform-details/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 94a607af94..fc8e4ad36c 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -88,7 +88,7 @@ else echo "15. sysctl" >> $OUTPUT_FILE if [[ ${CM_SUDO_USER} == "yes" ]]; then echo "${CM_SUDO} sysctl -a" - eval "${CM_SUDO} sudo sysctl -a" >> $OUTPUT_FILE + eval "${CM_SUDO} sysctl -a" >> $OUTPUT_FILE test $? -eq 0 || exit $? else echo "Requires SUDO permission" >> $OUTPUT_FILE From 7ed8fe8566b034be26633923b915487d7d4d0482 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 19:19:07 +0530 Subject: [PATCH 348/658] Update regex condition for new env keys --- script/detect-sudo/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/detect-sudo/_cm.yaml b/script/detect-sudo/_cm.yaml index 56c83b8304..64b60a5f6c 100644 --- a/script/detect-sudo/_cm.yaml +++ b/script/detect-sudo/_cm.yaml @@ -9,7 +9,7 @@ cache: false category: DevOps automation new_env_keys: - - CM_SUDO_* + - CM_SUDO* tags: - detect From 46762d948332a23c44853cf8e4b13ee8a09f90eb Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 19:49:00 +0530 Subject: [PATCH 349/658] Skip get-platform details for non linux systems --- script/get-platform-details/customize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index b3e72aaf55..d91997ce1f 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -13,7 +13,9 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - #if os_info['platform'] == "windows": + if os_info['platform'] != "linux": + print("WARNING: get-platform-details is not yet supported for non linux systems. Skipping for now...") + return {return: 0} # return {'return':1, 'error':'get-platform-details script not yet supported in windows!'} if not check_installation("numactl",os_info): From b15bc8a8cfcc7a56e4c186b6be3adf7226bfc74f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 19:49:35 +0530 Subject: [PATCH 350/658] fix typo --- script/get-platform-details/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index d91997ce1f..f0e613874f 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -15,7 +15,7 @@ def preprocess(i): if os_info['platform'] != "linux": print("WARNING: get-platform-details is not yet supported for non linux systems. Skipping for now...") - return {return: 0} + return {'return': 0} # return {'return':1, 'error':'get-platform-details script not yet supported in windows!'} if not check_installation("numactl",os_info): From 5d62c16e618b971a3bb906b1dd55d1952cc4caa8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:05:06 +0530 Subject: [PATCH 351/658] skip dependency installation on non supported system --- script/get-platform-details/_cm.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 384837a28e..6361933a1a 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -13,15 +13,35 @@ "tags": "detect,os" }, { + "skip_if_env": { + "CM_HOST_OS_TYPE": [ + "windows" + ] + }, "tags": "detect,sudo" }, { + "skip_if_env": { + "CM_HOST_OS_TYPE": [ + "windows" + ] + }, "tags": "get,sys-util,generic,_psmisc" }, { + "enable_if_env": { + "CM_HOST_OS_TYPE": [ + "linux" + ] + }, "tags": "get,sys-util,generic,_systemd" }, { + "enable_if_env": { + "CM_HOST_OS_TYPE": [ + "linux" + ] + }, "tags": "get,sys-util,generic,_dmidecode" } ], From c22bfc0a413f04eec707cbe860ba9cf818439c8a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 26 Oct 2024 16:16:13 +0100 Subject: [PATCH 352/658] Update default-config.yaml --- .../framework-version-default/default-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml index 7b24138acb..feed558e21 100644 --- a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml +++ b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml @@ -1,7 +1,7 @@ --- resnet50: Offline: - target_qps: 43000.0 + target_qps: 44000.0 Server: target_qps: 37000.0 retinanet: @@ -17,10 +17,10 @@ target_qps: 4000 3d-unet-99: Offline: - target_qps: 2.0 + target_qps: 8.0 3d-unet-99.9: Offline: - target_qps: 2.0 + target_qps: 8.0 gptj-99.9: Offline: target_qps: 4 From 23c7ea7bb0e3dc26671955910e81866a996f0f91 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 21:59:11 +0530 Subject: [PATCH 353/658] code clean --- script/detect-sudo/customize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index 23c2f9ad0c..bf9e4e349d 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -101,7 +101,6 @@ def prompt_sudo(): try: r = subprocess.check_output(["sudo", "-p", msg, "echo", "Check sudo"], stderr=subprocess.STDOUT, timeout=20) - print(r) print(r.decode('utf-8')) # Decode bytes to string return 0 except subprocess.TimeoutExpired: From 5bb654b96c9348bcb08a12a0ba0613736c4c4354 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 22:17:20 +0530 Subject: [PATCH 354/658] model mapping required only for open division --- .../customize.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 74e0ba2c45..21b0a059c2 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -163,26 +163,27 @@ def generate_submission(i): # Even the model mapping json file is present in root directory, the folders are traversed # and the data is updated provided not duplicated. models = [f for f in os.listdir(result_path) if not os.path.isfile(os.path.join(result_path, f))] - for model in models: - result_model_path = os.path.join(result_path, model) - scenarios = [f for f in os.listdir(result_model_path) if not os.path.isfile(os.path.join(result_model_path, f))] - for scenario in scenarios: - result_scenario_path = os.path.join(result_model_path, scenario) - modes = [f for f in os.listdir(result_scenario_path) if not os.path.isfile(os.path.join(result_scenario_path, f))] - for mode in modes: - result_mode_path = os.path.join(result_scenario_path,mode) - if mode == "performance": - compliance_performance_run_path = os.path.join(result_mode_path, "run_1") - # model mapping part - tmp_model_mapping_file_path = os.path.join(compliance_performance_run_path, "model_mapping.json") - if os.path.exists(tmp_model_mapping_file_path): - with open(tmp_model_mapping_file_path, 'r') as f: - new_model_mapping = json.load(f) - for new_custom_model in new_model_mapping: - if new_custom_model not in model_mapping_combined: - model_mapping_combined.update({new_custom_model:new_model_mapping[new_custom_model]}) - else: - return {"return":1, "error":f"model_mapping.json not found in {compliance_performance_run_path}"} + if division == "open": + for model in models: + result_model_path = os.path.join(result_path, model) + scenarios = [f for f in os.listdir(result_model_path) if not os.path.isfile(os.path.join(result_model_path, f))] + for scenario in scenarios: + result_scenario_path = os.path.join(result_model_path, scenario) + modes = [f for f in os.listdir(result_scenario_path) if not os.path.isfile(os.path.join(result_scenario_path, f))] + for mode in modes: + result_mode_path = os.path.join(result_scenario_path,mode) + if mode == "performance": + compliance_performance_run_path = os.path.join(result_mode_path, "run_1") + # model mapping part + tmp_model_mapping_file_path = os.path.join(compliance_performance_run_path, "model_mapping.json") + if os.path.exists(tmp_model_mapping_file_path): + with open(tmp_model_mapping_file_path, 'r') as f: + new_model_mapping = json.load(f) + for new_custom_model in new_model_mapping: + if new_custom_model not in model_mapping_combined: + model_mapping_combined.update({new_custom_model:new_model_mapping[new_custom_model]}) + else: + return {"return":1, "error":f"model_mapping.json not found in {compliance_performance_run_path}"} if check_dict_filled(sut_info.keys(), sut_info): system = sut_info["system_name"] From 2e8839f1bdfbd1393629716a876c6cac4c3f93df Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 26 Oct 2024 22:36:46 +0530 Subject: [PATCH 355/658] get hardware_name from cm-sut-info.json --- script/generate-mlperf-inference-submission/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 21b0a059c2..12f90a5900 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -131,7 +131,7 @@ def generate_submission(i): saved_system_meta_file_path = os.path.join(result_path, 'system_meta.json') # checks for json file containing system meta sut_info = { - "system_name": None, + "hardware_name": None, "implementation": None, "device": None, "framework": None, @@ -186,7 +186,7 @@ def generate_submission(i): return {"return":1, "error":f"model_mapping.json not found in {compliance_performance_run_path}"} if check_dict_filled(sut_info.keys(), sut_info): - system = sut_info["system_name"] + system = sut_info["hardware_name"] implementation = sut_info["implementation"] device = sut_info["device"] framework = sut_info["framework"].replace(" ","_") From ef1afb5ab07b0fa5685e61b2c57534331934f9d8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 27 Oct 2024 23:21:01 +0530 Subject: [PATCH 356/658] Donot reuse existing docker containers by default --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- script/run-docker-container/_cm.yaml | 1 + script/run-docker-container/customize.py | 2 +- script/run-mlperf-inference-app/customize.py | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index fcdffc9d28..0d40286fc3 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 diff --git a/script/run-docker-container/_cm.yaml b/script/run-docker-container/_cm.yaml index d070bcad36..38a8c794bd 100644 --- a/script/run-docker-container/_cm.yaml +++ b/script/run-docker-container/_cm.yaml @@ -15,6 +15,7 @@ category: Docker automation default_env: CM_DOCKER_DETACHED_MODE: 'yes' + CM_DOCKER_REUSE_EXISTING_CONTAINER: 'no' input_mapping: all_gpus: CM_DOCKER_ADD_ALL_GPUS diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index a3971e4482..20f8138bad 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -59,7 +59,7 @@ def preprocess(i): return {'return':1, 'error':'Docker is either not installed or not started:\n{}'.format(e)} output_split = docker_container.split("\n") - if len(output_split) > 1: #container exists + if len(output_split) > 1 and str(env.get('CM_DOCKER_REUSE_EXISTING_CONTAINER', '')).lower() in [ "1", "true", "yes" ]: #container exists out = output_split[1].split(" ") existing_container_id = out[0] env['CM_DOCKER_CONTAINER_ID'] = existing_container_id diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 1c80ee0b82..629bc3a254 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -201,6 +201,8 @@ def preprocess(i): if k.startswith("docker_"): docker_extra_input[k] = inp[k] inp = {} + if str(docker_dt).lower() in ["yes", "true", "1"]: + env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'yes' if env.get('CM_DOCKER_IMAGE_NAME', '') != '': docker_extra_input['docker_image_name'] = env['CM_DOCKER_IMAGE_NAME'] From 7873c7f8b6c4e8df94d0c2def8596c129e259fcc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 27 Oct 2024 23:27:53 +0530 Subject: [PATCH 357/658] Removed torch dep for nvidia-mlperf-inference-retinanet --- script/app-mlperf-inference-nvidia/_cm.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 1c12abbfe0..f2b9984275 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -430,8 +430,6 @@ variations: CM_ML_MODEL_WEIGHTS_DATA_TYPE: int8 deps: - tags: get,generic-python-lib,_Pillow - - tags: get,generic-python-lib,_torch - - tags: get,generic-python-lib,_torchvision - tags: get,generic-python-lib,_opencv-python - tags: get,generic-python-lib,_numpy - tags: get,generic-python-lib,_pycocotools @@ -527,6 +525,7 @@ variations: - tags: get,generic-python-lib,_transformers - tags: get,generic-python-lib,_safetensors - tags: get,generic-python-lib,_onnx + - tags: get,generic-python-lib,_package.sympy - tags: get,generic-python-lib,_onnx-graphsurgeon bert-99: @@ -613,7 +612,6 @@ variations: - CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH deps: - tags: get,dlrm,data,mlperf,inference,_nvidia - - tags: get,generic-python-lib,_torch - tags: get,generic-python-lib,_package.torchsnapshot - tags: get,generic-python-lib,_package.torchrec version: 0.3.2 From 9f87a287ac2a9f962c93c35ca1fe9222603a3e6f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 27 Oct 2024 17:58:57 +0000 Subject: [PATCH 358/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 0d40286fc3..e51634c468 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "28 08 * * *" #to be adjusted + - cron: "04 18 * * *" #to be adjusted jobs: build_nvidia: From a6050dd9b4c3e6da83997f0a54707c772304a470 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 27 Oct 2024 19:50:42 +0000 Subject: [PATCH 359/658] Update default-config.yaml --- .../default-config.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml index feed558e21..d2c66a716c 100644 --- a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml +++ b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml @@ -1,20 +1,20 @@ --- resnet50: Offline: - target_qps: 44000.0 + target_qps: 88000.0 Server: - target_qps: 37000.0 + target_qps: 73000.0 retinanet: Offline: - target_qps: 650.0 + target_qps: 1300.0 Server: - target_qps: 600 + target_qps: 1200 bert-99: Offline: - target_qps: 4000 + target_qps: 8000 bert-99.9: Offline: - target_qps: 4000 + target_qps: 8000 3d-unet-99: Offline: target_qps: 8.0 @@ -23,14 +23,14 @@ target_qps: 8.0 gptj-99.9: Offline: - target_qps: 4 + target_qps: 8 Server: - target_qps: 3.5 + target_qps: 7 gptj-99: Offline: - target_qps: 4 + target_qps: 8 Server: - target_qps: 3.5 + target_qps: 7 sdxl: Offline: target_qps: 2 From e7547483b43720c6c71ac8ecd93e9f0d9d4cb453 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:33:42 +0530 Subject: [PATCH 360/658] Updated logic for mounting non cache folder --- automation/script/module_misc.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 78d0115d19..998fe38ca3 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1086,7 +1086,9 @@ def doc(i): # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -def update_path_for_docker(path, mounts, force_path_target=''): +# This function takes in a pth and returns absolute path on host and container +# If mounts is passed, the function appends the host path and the container path to mounts in the form "host_path:container_path" +def update_path_for_docker(path, mounts=None, force_path_target=''): path_orig = '' path_target = '' @@ -1114,14 +1116,14 @@ def update_path_for_docker(path, mounts, force_path_target=''): x = path_orig + ':' + path_target # CHeck if no duplicates - to_add = True - for y in mounts: - if y.lower()==x.lower(): - to_add = False - break - - if to_add: - mounts.append(x) + if mounts != None: + to_add = True + for y in mounts: + if y.lower()==x.lower(): + to_add = False + break + if to_add: + mounts.append(x) return (path_orig, path_target) @@ -1617,8 +1619,11 @@ def get_container_path(value): new_path_split1 = new_path_split + path_split[repo_entry_index:repo_entry_index+3] new_path_split2 = new_path_split + path_split[repo_entry_index:] return "/".join(new_path_split1), "/".join(new_path_split2) + else: + orig_path,target_path = update_path_for_docker(path=value) + return target_path, target_path - return value, value + # return value, value ############################################################ From e929499d72a2b1114fdd9c39b850ac07496c75af Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 28 Oct 2024 17:30:23 +0530 Subject: [PATCH 361/658] Added get-gh-actions-runner script --- script/get-gh-actions-runner/_cm.yaml | 22 ++++++++++++++++++++++ script/get-gh-actions-runner/customize.py | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 script/get-gh-actions-runner/_cm.yaml create mode 100644 script/get-gh-actions-runner/customize.py diff --git a/script/get-gh-actions-runner/_cm.yaml b/script/get-gh-actions-runner/_cm.yaml new file mode 100644 index 0000000000..d3d1a5fba0 --- /dev/null +++ b/script/get-gh-actions-runner/_cm.yaml @@ -0,0 +1,22 @@ +alias: get-gh-actions-runner +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: false +tags: +- get +- gh +- actions-runner +- runner-code +- runner +- code +- gh-actions-runner +uid: 5b005c5a76f242a7 +new_env_keys: + - CM_GH_ACTIONS_RUNNER_CODE_PATH +deps: + - tags: detect-os + - tags: download-and-extract,_extract,_url.https://github.com/actions/runner/releases/download/v2.320.0/actions-runner-linux-x64-2.320.0.tar.gz + force_cache: yes + extra_cache_tags: gh-actions-runner-code + env: + CM_DAE_FINAL_ENV_NAME: CM_GH_ACTIONS_RUNNER_CODE_PATH diff --git a/script/get-gh-actions-runner/customize.py b/script/get-gh-actions-runner/customize.py new file mode 100644 index 0000000000..d12f9b3e1d --- /dev/null +++ b/script/get-gh-actions-runner/customize.py @@ -0,0 +1,22 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + return {'return':0} From 93a44417ee6b66ef4c8bb2c94e42fb8aa2f1f44a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 28 Oct 2024 12:13:40 +0000 Subject: [PATCH 362/658] Update module_misc.py --- automation/script/module_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 998fe38ca3..92c8a5f717 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1086,7 +1086,7 @@ def doc(i): # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# This function takes in a pth and returns absolute path on host and container +# This function takes in a host path and returns the absolute path on host and the container # If mounts is passed, the function appends the host path and the container path to mounts in the form "host_path:container_path" def update_path_for_docker(path, mounts=None, force_path_target=''): From cc3d2516a5cdf927274e62f7e6e229cba2fab769 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Mon, 28 Oct 2024 15:42:56 +0800 Subject: [PATCH 363/658] Updated fix for tests --- script/get-generic-sys-util/README.md | 424 ++++++++++++++++++++++++++ script/get-generic-sys-util/_cm.json | 357 +++++++++++++++------- 2 files changed, 677 insertions(+), 104 deletions(-) diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README.md index b1fc1a160e..bf09f83053 100644 --- a/script/get-generic-sys-util/README.md +++ b/script/get-generic-sys-util/README.md @@ -1 +1,425 @@ Please see [https://docs.mlcommons.org/cm4mlops/scripts/Detection-or-installation-of-tools-and-artifacts/get-generic-sys-util](https://docs.mlcommons.org/cm4mlops/scripts/Detection-or-installation-of-tools-and-artifacts/get-generic-sys-util) for the documentation of this CM script. + +# get-generic-sys-util +Below are the specific regexes and the format of output that they are expecting for each command used to check for versions. + +All commands are tested to be working on Ubuntu. + +Format: + +## Utility name +`regex` + +`command to obtain version` + +command output + +---- + +## g++-12 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`g++-9 --version` + +g++-9 (Ubuntu 9.5.0-1ubuntu1~22.04) 9.5.0
+Copyright (C) 2019 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ +## g++-11 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`g++-11 --version` + +g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
+Copyright (C) 2021 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ +## g++-12 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`g++-12 --version` + +g++-12 (Ubuntu 12.3.0-1ubuntu1~22.04) 12.3.0
+Copyright (C) 2022 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ + +## gcc-9 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`gcc-9 --version` + +gcc-9 (Ubuntu 9.5.0-1ubuntu1~22.04) 9.5.0
+Copyright (C) 2019 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ +## gcc-11 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`gcc-11 --version` + +gcc-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
+Copyright (C) 2021 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ + +## libgflags-dev +`([\d.]+)` + +`pkg-config --modversion gflags` + +2.2.2 + +## libglog-dev +`([\d.]+)` + +`pkg-config --modversion libglog` + +0.4.0 + +## libboost-all-dev +`([0-9]+(\w.[0-9]+)+)` + +`dpkg -s libboost-dev | grep 'Version'` + +Version: 1.74.0.3ubuntu7 + + +## libpng-dev +`([\d.]+)` + +`pkg-config --modversion libpng` + +1.6.37 + +## libre2-dev +`([\d.]+)` + +`pkg-config --modversion libre2` + +0.0.0 + +## libpci-dev +`([\d.]+)` + +`pkg-config --modversion libpci` + +3.7.0 + + +## libreadline_dev +`([\d.]+)` + +`pkg-config --modversion readline` + +8.1 + +## zlib +`([\d.]+)` + +`pkg-config --modversion zlib` + +1.2.11 + + +## libsqlite3_dev +`([\d.]+)` + +`pkg-config --modversion sqlite3` + +3.37.2 + +## libssl_dev +`OpenSSL\s+([\d.]+)` + +`openssl version` + +OpenSSL 3.0.2 15 Mar 2022 (Library: OpenSSL 3.0.2 15 Mar 2022) + +## libudev-dev +`([\d.]+)` + +`pkg-config --modversion libudev` + +249 + + +## libbz2_dev +`Version ([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)` + +`bzcat --version` + +bzip2, a block-sorting file compressor. Version 1.0.8, 13-Jul-2019. + +## libev_dev +dpkg here should be fine as only apt install is supported +`Version ([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)` + +`dpkg -s libev-dev | grep 'Version'` + +Version: 1:4.33-1 + +## libffi-dev +`([\d.]+)` + +`pkg-config --modversion libffi` + +3.4.2 + +## libffi_dev +`([\d.]+)` + +`pkg-config --modversion libffi` + +3.4.2 + +## libffi7 +`\d\.\d-[0-9]+` + +`dpkg -l libffi7 2>/dev/null | grep '^ii' | awk '{print $3}' || rpm -q libffi7 2>/dev/null || pacman -Q libffi7 2>/dev/null` + +3.3-5ubuntu1 + +## libffi8 +`\d\.\d\.\d-\d` + +`pkg-config --modversion libffi8"` + +3.4.2-4 + +## libgdbm_dev +dpkg here should be fine as only apt install is supported +`dpkg -s libgdbm-dev | grep 'Version'` + +`([\d]+\.[\d\.-]+)` + +Version: 1.23-1 + + +## libgmock +`([\d.]+)` + +`pkg-config --modversion libgmock` + +1.11.0 + +## liblzma_dev +`[A-Za-z]+\s\d\.\d\.\d` + +`xz --version` + +xz (XZ Utils) 5.2.5 +liblzma 5.2.5 + + +## libmpfr_dev +`([\d.]+)` + +`pkg-config --modversion mpfr` + +`4.1.0` + +## libncurses_dev +`([0-9]+(\.[0-9]+)+)` + +`ncurses5-config --version` + +6.3.20211021 + + + +## ninja-build +`([\d.]+)` + +`ninja --version` + +1.11.1 + +## md5sha1sum +`md5sum \(GNU coreutils\) ([\d.]+)` + +`md5sum --version` or `sha1sum --version` + +md5sum (GNU coreutils) 9.5 + +sha1sum (GNU coreutils) 9.5 + + +## nlohmann-json3-dev +`([\d.]+)` + +`pkg-config --modversion nlohmann_json` + +`3.10.5` + +## ntpdate +`([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)` + +`dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'` + +1:4.2.8p15+dfsg-1ubuntu2 + +## nvidia-cuda-toolkit +`release ([\d.]+)` + +`nvcc --version` + +nvcc: NVIDIA (R) Cuda compiler driver
+Copyright (c) 2005-2021 NVIDIA Corporation
+Built on Thu_Nov_18_09:45:25_PST_2021
+Cuda compilation tools, release 11.5, V11.5.119
+Build cuda_11.5.r11.5/compiler.30672275_0
+ + +## psmisc +`\(PSmisc\) ([\d.]+)` + +`pstree --version` + +pstree (PSmisc) 23.4 + +## rapidjson-dev +`([\d.]+)` + +`pkg-config --modversion RapidJSON` + +1.1.0 + +## cmake +`cmake version ([\d.]+)` + +`cmake --version` + +cmake version 3.30.4 + +## libnuma-dev +`([\d.]+)` + +`pkg-config --modversion numa` + +2.0.14 + + +## numactl +`([\d.]+)` + +`pkg-config --modversion numa` + +2.0.14 + +## wget +`Wget\s*([\d.]+)` + +`wget --version` + +GNU Wget 1.21.2 built on linux-gnu. + +## screen +`Screen version ([\d.]+)` + +`screen --version` + +Screen version 4.00.020 (FAU) 23-Oct-06 + +## xz +`xz \(XZ Utils\) ([\d.]+)` + +`xz --version` + +xz (XZ Utils) 5.2.5 +liblzma 5.2.5 + +## VIM +`VIM - Vi IMproved ([\d.]+` + +`vim --version` + +VIM - Vi IMproved 9.0 (2022 Jun 28, compiled Aug 3 2024 14:50:46) + +## rsync +`rsync\s+version\s+([\d.]+)` + +`rsync --version` + +rsync version 3.2.7 protocol version 31 + +## sox +`sox:\s+SoX\s+v([\d.]+)` + +`sox --version` + +sox: SoX v14.4.2 + + +## systemd +`systemd ([\d]+)` + +`systemctl --version` + +systemd 249 (249.11-0ubuntu3.12) + +## tk-dev +Probably fine to use `dpkg` here as only installation supported is for ubuntu + +`([0-9]+(\.[0-9]+)+)` + +`dpkg -s tk-dev | grep Version` + +Version: 8.6.11+1build2 + + +## transmission +`transmission-daemon ([\d.]+)` + +`transmission-daemon --version` + +transmission-daemon 3.00 (bb6b5a062e) + + +## wkhtmltopdf +`wkhtmltopdf ([\d.]+)` + +`wkhtmltopdf --version` + +wkhtmltopdf 0.12.6 + +## systemd +`systemd ([\d]+)` + +`systemd --version` + +systemd 255 (255.4-1ubuntu8.4) + + +## dmidecode +`([\d.]+)` + +`dmidecode --version` + +3.3 + +## git-lfs +`git-lfs/([\d.]+)` + +`git-lfs --version` + +git-lfs/3.4.1 (GitHub; linux arm64; go 1.22.2) + +## zlib1g +`([\d.]+)` + +`pkg-config --modversion zlib` + +1.2.11 + +## zlib1g_dev +`([\d.]+)` + +`pkg-config --modversion zlib` + +1.2.11 \ No newline at end of file diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a1f9d09918..7826a681b9 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -64,7 +64,9 @@ }, "cmake": { "env": { - "CM_SYS_UTIL_NAME": "cmake" + "CM_SYS_UTIL_NAME": "cmake", + "CM_SYS_UTIL_VERSION_CMD": "cmake --version", + "CM_SYS_UTIL_VERSION_RE": "cmake version ([\\d.]+)" }, "state": { "cmake": { @@ -73,11 +75,14 @@ "dnf": "cmake", "yum": "cmake" } - } + }, + "new_env_keys": ["CM_CMAKE_VERSION"] }, "dmidecode": { "env": { - "CM_SYS_UTIL_NAME": "dmidecode" + "CM_SYS_UTIL_NAME": "dmidecode", + "CM_SYS_UTIL_VERSION_CMD": "dmidecode --version", + "CM_SYS_UTIL_VERSION_RE": "version\\s+([\\d.]+)" }, "state": { "dmidecode": { @@ -86,64 +91,82 @@ "dnf": "dmidecode", "yum": "dmidecode" } - } + }, + "new_env_keys": ["CM_DMIDECODE_VERSION"] }, "g++-11": { "env": { - "CM_SYS_UTIL_NAME": "g++11" + "CM_SYS_UTIL_NAME": "g++11", + "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "g++11": { "apt": "g++-11", "dnf": "gcc-toolset-11-gcc-c++" } - } + }, + "new_env_keys": ["CM_GPP11_VERSION"] }, "g++-12": { "env": { - "CM_SYS_UTIL_NAME": "g++12" + "CM_SYS_UTIL_NAME": "g++12", + "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "g++12": { "apt": "g++-12", "dnf": "gcc-toolset-12-gcc-c++" } - } + }, + "new_env_keys": ["CM_GPP12_VERSION"] }, "g++-9": { "env": { - "CM_SYS_UTIL_NAME": "g++9" + "CM_SYS_UTIL_NAME": "g++9", + "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "g++9": { "apt": "g++-9", "dnf": "gcc-toolset-9-gcc-c++" } - } + }, + "new_env_keys": ["CM_GPP9_VERSION"] }, "gcc-11": { "env": { - "CM_SYS_UTIL_NAME": "gcc11" + "CM_SYS_UTIL_NAME": "gcc11", + "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "gcc11": { "apt": "gcc-11" } - } + }, + "new_env_keys": ["CM_GCC11_VERSION"] }, "gcc-9": { "env": { - "CM_SYS_UTIL_NAME": "gcc9" + "CM_SYS_UTIL_NAME": "gcc9", + "CM_SYS_UTIL_VERSION_CMD": "gcc-9 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "gcc9": { "apt": "gcc-9" } - } + }, + "new_env_keys": ["CM_GCC9_VERSION"] }, "gflags-dev": { "env": { - "CM_SYS_UTIL_NAME": "gflags-dev" + "CM_SYS_UTIL_NAME": "gflags-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gflags", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "gflags-dev": { @@ -152,12 +175,14 @@ "dnf": "gflags-devel", "yum": "gflags-devel" } - } + }, + "new_env_keys": ["CM_GFLAGS_DEV_VERSION"] }, "git-lfs": { "env": { "CM_SYS_UTIL_NAME": "git-lfs", - "CM_SYS_UTIL_CHECK_CMD": "git-lfs --version" + "CM_SYS_UTIL_VERSION_CMD": "git-lfs --version", + "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)" }, "state": { "git-lfs": { @@ -166,11 +191,14 @@ "dnf": "git-lfs", "yum": "git-lfs" } - } + }, + "new_env_keys": ["CM_GIT_LFS_VERSION"] }, "glog-dev": { "env": { - "CM_SYS_UTIL_NAME": "glog-dev" + "CM_SYS_UTIL_NAME": "glog-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libglog", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "glog-dev": { @@ -179,11 +207,14 @@ "dnf": "glog-devel", "yum": "glog-devel" } - } + }, + "new_env_keys": ["CM_GLOG_DEV_VERSION"] }, "libboost-all-dev": { "env": { - "CM_SYS_UTIL_NAME": "libboost-all-dev" + "CM_SYS_UTIL_NAME": "libboost-all-dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libboost-dev | grep 'Version'", + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" }, "state": { "libboost-all-dev": { @@ -192,11 +223,14 @@ "dnf": "boost-devel", "yum": "boost-devel" } - } + }, + "new_env_keys": ["CM_LIBBOOST_ALL_DEV_VERSION"] }, "libbz2-dev": { "env": { - "CM_SYS_UTIL_NAME": "libbz2_dev" + "CM_SYS_UTIL_NAME": "libbz2_dev", + "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", + "CM_SYS_UTIL_VERSION_RE": "Version ([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" }, "state": { "libbz2_dev": { @@ -206,31 +240,40 @@ "yum": "libbzip2-devel", "zlib-devel": "libbz2-devel" } - } + }, + "new_env_keys": ["CM_LIBBZ2_DEV_VERSION"] }, "libev-dev": { "env": { - "CM_SYS_UTIL_NAME": "libev_dev" + "CM_SYS_UTIL_NAME": "libev_dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libev-dev | grep 'Version'", + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" }, "state": { "libev_dev": { "apt": "libev-dev" } - } + }, + "new_env_keys": ["CM_LIBEV_DEV_VERSION"] }, "libffi": { "env": { - "CM_SYS_UTIL_NAME": "libffi" + "CM_SYS_UTIL_NAME": "libffi", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libffi7": { "apt": "libffi" } - } + }, + "new_env_keys": ["CM_LIBFFI_VERSION"] }, "libffi-dev": { "env": { - "CM_SYS_UTIL_NAME": "libffi_dev" + "CM_SYS_UTIL_NAME": "libffi_dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libffi_dev": { @@ -239,41 +282,53 @@ "dnf": "libffi-devel", "yum": "libffi-devel" } - } + }, + "new_env_keys": ["CM_LIBFFI_DEV_VERSION"] }, "libffi7": { "env": { - "CM_SYS_UTIL_NAME": "libffi7" + "CM_SYS_UTIL_NAME": "libffi7", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l libffi7 2>/dev/null | grep '^ii' | awk '{print $3}' || rpm -q libffi7 2>/dev/null || pacman -Q libffi7 2>/dev/null", + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d-[0-9]+" }, "state": { "libffi7": { "apt": "libffi7" } - } + }, + "new_env_keys": ["CM_LIBFFI7_VERSION"] }, "libffi8": { "env": { - "CM_SYS_UTIL_NAME": "libffi8" + "CM_SYS_UTIL_NAME": "libffi8", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi8", + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d" }, "state": { "libffi8": { "apt": "libffi8" } - } + }, + "new_env_keys": ["CM_LIBFFI8_VERSION"] }, "libgdbm-dev": { "env": { - "CM_SYS_UTIL_NAME": "libgdbm_dev" + "CM_SYS_UTIL_NAME": "libgdbm_dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libgdbm-dev | grep 'Version'", + "CM_SYS_UTIL_VERSION_RE": "([\\d]+\\.[\\d\\.-]+)" }, "state": { "libgdbm_dev": { "apt": "libgdbm-dev" } - } + }, + "new_env_keys": ["CM_LIBGDBM_DEV_VERSION"] }, "libgmock-dev": { "env": { - "CM_SYS_UTIL_NAME": "libgmock-dev" + "CM_SYS_UTIL_NAME": "libgmock-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gmock", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libgmock-dev": { @@ -282,21 +337,27 @@ "dnf": "gmock-devel", "yum": "gmock-devel" } - } + }, + "new_env_keys": ["CM_LIBGMOCK_DEV_VERSION"] }, "liblzma-dev": { "env": { - "CM_SYS_UTIL_NAME": "liblzma_dev" + "CM_SYS_UTIL_NAME": "liblzma_dev", + "CM_SYS_UTIL_VERSION_CMD": "xz --version", + "CM_SYS_UTIL_VERSION_RE": "[A-Za-z]+\\s\\d\\.\\d\\.\\d" }, "state": { "liblzma_dev": { "apt": "liblzma-dev" } - } + }, + "new_env_keys": ["CM_LIBLZMA_DEV_VERSION"] }, "libmkl-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmkl-dev" + "CM_SYS_UTIL_NAME": "libmkl-dev", + "CM_SYS_UTIL_VERSION_CMD": "ldconfig -p | grep libmkl", + "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)" }, "state": { "libmkl-dev": { @@ -305,11 +366,14 @@ "dnf": "", "yum": "" } - } + }, + "new_env_keys": ["CM_LIBMKL_DEV_VERSION"] }, "libmpfr-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmpfr-dev" + "CM_SYS_UTIL_NAME": "libmpfr-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion mpfr", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libmpfr-dev": { @@ -319,11 +383,14 @@ "yum": "mpfr-devel.x86_64", "zypper": "mpfr-devel" } - } + }, + "new_env_keys": ["CM_LIBMPFR_DEV_VERSION"] }, "libncurses-dev": { "env": { - "CM_SYS_UTIL_NAME": "libncurses_dev" + "CM_SYS_UTIL_NAME": "libncurses_dev", + "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" }, "state": { "libncurses_dev": { @@ -331,11 +398,14 @@ "dnf": "libncurses-devel", "yum": "libncurses-devel" } - } + }, + "new_env_keys": ["CM_LIBNCURSES_DEV_VERSION"] }, "libnuma-dev": { "env": { - "CM_SYS_UTIL_NAME": "libnuma-dev" + "CM_SYS_UTIL_NAME": "libnuma-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libnuma-dev": { @@ -344,11 +414,14 @@ "dnf": "numactl-libs", "yum": "numactl-libs" } - } + }, + "new_env_keys": ["CM_LIBNUMA_DEV_VERSION"] }, "libpci-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpci-dev" + "CM_SYS_UTIL_NAME": "libpci-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpci", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libpci-dev": { @@ -357,11 +430,14 @@ "dnf": "pciutils-devel", "yum": "pciutils-devel" } - } + }, + "new_env_keys": ["CM_LIBPCI_DEV_VERSION"] }, "libpng-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpng-dev" + "CM_SYS_UTIL_NAME": "libpng-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpng", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libpng-dev": { @@ -370,11 +446,14 @@ "dnf": "libpng-devel", "yum": "libpng-devel" } - } + }, + "new_env_keys": ["CM_LIBPNG_DEV_VERSION"] }, "libre2-dev": { "env": { - "CM_SYS_UTIL_NAME": "libre2-dev" + "CM_SYS_UTIL_NAME": "libre2-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion re2", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libre2-dev": { @@ -383,33 +462,42 @@ "dnf": "libre-devel", "yum": "libre-devel" } - } + }, + "new_env_keys": ["CM_LIBRE2_DEV_VERSION"] }, "libreadline-dev": { "env": { - "CM_SYS_UTIL_NAME": "libreadline_dev" + "CM_SYS_UTIL_NAME": "libreadline_dev", + "CM_SYS_UTIL_VERSION_CMD": "readline --version", + "CM_SYS_UTIL_VERSION_RE": "readline, version ([\\d.]+)" }, "state": { "libreadline_dev": { "apt": "libreadline-dev", "dnf": "libreadline-devel", - "yum": "lireadline-devel" + "yum": "readline-devel" } - } + }, + "new_env_keys": ["CM_LIBREADLINE_DEV_VERSION"] }, "libsqlite3-dev": { "env": { - "CM_SYS_UTIL_NAME": "libsqlite3_dev" + "CM_SYS_UTIL_NAME": "libsqlite3_dev", + "CM_SYS_UTIL_VERSION_CMD": "sqlite3 --version", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libsqlite3_dev": { "apt": "libsqlite3-dev" } - } + }, + "new_env_keys": ["CM_LIBSQLITE3_DEV_VERSION"] }, "libssl-dev": { "env": { - "CM_SYS_UTIL_NAME": "libssl_dev" + "CM_SYS_UTIL_NAME": "libssl_dev", + "CM_SYS_UTIL_VERSION_CMD": "openssl version", + "CM_SYS_UTIL_VERSION_RE": "OpenSSL\\s+([\\d.]+)" }, "state": { "libssl_dev": { @@ -418,11 +506,14 @@ "dnf": "libssl-devel", "yum": "libssl-devel" } - } + }, + "new_env_keys": ["CM_LIBSSL_DEV_VERSION"] }, "libudev-dev": { "env": { - "CM_SYS_UTIL_NAME": "libudev-dev" + "CM_SYS_UTIL_NAME": "libudev-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libudev", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libudev-dev": { @@ -431,7 +522,8 @@ "dnf": "libudev-devl", "yum": "libudev-devel" } - } + }, + "new_env_keys": ["CM_LIBUDEV_DEV_VERSION"] }, "linux-tools": { "deps": [ @@ -440,28 +532,36 @@ } ], "env": { - "CM_SYS_UTIL_NAME": "linux-tools" + "CM_SYS_UTIL_NAME": "linux-tools", + "CM_SYS_UTIL_VERSION_CMD": "linux-tools --version", + "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)" }, "state": { "linux-tools": { "apt": "linux-tools-<<>>" } - } + }, + "new_env_keys": ["CM_LINUX_TOOLS_VERSION"] }, "md5sha1sum": { "env": { - "CM_SYS_UTIL_NAME": "md5sha1sum" + "CM_SYS_UTIL_NAME": "md5sha1sum", + "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", + "CM_SYS_UTIL_VERSION_RE": "md5sum \\(GNU coreutils\\) ([\\d.]+)" }, "state": { "md5sha1sum": { "apt": "", "brew": "md5sha1sum" } - } + }, + "new_env_keys": ["CM_MD5SHA1SUM_VERSION"] }, "ninja-build": { "env": { - "CM_SYS_UTIL_NAME": "ninja-build" + "CM_SYS_UTIL_NAME": "ninja-build", + "CM_SYS_UTIL_VERSION_CMD": "ninja --version", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "ninja-build": { @@ -471,22 +571,28 @@ "yum": "ninja-build", "zypper": "ninja-build" } - } + }, + "new_env_keys": ["CM_NINJA_BUILD_VERSION"] }, "nlohmann-json3-dev": { "env": { - "CM_SYS_UTIL_NAME": "nlohmann_json3_dev" + "CM_SYS_UTIL_NAME": "nlohmann_json3_dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion nlohmann_json", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "nlohmann_json3_dev": { "apt": "nlohmann-json3-dev", - "dnf": "nlohmann-json-dev" + "dnf": "nlohmann-json-devel" } - } + }, + "new_env_keys": ["CM_NLOHMANN_JSON3_DEV_VERSION"] }, "ntpdate": { "env": { - "CM_SYS_UTIL_NAME": "ntpdate" + "CM_SYS_UTIL_NAME": "ntpdate", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'", + "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" }, "state": { "ntpdate": { @@ -495,7 +601,8 @@ "dnf": "ntpdate", "yum": "ntpdate" } - } + }, + "new_env_keys": ["CM_NTPDATE_VERSION"] }, "numactl": { "deps": [ @@ -514,7 +621,9 @@ } ], "env": { - "CM_SYS_UTIL_NAME": "numactl" + "CM_SYS_UTIL_NAME": "numactl", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "numactl": { @@ -522,11 +631,14 @@ "dnf": "numactl-devel", "yum": "numactl-devel" } - } + }, + "new_env_keys": ["CM_NUMACTL_VERSION"] }, "nvidia-cuda-toolkit": { "env": { - "CM_SYS_UTIL_NAME": "nvidia-cuda-toolkit" + "CM_SYS_UTIL_NAME": "nvidia-cuda-toolkit", + "CM_SYS_UTIL_VERSION_CMD": "nvcc --version", + "CM_SYS_UTIL_VERSION_RE": "release ([\\d.]+)" }, "state": { "nvidia-cuda-toolkit": { @@ -535,11 +647,14 @@ "dnf": "nvidia-cuda-toolkit", "yum": "nvidia-cuda-toolkit" } - } + }, + "new_env_keys": ["CM_NVIDIA_CUDA_TOOLKIT_VERSION"] }, "psmisc": { "env": { - "CM_SYS_UTIL_NAME": "psmisc" + "CM_SYS_UTIL_NAME": "psmisc", + "CM_SYS_UTIL_VERSION_CMD": "pstree --version", + "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) ([\\d.]+)" }, "state": { "psmisc": { @@ -548,11 +663,14 @@ "dnf": "psmisc", "yum": "psmisc" } - } + }, + "new_env_keys": ["CM_PSMISC_VERSION"] }, "rapidjson-dev": { "env": { - "CM_SYS_UTIL_NAME": "rapidjson-dev" + "CM_SYS_UTIL_NAME": "rapidjson-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion RapidJSON", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "rapidjson-dev": { @@ -561,12 +679,14 @@ "dnf": "rapidjson-devel", "yum": "rapidjson-devel" } - } + }, + "new_env_keys": ["CM_RAPIDJSON_DEV_VERSION"] }, "rsync": { "env": { "CM_SYS_UTIL_NAME": "rsync", - "CM_SYS_UTIL_CHECK_CMD": "rsync --version" + "CM_SYS_UTIL_VERSION_CMD": "rsync --version", + "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)" }, "state": { "rsync": { @@ -576,11 +696,14 @@ "yum": "rsync", "zypper": "rsync" } - } + }, + "new_env_keys": ["CM_RSYNC_VERSION"] }, "screen": { "env": { - "CM_SYS_UTIL_NAME": "screen" + "CM_SYS_UTIL_NAME": "screen", + "CM_SYS_UTIL_VERSION_CMD": "screen --version", + "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)" }, "state": { "screen": { @@ -590,11 +713,14 @@ "yum": "screen", "zypper": "rsync" } - } + }, + "new_env_keys": ["CM_SCREEN_VERSION"] }, "sox": { "env": { - "CM_SYS_UTIL_NAME": "sox" + "CM_SYS_UTIL_NAME": "sox", + "CM_SYS_UTIL_VERSION_CMD": "sox --version", + "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)" }, "state": { "sox": { @@ -602,11 +728,14 @@ "brew": "sox", "dnf": "sox" } - } + }, + "new_env_keys": ["CM_SOX_VERSION"] }, "systemd": { "env": { - "CM_SYS_UTIL_NAME": "systemd" + "CM_SYS_UTIL_NAME": "systemd", + "CM_SYS_UTIL_VERSION_CMD": "systemctl --version", + "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)" }, "state": { "systemd": { @@ -615,21 +744,27 @@ "dnf": "systemd", "yum": "systemd" } - } + }, + "new_env_keys": ["CM_SYSTEMD_VERSION"] }, "tk-dev": { "env": { - "CM_SYS_UTIL_NAME": "tk_dev" + "CM_SYS_UTIL_NAME": "tk_dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s tk-dev | grep Version", + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" }, "state": { "tk_dev": { "apt": "tk-dev" } - } + }, + "new_env_keys": ["CM_TK_DEV_VERSION"] }, "transmission": { "env": { - "CM_SYS_UTIL_NAME": "transmission" + "CM_SYS_UTIL_NAME": "transmission", + "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", + "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)" }, "state": { "transmission": { @@ -638,7 +773,8 @@ "dnf": "transmission-daemon", "yum": "transmission-daemon" } - } + }, + "new_env_keys": ["CM_TRANSMISSION_VERSION"] }, "wget": { "env": { @@ -659,19 +795,22 @@ "wkhtmltopdf": { "env": { "CM_SYS_UTIL_NAME": "wkhtmltopdf", - "CM_SYS_UTIL_CHECK_CMD": "wkhtmltopdf --version" + "CM_SYS_UTIL_VERSION_CMD": "wkhtmltopdf --version", + "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)" }, "state": { "wkhtmltopdf": { "apt": "wkhtmltopdf", "brew": "wkhtmltopdf" } - } + }, + "new_env_keys": ["CM_WKHTMLTOPDF_VERSION"] }, "vim-common": { "env": { "CM_SYS_UTIL_NAME": "vim_common", - "CM_SYS_UTIL_CHECK_CMD": "xxd --version" + "CM_SYS_UTIL_VERSION_CMD": "vim --version", + "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)" }, "state": { "vim_common": { @@ -681,11 +820,14 @@ "yum": "vim-common", "choco": "vim" } - } + }, + "new_env_keys": ["CM_VIM_COMMON_VERSION"] }, "xz": { "env": { - "CM_SYS_UTIL_NAME": "xz" + "CM_SYS_UTIL_NAME": "xz", + "CM_SYS_UTIL_VERSION_CMD": "xz --version", + "CM_SYS_UTIL_VERSION_RE": "xz \\(XZ Utils\\) ([\\d.]+)" }, "state": { "xz": { @@ -695,22 +837,28 @@ "yum": "xz", "choco": "xz" } - } + }, + "new_env_keys": ["CM_XZ_VERSION"] }, "zlib": { "env": { - "CM_SYS_UTIL_NAME": "zlib" + "CM_SYS_UTIL_NAME": "zlib", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "zlib": { "apt": "zlib1g", "choco": "zlib" } - } + }, + "new_env_keys": ["CM_ZLIB_VERSION"] }, "zlib1g-dev": { "env": { - "CM_SYS_UTIL_NAME": "zlib1g_dev" + "CM_SYS_UTIL_NAME": "zlib1g_dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "zlib1g_dev": { @@ -719,7 +867,8 @@ "yum": "zlib-devel", "zypper": "zlib-devel" } - } + }, + "new_env_keys": ["CM_ZLIB1G_DEV_VERSION"] } } } From 262b5565583c6c7659fde1fe2eb2c238c3e77781 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 28 Oct 2024 17:55:27 +0530 Subject: [PATCH 364/658] Fixed torch deps for mlperf inference cuda --- script/app-mlperf-inference-nvidia/_cm.yaml | 4 +- script/get-generic-python-lib/_cm.json | 2 - script/process-mlperf-accuracy/_cm.json | 93 ++++++++++++--------- 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index f2b9984275..3c3ea70733 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -593,10 +593,10 @@ variations: CM_ML_MODEL_WEIGHTS_DATA_TYPE: fp16 deps: - tags: get,generic-python-lib,_toml - - tags: get,generic-python-lib,_torchvision + - tags: get,generic-python-lib,_torchvision_cuda names: - torchvision - - tags: get,generic-python-lib,_torch + - tags: get,generic-python-lib,_torch_cuda - tags: get,generic-python-lib,_nvidia-apex - tags: get,generic-python-lib,_unidecode - tags: get,generic-python-lib,_inflect diff --git a/script/get-generic-python-lib/_cm.json b/script/get-generic-python-lib/_cm.json index 38ea8f9e09..cdfe55bade 100644 --- a/script/get-generic-python-lib/_cm.json +++ b/script/get-generic-python-lib/_cm.json @@ -942,7 +942,6 @@ }, "torch_cuda": { "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS1": "torch" }, "deps": [ { @@ -954,7 +953,6 @@ ], "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", - "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL1": "https://download.pytorch.org/whl/<<>>", "CM_TORCH_VERSION_EXTRA": "CUDA" }, "new_env_keys": [ diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index 1addc3cbf0..efbf9b901e 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -45,7 +45,7 @@ }, { "tags": "get,generic-python-lib,_package.datasets", - "names": + "names": [ "pip-package", "datasets" @@ -87,55 +87,55 @@ }, { "tags": "download,file,_url.https://raw.githubusercontent.com/mlcommons/inference_results_v4.0/main/closed/Intel/code/gptj-99/ITREX/evaluation.py", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": [ - "intel" + "intel" ] - }, - "force_cache": true, - "extra_cache_tags": "intel,accuracy,file,gptj,mlperf,inference", - "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_ACCURACY_FILE_WITH_PATH" - } - }, + }, + "force_cache": true, + "extra_cache_tags": "intel,accuracy,file,gptj,mlperf,inference", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_ACCURACY_FILE_WITH_PATH" + } + }, { "tags": "download,file,_url.https://raw.githubusercontent.com/mlcommons/inference_results_v4.0/main/closed/Intel/code/gptj-99/ITREX/dataset.py", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": [ - "intel" + "intel" ] - }, - "force_cache": true, - "extra_cache_tags": "intel,dataset,file,gptj,mlperf,inference", - "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_FILE_WITH_PATH" - } - }, + }, + "force_cache": true, + "extra_cache_tags": "intel,dataset,file,gptj,mlperf,inference", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_FILE_WITH_PATH" + } + }, { "tags": "download,file,_url.https://raw.githubusercontent.com/mlcommons/inference_results_v4.0/main/closed/Intel/code/gptj-99/ITREX/item.py", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": [ - "intel" + "intel" ] - }, - "force_cache": true, - "extra_cache_tags": "intel,dataset,item,file,gptj,mlperf,inference", - "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_ITEM_FILE_WITH_PATH" - } - }, + }, + "force_cache": true, + "extra_cache_tags": "intel,dataset,item,file,gptj,mlperf,inference", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_ITEM_FILE_WITH_PATH" + } + }, { "tags": "get,ml-model,gptj,_fp32,_pytorch", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": [ - "intel" + "intel" ] - } - } + } + } ], "env": { "CM_DATASET": "cnndm" @@ -285,7 +285,20 @@ "tags": "get,dataset-aux,squad-vocab" }, { - "tags": "get,generic-python-lib,_torch" + "tags": "get,generic-python-lib,_torch", + "skip_if_env": { + "CM_MLPERF_DEVICE": [ + "cuda" + ] + } + }, + { + "tags": "get,generic-python-lib,_torch_cuda", + "enable_if_env": { + "CM_MLPERF_DEVICE": [ + "cuda" + ] + } }, { "tags": "get,generic-python-lib,_tokenization" @@ -340,7 +353,7 @@ }, "openorca-gsm8k-mbxp": { "deps": [ - { + { "tags": "get,generic-python-lib,_package.rouge_score", "names": [ @@ -352,7 +365,7 @@ "names": [ "openorca-gsm8k-mbxp-combined" ], - "skip_if_env": { + "skip_if_env": { "CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [ "yes" ] @@ -363,7 +376,7 @@ "names": [ "mixtral-8x7b-model" ], - "skip_if_env": { + "skip_if_env": { "CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST": [ "yes" ] From b4601b70d1d52ce3e35c9b97befb15258e462275 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 28 Oct 2024 16:32:39 +0000 Subject: [PATCH 365/658] Fix version read in mlperf-inference --- script/get-mlperf-inference-src/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-src/customize.py b/script/get-mlperf-inference-src/customize.py index 2397d25206..8f5a160923 100644 --- a/script/get-mlperf-inference-src/customize.py +++ b/script/get-mlperf-inference-src/customize.py @@ -103,7 +103,7 @@ def postprocess(i): if os.path.exists(os.path.join(inference_root, "loadgen", "VERSION.txt")): with open(os.path.join(inference_root, "loadgen", "VERSION.txt")) as f: - version_info = f.read() + version_info = f.read().strip() env['CM_MLPERF_INFERENCE_SOURCE_VERSION'] = version_info if env.get('CM_GET_MLPERF_IMPLEMENTATION_ONLY', '') == "yes": From 1b2f2b7bc70fa9921d42e715fcd756fda71000c2 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:34:45 +0530 Subject: [PATCH 366/658] added compressed_tools module --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 0576f6eca0..8a06f987cd 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -995,6 +995,9 @@ variations: - tags: get,generic-python-lib,_package.more-itertools names: - more-itertools + - tags: get,generic-python-lib,_package.compressed_tensors + names: + - compressed_tensors llama2-70b-99: group: models From 31723a83a049bee4bee1455e1fba02720c9276e6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 01:54:19 +0530 Subject: [PATCH 367/658] corrected typo --- script/detect-sudo/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index bf9e4e349d..31c72ba9d6 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -104,7 +104,7 @@ def prompt_sudo(): print(r.decode('utf-8')) # Decode bytes to string return 0 except subprocess.TimeoutExpired: - reset_terminal() # Reset terminal to same state + reset_terminal() # Reset terminal to sane state if not prompt_retry(): # If the user chooses not to retry or times out return -1 except subprocess.CalledProcessError as e: From 1e7d5878381a9935fb9c7321935bcb951401740c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:12:47 +0530 Subject: [PATCH 368/658] WIP - handle workload of different OS through different scripts --- script/get-platform-details/customize.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index f0e613874f..1de6969afe 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -13,10 +13,8 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - if os_info['platform'] != "linux": - print("WARNING: get-platform-details is not yet supported for non linux systems. Skipping for now...") - return {'return': 0} - # return {'return':1, 'error':'get-platform-details script not yet supported in windows!'} + if os_info['platform'] == "darwin": + i['run_script_input']['script_name'] = "run-macos" if not check_installation("numactl",os_info): env['CM_INSTALL_NUMACTL'] = 'True' From db93c53dd00b17f47cb1ab5f347f3a45571d42f3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:13:45 +0530 Subject: [PATCH 369/658] updated warning message --- script/get-platform-details/run.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-platform-details/run.bat b/script/get-platform-details/run.bat index 73412b64ab..fcde181c03 100644 --- a/script/get-platform-details/run.bat +++ b/script/get-platform-details/run.bat @@ -1 +1 @@ -echo "This CM script not supported for windows yet" +echo "WARNING: get-platform-details script is fully supported on linux systems only." From 984c30be1c2bf03e7758ebe4e5cade253d177613 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:14:30 +0530 Subject: [PATCH 370/658] Create run-macos.sh --- script/get-platform-details/run-macos.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 script/get-platform-details/run-macos.sh diff --git a/script/get-platform-details/run-macos.sh b/script/get-platform-details/run-macos.sh new file mode 100644 index 0000000000..fcde181c03 --- /dev/null +++ b/script/get-platform-details/run-macos.sh @@ -0,0 +1 @@ +echo "WARNING: get-platform-details script is fully supported on linux systems only." From 96c6b7515919c9c0684512367237ee0c95a0528e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:26:36 +0530 Subject: [PATCH 371/658] improved submission generation logic --- .../customize.py | 59 ++++++++++++------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 12f90a5900..2f0f4067ae 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -28,6 +28,20 @@ def check_dict_filled(keys, sut_info): return False return True +# The function checks whether the submitting model name belongs standard model names for MLPef Inference +def model_in_valid_models(model, mlperf_version): + import submission_checker as checker + config = checker.MODEL_CONFIG + + if model not in config[mlperf_version]['models']: + internal_model_name = config[mlperf_version]["model_mapping"].get(model, '') # resnet50 -> resnet + if internal_model_name == '': + return (False, None) # Indicate failure with no internal model name + else: + return (True, internal_model_name) # Indicate success with internal model name + else: + return (True, model) + def generate_submission(i): # Save current user directory @@ -165,26 +179,31 @@ def generate_submission(i): models = [f for f in os.listdir(result_path) if not os.path.isfile(os.path.join(result_path, f))] if division == "open": for model in models: - result_model_path = os.path.join(result_path, model) - scenarios = [f for f in os.listdir(result_model_path) if not os.path.isfile(os.path.join(result_model_path, f))] - for scenario in scenarios: - result_scenario_path = os.path.join(result_model_path, scenario) - modes = [f for f in os.listdir(result_scenario_path) if not os.path.isfile(os.path.join(result_scenario_path, f))] - for mode in modes: - result_mode_path = os.path.join(result_scenario_path,mode) - if mode == "performance": - compliance_performance_run_path = os.path.join(result_mode_path, "run_1") - # model mapping part - tmp_model_mapping_file_path = os.path.join(compliance_performance_run_path, "model_mapping.json") - if os.path.exists(tmp_model_mapping_file_path): - with open(tmp_model_mapping_file_path, 'r') as f: - new_model_mapping = json.load(f) - for new_custom_model in new_model_mapping: - if new_custom_model not in model_mapping_combined: - model_mapping_combined.update({new_custom_model:new_model_mapping[new_custom_model]}) - else: - return {"return":1, "error":f"model_mapping.json not found in {compliance_performance_run_path}"} - + is_valid, returned_model_name = model_in_valid_models(model, env.get('CM_MLPERF_LAST_RELEASE', 'v4.1')) + if not is_valid: + result_model_path = os.path.join(result_path, model) + scenarios = [f for f in os.listdir(result_model_path) if not os.path.isfile(os.path.join(result_model_path, f))] + for scenario in scenarios: + result_scenario_path = os.path.join(result_model_path, scenario) + modes = [f for f in os.listdir(result_scenario_path) if not os.path.isfile(os.path.join(result_scenario_path, f))] + for mode in modes: + result_mode_path = os.path.join(result_scenario_path,mode) + if mode == "performance": + compliance_performance_run_path = os.path.join(result_mode_path, "run_1") + # model mapping part + tmp_model_mapping_file_path = os.path.join(compliance_performance_run_path, "model_mapping.json") + if os.path.exists(tmp_model_mapping_file_path): + with open(tmp_model_mapping_file_path, 'r') as f: + new_model_mapping = json.load(f) + for new_custom_model in new_model_mapping: + if new_custom_model not in model_mapping_combined: + model_mapping_combined.update({new_custom_model:new_model_mapping[new_custom_model]}) + else: + return {"return":1, "error":f"model_mapping.json not found in {compliance_performance_run_path}"} + else: + if returned_model_name != model: + model_mapping_combined.update({model:returned_model_name}) + if check_dict_filled(sut_info.keys(), sut_info): system = sut_info["hardware_name"] implementation = sut_info["implementation"] From 2f0191efdaf8a33d723ffdcc0d0c74b071aaa2a3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:31:24 +0530 Subject: [PATCH 372/658] enable numactl and linux-tools for linux system only --- script/get-platform-details/_cm.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 6361933a1a..11c57baa18 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -50,6 +50,9 @@ "enable_if_env": { "CM_INSTALL_NUMACTL": [ "True" + ], + "CM_HOST_OS_TYPE": [ + "linux" ] }, "tags": "get,sys-util,generic,_numactl" @@ -58,6 +61,9 @@ "enable_if_env": { "CM_INSTALL_CPUPOWER": [ "True" + ], + "CM_HOST_OS_TYPE": [ + "linux" ] }, "tags": "get,sys-util,generic,_linux-tools" From c9dbd4034040db8ff7ec2e9a40e7d7e0d6e03218 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:35:05 +0530 Subject: [PATCH 373/658] Code clean --- script/get-platform-details/run.sh | 242 ++++++++++++++--------------- 1 file changed, 119 insertions(+), 123 deletions(-) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index fc8e4ad36c..d3fc7e0d37 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -5,138 +5,134 @@ OUTPUT_FILE="$CM_PLATFORM_DETAILS_FILE_PATH" #echo $OUTPUT_FILE echo "WARNING: sudo permission is needed for some of the below commands" -if [[ ${CM_HOST_OS_FLAVOR} == "macos" ]]; then - echo "WARNING: To be done for the mac os" -else - echo "Platform Details" > $OUTPUT_FILE - echo "" >> $OUTPUT_FILE - echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "1. uname -a" >> $OUTPUT_FILE - eval "uname -a" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "2. w" >> $OUTPUT_FILE - eval "w" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "3. Username" >> $OUTPUT_FILE - echo "From environment variable \$USER: $USER" >> $OUTPUT_FILE - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "4. ulimit -a" >> $OUTPUT_FILE - eval "ulimit -a" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "5. sysinfo process ancestry" >> $OUTPUT_FILE - eval "pstree" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "6. /proc/cpuinfo" >> $OUTPUT_FILE - eval "cat /proc/cpuinfo" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "7. lscpu" >> $OUTPUT_FILE - eval "lscpu" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "8. numactl --hardware" >> $OUTPUT_FILE - if [[ ${CM_SUDO_USER} == "yes" ]]; then - echo "${CM_SUDO} numactl --hardware" - eval "${CM_SUDO} numactl --hardware" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - else - echo "Requires SUDO permission" >> $OUTPUT_FILE - fi - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "9. /proc/meminfo" >> $OUTPUT_FILE - eval "cat /proc/meminfo" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "10. who -r" >> $OUTPUT_FILE - eval "who -r" >> $OUTPUT_FILE +echo "Platform Details" > $OUTPUT_FILE +echo "" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "1. uname -a" >> $OUTPUT_FILE +eval "uname -a" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "2. w" >> $OUTPUT_FILE +eval "w" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "3. Username" >> $OUTPUT_FILE +echo "From environment variable \$USER: $USER" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "4. ulimit -a" >> $OUTPUT_FILE +eval "ulimit -a" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "5. sysinfo process ancestry" >> $OUTPUT_FILE +eval "pstree" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "6. /proc/cpuinfo" >> $OUTPUT_FILE +eval "cat /proc/cpuinfo" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "7. lscpu" >> $OUTPUT_FILE +eval "lscpu" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "8. numactl --hardware" >> $OUTPUT_FILE +if [[ ${CM_SUDO_USER} == "yes" ]]; then + echo "${CM_SUDO} numactl --hardware" + eval "${CM_SUDO} numactl --hardware" >> $OUTPUT_FILE test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "11. Systemd service manager version" >> $OUTPUT_FILE - eval "systemctl --version | head -n 1" >> $OUTPUT_FILE +else + echo "Requires SUDO permission" >> $OUTPUT_FILE +fi +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "9. /proc/meminfo" >> $OUTPUT_FILE +eval "cat /proc/meminfo" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "10. who -r" >> $OUTPUT_FILE +eval "who -r" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "11. Systemd service manager version" >> $OUTPUT_FILE +eval "systemctl --version | head -n 1" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "12. Services, from systemctl list-unit-files" >> $OUTPUT_FILE +eval "systemctl list-unit-files" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> $OUTPUT_FILE +eval "cat /proc/cmdline" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "14. cpupower frequency-info" >> $OUTPUT_FILE +eval "cpupower frequency-info" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "15. sysctl" >> $OUTPUT_FILE +if [[ ${CM_SUDO_USER} == "yes" ]]; then + echo "${CM_SUDO} sysctl -a" + eval "${CM_SUDO} sysctl -a" >> $OUTPUT_FILE test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +else + echo "Requires SUDO permission" >> $OUTPUT_FILE +fi +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "12. Services, from systemctl list-unit-files" >> $OUTPUT_FILE - eval "systemctl list-unit-files" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE +eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> $OUTPUT_FILE - eval "cat /proc/cmdline" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> $OUTPUT_FILE +eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "14. cpupower frequency-info" >> $OUTPUT_FILE - eval "cpupower frequency-info" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "15. sysctl" >> $OUTPUT_FILE - if [[ ${CM_SUDO_USER} == "yes" ]]; then - echo "${CM_SUDO} sysctl -a" - eval "${CM_SUDO} sysctl -a" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - else - echo "Requires SUDO permission" >> $OUTPUT_FILE - fi - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE - eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "18. OS release" >> $OUTPUT_FILE +eval "cat /etc/os-release" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> $OUTPUT_FILE - eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "19. Disk information" >> $OUTPUT_FILE +eval "lsblk" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "18. OS release" >> $OUTPUT_FILE - eval "cat /etc/os-release" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "20. /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE +eval "ls /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE +test $? -eq 0 || exit $? +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "19. Disk information" >> $OUTPUT_FILE - eval "lsblk" >> $OUTPUT_FILE +echo "21. dmidecode" >> $OUTPUT_FILE +if [[ ${CM_SUDO_USER} == "yes" ]]; then + eval "${CM_SUDO} dmidecode" >> $OUTPUT_FILE test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE +else + echo "Requires SUDO permission" >> $OUTPUT_FILE +fi +echo "------------------------------------------------------------" >> $OUTPUT_FILE - echo "20. /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE - eval "ls /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE +echo "22. BIOS" >> $OUTPUT_FILE +if [[ ${CM_SUDO_USER} == "yes" ]]; then + eval "${CM_SUDO} dmidecode -t bios" >> $OUTPUT_FILE test $? -eq 0 || exit $? - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "21. dmidecode" >> $OUTPUT_FILE - if [[ ${CM_SUDO_USER} == "yes" ]]; then - eval "${CM_SUDO} dmidecode" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - else - echo "Requires SUDO permission" >> $OUTPUT_FILE - fi - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "22. BIOS" >> $OUTPUT_FILE - if [[ ${CM_SUDO_USER} == "yes" ]]; then - eval "${CM_SUDO} dmidecode -t bios" >> $OUTPUT_FILE - test $? -eq 0 || exit $? - else - echo "Requires SUDO permission" >> $OUTPUT_FILE - fi - echo "------------------------------------------------------------" >> $OUTPUT_FILE - - echo "System information has been saved to $PWD/$OUTPUT_FILE" +else + echo "Requires SUDO permission" >> $OUTPUT_FILE fi +echo "------------------------------------------------------------" >> $OUTPUT_FILE + +echo "System information has been saved to $PWD/$OUTPUT_FILE" From fc3ec064700ff5a414d06ba7ca3d14d34df8a663 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 02:45:18 +0530 Subject: [PATCH 374/658] Update mlperf_utils for latest MLPerf inference submission checker changes --- script/get-mlperf-inference-utils/mlperf_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index 1da27dfae4..c4381e552e 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -152,8 +152,15 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res, inferred = False result = {} + inference_src_version = os.environ.get('CM_MLPERF_INFERENCE_SOURCE_VERSION', '') + version_tuple = None + if inference_src_version: + version_tuple = tuple(map(int, inference_src_version.split('.'))) - performance_result = checker.get_performance_metric(config, mlperf_model, performance_path, scenario, None, None, has_power) + if version_tuple and version_tuple >= (4,1,22): + performance_result = checker.get_performance_metric(config, mlperf_model, performance_path, scenario) + else: + performance_result = checker.get_performance_metric(config, mlperf_model, performance_path, scenario, None, None) if "stream" in scenario.lower(): performance_result_ = performance_result / 1000000 #convert to milliseconds else: From 1c424306191840e958284595d44bad021a367a7d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 03:02:39 +0530 Subject: [PATCH 375/658] Condition when model mapping json is in root folder --- script/generate-mlperf-inference-submission/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 2f0f4067ae..d6b5df98ff 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -177,7 +177,7 @@ def generate_submission(i): # Even the model mapping json file is present in root directory, the folders are traversed # and the data is updated provided not duplicated. models = [f for f in os.listdir(result_path) if not os.path.isfile(os.path.join(result_path, f))] - if division == "open": + if division == "open" and len(model_mapping_combined) == 0: for model in models: is_valid, returned_model_name = model_in_valid_models(model, env.get('CM_MLPERF_LAST_RELEASE', 'v4.1')) if not is_valid: From 857f3c8ccda834f0eda0aaa2e5082c798498833b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 03:02:55 +0530 Subject: [PATCH 376/658] Update mlperf_utils for latest MLPerf inference submission checker changes --- script/app-mlperf-inference/customize.py | 2 +- script/get-mlperf-inference-utils/mlperf_utils.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index d0587b80db..64cb42a8ed 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -242,7 +242,7 @@ def postprocess(i): if os.path.exists(env['CM_MLPERF_USER_CONF']): shutil.copy(env['CM_MLPERF_USER_CONF'], 'user.conf') - result, valid, power_result = mlperf_utils.get_result_from_log(env['CM_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode) + result, valid, power_result = mlperf_utils.get_result_from_log(env['CM_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode, env.get('CM_MLPERF_INFERENCE_SOURCE_VERSION')) power = None power_efficiency = None if power_result: diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index c4381e552e..e09448caee 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -4,7 +4,7 @@ from log_parser import MLPerfLog -def get_result_from_log(version, model, scenario, result_path, mode): +def get_result_from_log(version, model, scenario, result_path, mode, inference_src_version = None): config = checker.Config( version, @@ -20,7 +20,14 @@ def get_result_from_log(version, model, scenario, result_path, mode): valid = {} if mode == "performance": has_power = os.path.exists(os.path.join(result_path, "..", "power")) - result_ = checker.get_performance_metric(config, mlperf_model, result_path, scenario, None, None, has_power) + version_tuple = None + if inference_src_version: + version_tuple = tuple(map(int, inference_src_version.split('.'))) + + if version_tuple and version_tuple >= (4,1,22): + result_ = checker.get_performance_metric(config, mlperf_model, result_path, scenario) + else: + result_ = checker.get_performance_metric(config, mlperf_model, result_path, scenario, None, None, has_power) mlperf_log = MLPerfLog(os.path.join(result_path, "mlperf_log_detail.txt")) if ( "result_validity" not in mlperf_log.get_keys() @@ -133,7 +140,7 @@ def get_accuracy_metric(config, model, path): return is_valid, acc_results, acc_targets, acc_limits -def get_result_string(version, model, scenario, result_path, has_power, sub_res, division="open", system_json=None, model_precision="fp32"): +def get_result_string(version, model, scenario, result_path, has_power, sub_res, division="open", system_json=None, model_precision="fp32", inference_src_version = None): config = checker.Config( version, @@ -152,7 +159,6 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res, inferred = False result = {} - inference_src_version = os.environ.get('CM_MLPERF_INFERENCE_SOURCE_VERSION', '') version_tuple = None if inference_src_version: version_tuple = tuple(map(int, inference_src_version.split('.'))) From 80aee354e4ebe4df2e7f136376e1160d1bf8e9dc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 03:14:07 +0530 Subject: [PATCH 377/658] Update mlperf_utils for latest MLPerf inference submission checker changes --- script/generate-mlperf-inference-submission/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 74e0ba2c45..bcaf11271e 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -417,7 +417,7 @@ def generate_submission(i): f.write("TBD") #create an empty README else: readme_suffix = "" - result_string, result = mlperf_utils.get_result_string(env['CM_MLPERF_LAST_RELEASE'], model, scenario, result_scenario_path, power_run, sub_res, division, system_file, model_precision) + result_string, result = mlperf_utils.get_result_string(env['CM_MLPERF_LAST_RELEASE'], model, scenario, result_scenario_path, power_run, sub_res, division, system_file, model_precision, env.get('CM_MLPERF_INFERENCE_SOURCE_VERSION')) for key in result: results[model][scenario][key] = result[key] From 1b3793c7c3293cc8de2bfbc3c1e82e6ce10d5550 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 28 Oct 2024 22:27:23 +0000 Subject: [PATCH 378/658] Update customize.py --- script/get-platform-details/customize.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index 1de6969afe..9c26dae026 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -13,9 +13,6 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - if os_info['platform'] == "darwin": - i['run_script_input']['script_name'] = "run-macos" - if not check_installation("numactl",os_info): env['CM_INSTALL_NUMACTL'] = 'True' From 690ef3953caa2827d765b90861a331edfb0f65aa Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:55:02 +0530 Subject: [PATCH 379/658] initial commit - cm4mlops getting started --- docs/getting-started.md | 98 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/getting-started.md diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000000..da6aee9822 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,98 @@ +# cm-docs + +## CM Framework + +**CM** (Collective Mind) is a Python package with a CLI and API designed to create and manage automations. Two key automations developed using CM are **Script** and **Cache**, which streamline ML workflows, including managing Docker runs. +The **CM Python** package was developed by **Grigori Fursin**. The **Script** and **Cache** automations are part of the [cm4mlops](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference) repository, created by **Grigori Fursin** and **Arjun Suresh** and sponsored by OctoML, cKnowledge, cTuning and MLCommons. +The **CM scripts**, also housed in the [cm4mlops](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference) repository, are created and maintained by **Arjun Suresh**, **Anandhu Sooraj**, and **Grigori Fursin** with the help of the MLCommons community. + +## Simple script automation execution in cm + +A simple script automation execution in CM is as follows: + +``` +cm run script --tags=detect,os -j +``` + +This would capture the OS details of the system which it is run. Example details could be found here: +``` + "CM_HOST_OS_TYPE": "linux", + "CM_HOST_OS_BITS": "64", + "CM_HOST_OS_FLAVOR": "ubuntu", + "CM_HOST_OS_FLAVOR_LIKE": "debian", + "CM_HOST_OS_VERSION": "24.04", + "CM_HOST_OS_KERNEL_VERSION": "6.8.0-45-generic", + "CM_HOST_OS_GLIBC_VERSION": "2.39", + "CM_HOST_OS_MACHINE": "x86_64", + "CM_HOST_OS_PACKAGE_MANAGER": "apt", + "CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD": "DEBIAN_FRONTEND=noninteractive apt-get install -y", + "CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD": "apt-get update -y", + "+CM_HOST_OS_DEFAULT_LIBRARY_PATH": [ + "/usr/local/lib/x86_64-linux-gnu", + "/lib/x86_64-linux-gnu", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/x86_64-linux-gnu64", + "/usr/local/lib64", + "/lib64", + "/usr/lib64", + "/usr/local/lib", + "/lib", + "/usr/lib", + "/usr/x86_64-linux-gnu/lib64", + "/usr/x86_64-linux-gnu/lib" + ], + "CM_HOST_PLATFORM_FLAVOR": "x86_64", + "CM_HOST_PYTHON_BITS": "64", + "CM_HOST_SYSTEM_NAME": "intel-spr-i9" +``` + +## CM automation recipe for reproducing nvidia r4.0 GPT-J implementation + +We designed CM as a [small Python library](https://github.com/mlcommons/ck/tree/master/cm) +with a human-friendly command line, simple Python API and minimal dependencies +needed to implement automation recipes (Python 3.7+, PIP, pyyaml, git, wget) +and chain them into portable workflows. CM scripts can run natively (development mode) +or inside containers that CM generates on the fly (stable mode). + +Most of the time, these dependencies are already installed on your platform. +In such case, you should be able to prepare and run image classification with ONNX, +ImageNet validation data set and ResNet-50 on Linux, MacOS, Windows and any other +operating system using a few CM commands: + + + +```bash +pip install cm4mlops +cm run script --tags=run-mlperf,inference,_find-performance,_full,_r4.1-dev --model=gptj-99 --implementation=nvidia --framework=tensorrt --category=datacenter --scenario=Offline --execution_mode=test --device=cuda --docker --quiet --test_query_count=50 +``` + + + +*Note that you may need to re-login when you install cmind for the first time + to let your platform pick up path to the `cm` command line front-end.* + +## How CM scripts works? + +Next, we briefly explain how CM commands work. + +Whenever you run `cm run script --tags=run-mlperf,inference,_find-performance,_full,_r4.1-dev --model=gptj-99 --implementation=nvidia --framework=tensorrt --category=datacenter --scenario=Offline --execution_mode=test --device=cuda --docker --quiet --test_query_count=50`, +the [CM script automation](https://github.com/mlcommons/ck/blob/master/cm-mlops/automation/script/module.py) +will simply search for `_cm.yaml` and `_cm.json` files (CM meta-description dictionary) in all `script` +directories in all software projects registered in CM via `cm pull repo`. + +In our case, when we installed CM through `pip install cm4mlops`, we have installed `cmind` and have pulled [github.com/mlcommons/cm4mlops](https://github.com/mlcommons/cm4mlops) +that has MLCommons' MLPerf CM automation recipes embedded +in a [`script` directory](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). + +*Note that you can pull any public or private Git repository, download any software project + or register any local directory in the CM to search for embedded automation recipes.* + +CM will then try to match all your tags without `_` prefix (`_` in tags mark +the so-called CM script variations that customize a give script behavior +and will be described later) with a `tags` list in the CM meta-description dictionary. +In our case, it will match the corresponding [`_cm.yaml`](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L9) +in `$HOME/CM/repos/mlcommons@cm4mlops/script/app-image-classification-onnx-py/_cm.yaml` - +a wrapper for a given CM automation recipe. + +*Note that if you use unique ID instead of tags to identify automation (such as `3d5e908e472b417e`), + CM will try to match `uid` string in the CM meta descriptions instead of tags.* From 9c9531714cc6d344fd9318b4dd6b3e0c11a8d650 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:00:11 +0530 Subject: [PATCH 380/658] How CM runs automation recipes --- docs/getting-started.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index da6aee9822..fed00c50d1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -96,3 +96,21 @@ a wrapper for a given CM automation recipe. *Note that if you use unique ID instead of tags to identify automation (such as `3d5e908e472b417e`), CM will try to match `uid` string in the CM meta descriptions instead of tags.* + +## How CM runs automation recipes? + +Whenever CM finds a directory with a requested automation recipe, +it performs the following steps: +* run `preprocess` function in `customize.py` if exists +* run `run.sh` (Linux) or `run.bat` (Windows) if exists +* run `postprocess` function in `customize.py` if exists + +Such organization makes it possible to use either Python or native OS scripts or +both to implement CM automation recipes while minimizing the learning curve +for CM understanding, development and debugging as requested by CM users. + +Furthermore, CM scripts can keep the source code of implementations example - +image classification (as shown [here](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/app-image-classification-onnx-py/src)) +that we can easily move around +between projects without hardwiring paths and names. + From ee8b224d84d3bf0eaebe7613d7feb5d21cdd6db8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:08:29 +0530 Subject: [PATCH 381/658] unify inputs, outputs and env variables --- docs/getting-started.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index fed00c50d1..e2a1e4c903 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -114,3 +114,25 @@ image classification (as shown [here](https://github.com/mlcommons/ck/tree/maste that we can easily move around between projects without hardwiring paths and names. +## How CM unifies inputs, outputs and environment variables? + +CM allows you to pass environment variables to `customize.py` +and native scripts using `--env.ENV=VALUE`. + +When you use some flags such as `--model` in previouslyu mentioned MLPerf Inference GPT-J +example, it will be also converted into an environment variable +using [`input_mapping` dictionary](https://github.com/anandhu-eng/cm4mlops/blob/a7abc554cfee99f7de4eb508c34f8abbe4cdd663/script/run-mlperf-inference-app/_cm.yaml#L69) +in the CM meta description of this script. + +All environment variables are aggregated in `env` dictionary inside CM +and then passed to `preprocess` function in `customize.py` where you can modify +it programmatically. + +They are then passed to the `run` script. + +**Note:** + +Since new environment variables +are not preserved after `run` script, one can pass new environment variables +back to CM using `tmp-run-env.out` with ENV=KEY strings as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/run.sh#L37) +or using `tmp-run-state.json` as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/src/onnx_classify.py#L171). From 5a8858101e34fd02b3563019f86d8c50383e67d1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:19:41 +0530 Subject: [PATCH 382/658] chain automation recipes --- docs/getting-started.md | 109 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index e2a1e4c903..ef6cf1b8bd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -136,3 +136,112 @@ Since new environment variables are not preserved after `run` script, one can pass new environment variables back to CM using `tmp-run-env.out` with ENV=KEY strings as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/run.sh#L37) or using `tmp-run-state.json` as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/src/onnx_classify.py#L171). + +## How CM chains automation recipes into portable workflows? + +CM scripts provide a technology-agnostic wrapper with simple tags, CLI and Python API to prepare and run +user code snippets and native scripts/tools while unifying their inputs and outputs, paths and environment variables. + +Such architecture makes it possible to easily chain existing user scripts and tools into portable, technology-agnostic and powerful workflows +instead of substituting or rewriting them. + +It is possible to chain CM scripts using simple +[`deps` list](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L23) +in a meta description of a given script: + + + +```yaml +deps: +- tags: detect,os +- tags: get,sys-utils-cm +- names: + - python + - python3 + tags: get,python3 + +- tags: get,cuda + names: + - cuda + enable_if_env: + USE_CUDA: + - yes +- tags: get,cudnn + names: + - cudnn + enable_if_env: + USE_CUDA: + - yes + +- tags: get,dataset,imagenet,image-classification,original +- tags: get,dataset-aux,imagenet-aux,image-classification +- tags: get,ml-model,resnet50,_onnx,image-classification + names: + - ml-model + +- tags: get,generic-python-lib,_package.Pillow +- tags: get,generic-python-lib,_package.numpy +- tags: get,generic-python-lib,_package.opencv-python + + +- tags: get,generic-python-lib,_onnxruntime + names: + - onnxruntime + skip_if_env: + USE_CUDA: + - yes +- tags: get,generic-python-lib,_onnxruntime_gpu + names: + - onnxruntime + enable_if_env: + USE_CUDA: + - yes + +``` + + + +Each entry in this list is a dictionary that specifies which CM script to run using `tags`. +Internally, CM will be updating `env` dictionary (flat environment) and `state` dictionary +(to let scripts exchange complex data structures besides environment variables). + +If you run CM via command line, you can see internal `env` and `state` dictionaries by adding `-j` flag: + +```bash +cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg -j +``` + +*Note that we use similar approach for updating environment variables similar + to calling native scripts - by default, they do not alter environment + variables at the host. However, CM allows you to do that + by explicitly specifying which environment variables and state keys + will be updated at the host using `new_env_keys` and `new_state_keys` + in the meta of a given script as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L88). + This helped us make behavior of complex CM workflows more deterministic + and reproducible.* + +Each sub-dependency can be turned on or off using environment variables +using `enable_if_env` dictionary or `disable_if_env` dictionary. + +You can also specify `version_min`, `version_max` and `version` in these +dependencies. You can also give them some specific names such as `python` +and pass versions and environment variables only to a specific script in a pipeline as follows: +```bash +cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg --adr.python.version_min=3.9 +``` + +This functionality is usually implemented inside ad-hoc bash or shell scripts +with many hardwired paths and names - CM simply makes such scripts and tools +portable and reusable while enabling technology-agnostic automation workflows +with a unified interface that can adapt to any operating system and are easy +to understand. + +We can now assemble complex automation workflows by reusing all portable +scripts from [the community](https://access.cknowledge.org/playground/?action=scripts). + +In our example, we reused CM scripts to [detect OS features](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os), +install system dependencies on [any supported OS](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-sys-utils-cm) +(Ubuntu, MacOS, RHEL, Arch, Debian, SLES, Windows, etc), +detect or install Python and PIP packages, download and preprocess data sets and models, etc. + + From 2c04d05dc3ee15833919af1b0c67e9c55f59165e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:28:39 +0530 Subject: [PATCH 383/658] initial commit for script automation --- docs/getting-started.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index ef6cf1b8bd..8128eee367 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -244,4 +244,38 @@ install system dependencies on [any supported OS](https://github.com/mlcommons/c (Ubuntu, MacOS, RHEL, Arch, Debian, SLES, Windows, etc), detect or install Python and PIP packages, download and preprocess data sets and models, etc. +## CM Script Automation + +CM script automation manages the creation and execution of CM scripts. It closely relates to CM cache automation as the output of any CM script can be cached. +It also has extensions: + +1. Docker: allows CM script to be run inside a docker container +2. Docs: used to automatically generate the README file for a CM script + +## How to add new CM scripts? + +One of the main requirement for CM was to provide a very light-weight connectors +between existing automation scripts and tools rather than substituting them. + +You can add your own scripts and tools to CM using the following command +that will create a ready-to-use dummy CM script: + +```bash +cm add script my-script --tags=my,script +``` + +You can already run this dummy script and plug it into other CM workflows: +```bash +cmr "my script" +``` + +You can also run it from python as follows: +```bash +import cmind +output=cmind.access({'action':'run', + 'automation':'script', + 'tags':'my,script}) +if output['return']==0: print (output) +``` + From 8454357f8920d769c15203a32080bfe717bd48e4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:15:25 +0530 Subject: [PATCH 384/658] add new cm scripts --- docs/getting-started.md | 42 ++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 8128eee367..66d1e471bd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -252,21 +252,38 @@ It also has extensions: 1. Docker: allows CM script to be run inside a docker container 2. Docs: used to automatically generate the README file for a CM script -## How to add new CM scripts? +More theoretical explanation on CM Script and it's workflows could be found [here](https://github.com/mlcommons/cm4mlops/blob/anandhu-eng-patch-1/docs/index.md) + +### How to add new CM scripts? One of the main requirement for CM was to provide a very light-weight connectors between existing automation scripts and tools rather than substituting them. You can add your own scripts and tools to CM using the following command -that will create a ready-to-use dummy CM script: +that will create a ready-to-use dummy CM script named `hello-world`: ```bash -cm add script my-script --tags=my,script +cm add script hello-world --tags=hello-world,display,test ``` -You can already run this dummy script and plug it into other CM workflows: -```bash -cmr "my script" +This creates a bare cm script inside the local repo. The new folder structure would look something like this: +``` +├── CM +│ ├── index.json +│ ├── repos +│ │ ├── local +│ │ │ ├── cfg +│ │ │ ├── cache +│ │ │ ├── cmr.yaml +│ │ │ └── script +│ │ │ ├── hello-world +│ │ │ ├── _cm.yaml +│ │ │ ├── customize.py +│ │ │ ├── README-extra.md +│ │ │ ├── run.bat +│ │ │ └── run.sh +│ │ └── mlcommons@cm4mlops +│ └── repos.json ``` You can also run it from python as follows: @@ -274,8 +291,19 @@ You can also run it from python as follows: import cmind output=cmind.access({'action':'run', 'automation':'script', - 'tags':'my,script}) + 'tags':'hello-world,display,test`}) if output['return']==0: print (output) ``` +If you find that the script you are newly creating is similar to any existing scripts in any cm repository, +you could create a new script by copying another script entirely using the following command: + +``` +cm copy script .: +``` + +The `source_script` contains the name of the script that you want to make a copy and `target_script` contains the name of the new script that will be created as a copy of the `source_script`. +The existing script names in `cm4mlops` repo could be found [here](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). + + From d0868ee7d9b2d5806b0e922426e315eaf7685697 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:16:09 +0530 Subject: [PATCH 385/658] added information about variations --- docs/getting-started.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 66d1e471bd..1486958e79 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -305,5 +305,27 @@ cm copy script .: The `source_script` contains the name of the script that you want to make a copy and `target_script` contains the name of the new script that will be created as a copy of the `source_script`. The existing script names in `cm4mlops` repo could be found [here](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). +## How to customize CM scripts using variations? + +Sometimes we need to set multiple environment variables or run a set of extra CM scripts +for a specific purpose (different hardware target or model or dataset). + +We introduced special tags with `_`, called *variations* or *variation tags*, +that allow you to update a set of environment variables and add extra scripts +to the chain of dependencies. + +Such variations are defined using [`variations` dictionary](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L69) +in the meta description of a given CM script. + +For example, our script has 2 variations `_cuda` and `_cpu`. + +If you want to use CUDA implementation of the image classification example, +you can add this variation to the tags that will set `USE_CUDA` environment to `yes` +and will turn on a specific CM script in `deps` to install ONNX for CUDA: + +```bash +cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg +``` + From 6f404086a367734c55f8d76c3044059044bc4355 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:32:29 +0530 Subject: [PATCH 386/658] CM cache --- docs/getting-started.md | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 1486958e79..ecf5df53f9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -327,5 +327,57 @@ and will turn on a specific CM script in `deps` to install ONNX for CUDA: cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg ``` +## How to cache and reuse CM scripts' output? + +By default, CM scripts run in the current directory and record all new files there. + +For example, the following universal download script will download +computer mouse image to the current directory: + + + +```bash +cm run script "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e +``` + + + +In some cases, we want to cache and reuse the output of automation recipes (such as downloading models, preprocessing data sets or building some applications) +rather than just downloading it to the current directory. + +Following the feedback from our users, we implemented a `cache` automation in CM similar to `script`. +Whenever CM encounters `"cache":true` in a meta description of a given script, it will create +a `cache` directory in `$HOME/CM/repos/local` with some unique ID and the same tags as `script`, +and will execute that script there to record all the data in cache. + +Whenever the same CM script is executed and CM finds an associated cache entry, +it will skip execution and will reuse files from that entry. + +Furthermore, it is possible to reuse large cached files in other projects that call the same CM scripts! + +You can see cache entries and find a specific one as follows: + +```bash +cmr "get ml-model resnet50 _onnx" -j + +cm show cache +cm show cache "get ml-model resnet50 _onnx" +cm find cache "download file ml-model resnet50 _onnx" +cm info cache "download file ml-model resnet50 _onnx" +``` + +You can clean some cache entries as follows: +```bash +cm rm cache --tags=ml-model,resnet50 +``` + +You can also clean all CM `cache` entries and start from scratch as follows: +```bash +cm rm cache -f +``` + +In fact, you can remove `$HOME/CM` to reset CM framework completely +and remove all downloaded repositories and cached entries. + From 9f31c33f5522c4e9779d55c6c4b6b41cc2ccba62 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:41:19 +0530 Subject: [PATCH 387/658] virtualenv, debugging and CM with containers --- docs/getting-started.md | 122 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index ecf5df53f9..0e093e4d96 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -380,4 +380,126 @@ In fact, you can remove `$HOME/CM` to reset CM framework completely and remove all downloaded repositories and cached entries. +## How to use CM with Python virtual environments? + + +Using CM `cache` makes it possible to run CM automations for multiple virtual environments +installed inside CM `cache` entries. It is possible to run CM automations with different Python +virtual environments transparently to users while avoiding messing up native user environment. + +We created the following CM automation recipe to create virtual environments: + +```bash +cmr "install python-venv" --name=mlperf +cm show cache "python-venv name-mlperf" +export CM_SCRIPT_EXTRA_CMD="--adr.python.name=mlperf" +``` + +If you now run our image classification automation recipe, +it will reuse model and dataset from the cache, but will +use the newly created virtual environment `mlperf` for running the script. + + +## How to debug CM scripts? + +One of the requirements from CM users was to avoid new and/or complex ways to debug CM automations. +Using native scripts and Python code makes it possible to apply standard techniques and tools to debug CM automations. + +We were also asked to add `--debug` flag to open a shell after the last native script is executed - +this allows users to rerun the last command line with all environment variables and paths assembled by CM +while having a full and native access to change environment and run the final command +(such as pinning threads, changing batch sizes, modifying files, etc). + +You can try it as follows on Linux, MacOS, Windows or other operating system as follows: + +```bash +cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg --debug + +``` + +You can also use GDB via environment variable `--env.CM_RUN_PREFIX="gdb --args "` +to run the final command via GDB. + +## How to use CM with containers? + +One of the key requirements for CM was to run automation natively or inside containers in the same way. + +We want CM scripts to adapt to the current/latest environment natively or run in the +container automatically generated on the fly when requested by user for more stability and determinism. + +In such case, we can get rid of separate development of native scripts/workflows and Dockerfile +and use the same CM commands instead. + +To run a given script in an automatically-generated container, you can simply substitute `cm run script` +with `cm docker script` or `cmr` with `cmrd`: + +```bash +cm docker script "python app image-classification onnx _cpu" +``` + +CM will automatically generate a Dockerfile with Ubuntu 22.04 in the `dockerfiles` +directory of a given script, will build container with the same CM command +and will run it inside container. + +* If you want to stay in the container, you can add flag `--docker_it`. +* You can change OS inside container using `--docker_base_image`, `--docker_os` and `--docker_os_version`. + +The tricky part is when we want to use host files and directories with a given CM script inside container. +To make it easier for users, we have implemented automatic detection and mounting of files and directories +in CM script. + +Developers of a CM script just need to specify which flags and environment variables are local files or directories +using `input_paths` in `docker` dictionary of the meta-description of this script: + +```yaml +docker: + skip_run_cmd: 'no' + all_gpus: 'yes' + input_paths: + - input + - env.CM_IMAGE + - output + skip_input_for_fake_run: + - input + - env.CM_IMAGE + - output + - j + pre_run_cmds: + - echo \"CM pre run commands\" +``` + +When you run the same script via container with the local computer_mouse.jpg file as an input, +CM will automatically mount current directory and will update the input to the CM script +inside container with the internal path: + + + +```bash +cm docker script "python app image-classification onnx _cpu" --input=computer_mouse.jpg + +... + +docker build -f D:\Work1\CM\ck\cm-mlops\script\app-image-classification-onnx-py\dockerfiles\ubuntu_22.04.Dockerfile \ + -t cknowledge/cm-script-app-image-classification-onnx-py:ubuntu-22.04-latest . + +... + +Container launch command: +docker run --entrypoint "" --gpus=all -v D:\Work1\CM\ck\docs\computer_mouse.jpg:/cm-mount/Work1/CM/ck/docs/computer_mouse.jpg + cknowledge/cm-script-app-image-classification-onnx-py:ubuntu-22.04-latest + bash -c "echo \"CM pre run commands\" && + cm run script --tags=python,app,image-classification,onnx,_cpu + --input=/cm-mount/Work1/CM/ck/docs/computer_mouse.jpg " + +CM pre run commands + + +``` + + + +It is now possible to download large data sets and models to the host from CM containers +or pass host scratch pads and data to CM containers transparently to a user! + + From 3ba2dc9a55802d0357014f0d903426c67684bde8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:43:57 +0530 Subject: [PATCH 388/658] MLPerf Benchmarks via cm --- docs/getting-started.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 0e093e4d96..5dd3d0c66c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -501,5 +501,26 @@ CM pre run commands It is now possible to download large data sets and models to the host from CM containers or pass host scratch pads and data to CM containers transparently to a user! +## How to run MLPerf benchmarks via CM? + +CM was originally designed to make it easier to run [MLPerf inference benchmarks](https://arxiv.org/abs/1911.02549). + +While MLPerf inference has a common benchmarking engine called [loadgen](https://github.com/mlcommons/inference/tree/master/loadgen), +setting up a given platform, installing all tools, downloading and preprocessing all models and data sets, +updating paths and environment variables, figuring out default parameters for various scenarios, preparing a loadgen command line, +keeping track of continuous updates in MLPerf rules, running multiple experiments and submitting results +is a major challenge for old and new submitters (see [MLPerf inference v4.0 submitter orientation for automation](https://doi.org/10.5281/zenodo.10605079). + +We created several CM scripts to prepare and run different implementations of MLPerf inference (reference, Nvidia, Intel, Qualcomm, Deep Sparse, etc) +with a master CM script to run them all out-of-the-box natively or inside automatically-generated containers +[run-mlperf-inference-app](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script/run-mlperf-inference-app). +CM helped us to implement it as a simple pipeline with a common and human-friendly interface while reusing all existing automation recipes. + +This script was successfully validated to [modularize MLPerf inference benchmarks](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/README.md) +and help the community automate more than 95% of all performance and power submissions in the v3.1 round +across more than 120 system configurations (models, frameworks, hardware) +while reducing development and maintenance costs. + +Please check this [documentation](mlperf/inference) for more details. From 9c037357a32e76f17240a1fec8b29582be69351e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:15:00 +0530 Subject: [PATCH 389/658] small fix --- docs/getting-started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 5dd3d0c66c..303f1dd914 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -90,8 +90,8 @@ in a [`script` directory](https://github.com/mlcommons/cm4mlops/tree/mlperf-infe CM will then try to match all your tags without `_` prefix (`_` in tags mark the so-called CM script variations that customize a give script behavior and will be described later) with a `tags` list in the CM meta-description dictionary. -In our case, it will match the corresponding [`_cm.yaml`](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L9) -in `$HOME/CM/repos/mlcommons@cm4mlops/script/app-image-classification-onnx-py/_cm.yaml` - +In our case, it will match the corresponding [`_cm.yaml`](https://github.com/mlcommons/cm4mlops/blob/2592e2bccf09e4234070e38299d9d8d697f76eda/script/run-mlperf-inference-app/_cm.yaml#L19) +in `$HOME/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-app/_cm.yaml` - a wrapper for a given CM automation recipe. *Note that if you use unique ID instead of tags to identify automation (such as `3d5e908e472b417e`), From 247fd043cf19e8098af6c000a8e40afdfd2b7ec4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:16:56 +0530 Subject: [PATCH 390/658] uid update --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 303f1dd914..85067977c2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -94,7 +94,7 @@ In our case, it will match the corresponding [`_cm.yaml`](https://github.com/mlc in `$HOME/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-app/_cm.yaml` - a wrapper for a given CM automation recipe. -*Note that if you use unique ID instead of tags to identify automation (such as `3d5e908e472b417e`), +*Note that if you use unique ID instead of tags to identify automation (such as `4a5d5b13fd7e4ac8`), CM will try to match `uid` string in the CM meta descriptions instead of tags.* ## How CM runs automation recipes? From 90dfe077320e71f2d576a2f153256bff1fd17bae Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:22:55 +0530 Subject: [PATCH 391/658] styled --- docs/getting-started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 85067977c2..1f48f48240 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -249,8 +249,8 @@ detect or install Python and PIP packages, download and preprocess data sets and CM script automation manages the creation and execution of CM scripts. It closely relates to CM cache automation as the output of any CM script can be cached. It also has extensions: -1. Docker: allows CM script to be run inside a docker container -2. Docs: used to automatically generate the README file for a CM script +1. **Docker**: allows CM script to be run inside a docker container +2. **Docs**: used to automatically generate the README file for a CM script More theoretical explanation on CM Script and it's workflows could be found [here](https://github.com/mlcommons/cm4mlops/blob/anandhu-eng-patch-1/docs/index.md) From 9131e15ab7e4bd96bfa1e5aea4eec1ead3ea7a7c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:25:37 +0530 Subject: [PATCH 392/658] update folder structure - cm add script --- docs/getting-started.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 1f48f48240..25293200dc 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -268,22 +268,22 @@ cm add script hello-world --tags=hello-world,display,test This creates a bare cm script inside the local repo. The new folder structure would look something like this: ``` -├── CM -│ ├── index.json -│ ├── repos -│ │ ├── local -│ │ │ ├── cfg -│ │ │ ├── cache -│ │ │ ├── cmr.yaml -│ │ │ └── script -│ │ │ ├── hello-world -│ │ │ ├── _cm.yaml -│ │ │ ├── customize.py -│ │ │ ├── README-extra.md -│ │ │ ├── run.bat -│ │ │ └── run.sh -│ │ └── mlcommons@cm4mlops -│ └── repos.json +└── CM + ├── index.json + ├── repos + │ ├── local + │ │ ├── cfg + │ │ ├── cache + │ │ ├── cmr.yaml + │ │ └── script + │ │ └── hello-world + │ │ ├── _cm.yaml + │ │ ├── customize.py + │ │ ├── README-extra.md + │ │ ├── run.bat + │ │ └── run.sh + │ └── mlcommons@cm4mlops + └── repos.json ``` You can also run it from python as follows: From 529c264aecdef13fd79a1f9b6280a71f1d15343d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:35:05 +0530 Subject: [PATCH 393/658] Updated link to MLPerf inference docs --- docs/getting-started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 25293200dc..40003c9cb6 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -509,7 +509,7 @@ While MLPerf inference has a common benchmarking engine called [loadgen](https:/ setting up a given platform, installing all tools, downloading and preprocessing all models and data sets, updating paths and environment variables, figuring out default parameters for various scenarios, preparing a loadgen command line, keeping track of continuous updates in MLPerf rules, running multiple experiments and submitting results -is a major challenge for old and new submitters (see [MLPerf inference v4.0 submitter orientation for automation](https://doi.org/10.5281/zenodo.10605079). +is a major challenge for old and new submitters (see [MLPerf inference v4.0 submitter orientation for automation](https://doi.org/10.5281/zenodo.10605079)). We created several CM scripts to prepare and run different implementations of MLPerf inference (reference, Nvidia, Intel, Qualcomm, Deep Sparse, etc) with a master CM script to run them all out-of-the-box natively or inside automatically-generated containers @@ -521,6 +521,6 @@ and help the community automate more than 95% of all performance and power submi across more than 120 system configurations (models, frameworks, hardware) while reducing development and maintenance costs. -Please check this [documentation](mlperf/inference) for more details. +Please check this [documentation](https://docs.mlcommons.org/inference/) for more details. From f27a2bdaf380224133c8c76c7474043fcbc2db29 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:42:55 +0000 Subject: [PATCH 394/658] Update index.md --- docs/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 175d04c0d4..9a74cd2b34 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ Please check the [CM documentation](https://docs.mlcommons.org/ck) for more deta See the [automatically generated catalog](scripts/index.md) of all CM scripts from MLCommons. -## Getting started with CM scripts +## Understanding CM scripts * A CM script is identified by a set of tags and by unique ID. * Further each CM script can have multiple variations and they are identified by variation tags which are treated in the same way as tags and identified by a `_` prefix. @@ -69,8 +69,8 @@ Sometimes it is difficult to add all variations needed for a script like say `ba * By using `--new` input, a new cache entry can be forced even when an old one exist. * By default no depndencies are run for a cached entry unless `dynamic` key is set for it. -### Updating ENV from inside the run script -* [TBD] + +Please see [here](getting-started.md) for trying CM scripts. From 86fc0e2ed88687ce7b5ee14a0efb0761e2040546 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:47:31 +0000 Subject: [PATCH 395/658] Update getting-started.md --- docs/getting-started.md | 259 +--------------------------------------- 1 file changed, 4 insertions(+), 255 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 40003c9cb6..49f81c6939 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,10 +1,4 @@ -# cm-docs - -## CM Framework - -**CM** (Collective Mind) is a Python package with a CLI and API designed to create and manage automations. Two key automations developed using CM are **Script** and **Cache**, which streamline ML workflows, including managing Docker runs. -The **CM Python** package was developed by **Grigori Fursin**. The **Script** and **Cache** automations are part of the [cm4mlops](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference) repository, created by **Grigori Fursin** and **Arjun Suresh** and sponsored by OctoML, cKnowledge, cTuning and MLCommons. -The **CM scripts**, also housed in the [cm4mlops](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference) repository, are created and maintained by **Arjun Suresh**, **Anandhu Sooraj**, and **Grigori Fursin** with the help of the MLCommons community. +# Getting Started ## Simple script automation execution in cm @@ -46,213 +40,7 @@ This would capture the OS details of the system which it is run. Example details "CM_HOST_SYSTEM_NAME": "intel-spr-i9" ``` -## CM automation recipe for reproducing nvidia r4.0 GPT-J implementation - -We designed CM as a [small Python library](https://github.com/mlcommons/ck/tree/master/cm) -with a human-friendly command line, simple Python API and minimal dependencies -needed to implement automation recipes (Python 3.7+, PIP, pyyaml, git, wget) -and chain them into portable workflows. CM scripts can run natively (development mode) -or inside containers that CM generates on the fly (stable mode). - -Most of the time, these dependencies are already installed on your platform. -In such case, you should be able to prepare and run image classification with ONNX, -ImageNet validation data set and ResNet-50 on Linux, MacOS, Windows and any other -operating system using a few CM commands: - - - -```bash -pip install cm4mlops -cm run script --tags=run-mlperf,inference,_find-performance,_full,_r4.1-dev --model=gptj-99 --implementation=nvidia --framework=tensorrt --category=datacenter --scenario=Offline --execution_mode=test --device=cuda --docker --quiet --test_query_count=50 -``` - - - -*Note that you may need to re-login when you install cmind for the first time - to let your platform pick up path to the `cm` command line front-end.* - -## How CM scripts works? - -Next, we briefly explain how CM commands work. - -Whenever you run `cm run script --tags=run-mlperf,inference,_find-performance,_full,_r4.1-dev --model=gptj-99 --implementation=nvidia --framework=tensorrt --category=datacenter --scenario=Offline --execution_mode=test --device=cuda --docker --quiet --test_query_count=50`, -the [CM script automation](https://github.com/mlcommons/ck/blob/master/cm-mlops/automation/script/module.py) -will simply search for `_cm.yaml` and `_cm.json` files (CM meta-description dictionary) in all `script` -directories in all software projects registered in CM via `cm pull repo`. - -In our case, when we installed CM through `pip install cm4mlops`, we have installed `cmind` and have pulled [github.com/mlcommons/cm4mlops](https://github.com/mlcommons/cm4mlops) -that has MLCommons' MLPerf CM automation recipes embedded -in a [`script` directory](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). - -*Note that you can pull any public or private Git repository, download any software project - or register any local directory in the CM to search for embedded automation recipes.* - -CM will then try to match all your tags without `_` prefix (`_` in tags mark -the so-called CM script variations that customize a give script behavior -and will be described later) with a `tags` list in the CM meta-description dictionary. -In our case, it will match the corresponding [`_cm.yaml`](https://github.com/mlcommons/cm4mlops/blob/2592e2bccf09e4234070e38299d9d8d697f76eda/script/run-mlperf-inference-app/_cm.yaml#L19) -in `$HOME/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-app/_cm.yaml` - -a wrapper for a given CM automation recipe. - -*Note that if you use unique ID instead of tags to identify automation (such as `4a5d5b13fd7e4ac8`), - CM will try to match `uid` string in the CM meta descriptions instead of tags.* - -## How CM runs automation recipes? - -Whenever CM finds a directory with a requested automation recipe, -it performs the following steps: -* run `preprocess` function in `customize.py` if exists -* run `run.sh` (Linux) or `run.bat` (Windows) if exists -* run `postprocess` function in `customize.py` if exists - -Such organization makes it possible to use either Python or native OS scripts or -both to implement CM automation recipes while minimizing the learning curve -for CM understanding, development and debugging as requested by CM users. - -Furthermore, CM scripts can keep the source code of implementations example - -image classification (as shown [here](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/app-image-classification-onnx-py/src)) -that we can easily move around -between projects without hardwiring paths and names. - -## How CM unifies inputs, outputs and environment variables? - -CM allows you to pass environment variables to `customize.py` -and native scripts using `--env.ENV=VALUE`. - -When you use some flags such as `--model` in previouslyu mentioned MLPerf Inference GPT-J -example, it will be also converted into an environment variable -using [`input_mapping` dictionary](https://github.com/anandhu-eng/cm4mlops/blob/a7abc554cfee99f7de4eb508c34f8abbe4cdd663/script/run-mlperf-inference-app/_cm.yaml#L69) -in the CM meta description of this script. - -All environment variables are aggregated in `env` dictionary inside CM -and then passed to `preprocess` function in `customize.py` where you can modify -it programmatically. - -They are then passed to the `run` script. - -**Note:** - -Since new environment variables -are not preserved after `run` script, one can pass new environment variables -back to CM using `tmp-run-env.out` with ENV=KEY strings as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/run.sh#L37) -or using `tmp-run-state.json` as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/src/onnx_classify.py#L171). - -## How CM chains automation recipes into portable workflows? - -CM scripts provide a technology-agnostic wrapper with simple tags, CLI and Python API to prepare and run -user code snippets and native scripts/tools while unifying their inputs and outputs, paths and environment variables. - -Such architecture makes it possible to easily chain existing user scripts and tools into portable, technology-agnostic and powerful workflows -instead of substituting or rewriting them. - -It is possible to chain CM scripts using simple -[`deps` list](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L23) -in a meta description of a given script: - - - -```yaml -deps: -- tags: detect,os -- tags: get,sys-utils-cm -- names: - - python - - python3 - tags: get,python3 - -- tags: get,cuda - names: - - cuda - enable_if_env: - USE_CUDA: - - yes -- tags: get,cudnn - names: - - cudnn - enable_if_env: - USE_CUDA: - - yes - -- tags: get,dataset,imagenet,image-classification,original -- tags: get,dataset-aux,imagenet-aux,image-classification -- tags: get,ml-model,resnet50,_onnx,image-classification - names: - - ml-model - -- tags: get,generic-python-lib,_package.Pillow -- tags: get,generic-python-lib,_package.numpy -- tags: get,generic-python-lib,_package.opencv-python - - -- tags: get,generic-python-lib,_onnxruntime - names: - - onnxruntime - skip_if_env: - USE_CUDA: - - yes -- tags: get,generic-python-lib,_onnxruntime_gpu - names: - - onnxruntime - enable_if_env: - USE_CUDA: - - yes - -``` - - - -Each entry in this list is a dictionary that specifies which CM script to run using `tags`. -Internally, CM will be updating `env` dictionary (flat environment) and `state` dictionary -(to let scripts exchange complex data structures besides environment variables). - -If you run CM via command line, you can see internal `env` and `state` dictionaries by adding `-j` flag: - -```bash -cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg -j -``` - -*Note that we use similar approach for updating environment variables similar - to calling native scripts - by default, they do not alter environment - variables at the host. However, CM allows you to do that - by explicitly specifying which environment variables and state keys - will be updated at the host using `new_env_keys` and `new_state_keys` - in the meta of a given script as shown [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L88). - This helped us make behavior of complex CM workflows more deterministic - and reproducible.* - -Each sub-dependency can be turned on or off using environment variables -using `enable_if_env` dictionary or `disable_if_env` dictionary. - -You can also specify `version_min`, `version_max` and `version` in these -dependencies. You can also give them some specific names such as `python` -and pass versions and environment variables only to a specific script in a pipeline as follows: -```bash -cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg --adr.python.version_min=3.9 -``` - -This functionality is usually implemented inside ad-hoc bash or shell scripts -with many hardwired paths and names - CM simply makes such scripts and tools -portable and reusable while enabling technology-agnostic automation workflows -with a unified interface that can adapt to any operating system and are easy -to understand. -We can now assemble complex automation workflows by reusing all portable -scripts from [the community](https://access.cknowledge.org/playground/?action=scripts). - -In our example, we reused CM scripts to [detect OS features](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os), -install system dependencies on [any supported OS](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-sys-utils-cm) -(Ubuntu, MacOS, RHEL, Arch, Debian, SLES, Windows, etc), -detect or install Python and PIP packages, download and preprocess data sets and models, etc. - -## CM Script Automation - -CM script automation manages the creation and execution of CM scripts. It closely relates to CM cache automation as the output of any CM script can be cached. -It also has extensions: - -1. **Docker**: allows CM script to be run inside a docker container -2. **Docs**: used to automatically generate the README file for a CM script - -More theoretical explanation on CM Script and it's workflows could be found [here](https://github.com/mlcommons/cm4mlops/blob/anandhu-eng-patch-1/docs/index.md) ### How to add new CM scripts? @@ -305,27 +93,6 @@ cm copy script .: The `source_script` contains the name of the script that you want to make a copy and `target_script` contains the name of the new script that will be created as a copy of the `source_script`. The existing script names in `cm4mlops` repo could be found [here](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). -## How to customize CM scripts using variations? - -Sometimes we need to set multiple environment variables or run a set of extra CM scripts -for a specific purpose (different hardware target or model or dataset). - -We introduced special tags with `_`, called *variations* or *variation tags*, -that allow you to update a set of environment variables and add extra scripts -to the chain of dependencies. - -Such variations are defined using [`variations` dictionary](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml#L69) -in the meta description of a given CM script. - -For example, our script has 2 variations `_cuda` and `_cpu`. - -If you want to use CUDA implementation of the image classification example, -you can add this variation to the tags that will set `USE_CUDA` environment to `yes` -and will turn on a specific CM script in `deps` to install ONNX for CUDA: - -```bash -cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg -``` ## How to cache and reuse CM scripts' output? @@ -337,7 +104,7 @@ computer mouse image to the current directory: ```bash -cm run script "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e +cm run script --tags=download,file,_wget --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e ``` @@ -434,7 +201,7 @@ To run a given script in an automatically-generated container, you can simply su with `cm docker script` or `cmr` with `cmrd`: ```bash -cm docker script "python app image-classification onnx _cpu" +cm docker script --tags=python,app,image-classification,onnx,_cpu" ``` CM will automatically generate a Dockerfile with Ubuntu 22.04 in the `dockerfiles` @@ -475,7 +242,7 @@ inside container with the internal path: ```bash -cm docker script "python app image-classification onnx _cpu" --input=computer_mouse.jpg +cm docker script --tags=python,app,image-classification,onnx,_cpu --input=computer_mouse.jpg ... @@ -503,24 +270,6 @@ or pass host scratch pads and data to CM containers transparently to a user! ## How to run MLPerf benchmarks via CM? -CM was originally designed to make it easier to run [MLPerf inference benchmarks](https://arxiv.org/abs/1911.02549). - -While MLPerf inference has a common benchmarking engine called [loadgen](https://github.com/mlcommons/inference/tree/master/loadgen), -setting up a given platform, installing all tools, downloading and preprocessing all models and data sets, -updating paths and environment variables, figuring out default parameters for various scenarios, preparing a loadgen command line, -keeping track of continuous updates in MLPerf rules, running multiple experiments and submitting results -is a major challenge for old and new submitters (see [MLPerf inference v4.0 submitter orientation for automation](https://doi.org/10.5281/zenodo.10605079)). - -We created several CM scripts to prepare and run different implementations of MLPerf inference (reference, Nvidia, Intel, Qualcomm, Deep Sparse, etc) -with a master CM script to run them all out-of-the-box natively or inside automatically-generated containers -[run-mlperf-inference-app](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script/run-mlperf-inference-app). -CM helped us to implement it as a simple pipeline with a common and human-friendly interface while reusing all existing automation recipes. - -This script was successfully validated to [modularize MLPerf inference benchmarks](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/README.md) -and help the community automate more than 95% of all performance and power submissions in the v3.1 round -across more than 120 system configurations (models, frameworks, hardware) -while reducing development and maintenance costs. - Please check this [documentation](https://docs.mlcommons.org/inference/) for more details. From 417c5edd1f6533952cb67a561d1fb63bd733e9ad Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:48:52 +0000 Subject: [PATCH 396/658] Update getting-started.md --- docs/getting-started.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 49f81c6939..88b4b6425c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -157,8 +157,8 @@ virtual environments transparently to users while avoiding messing up native use We created the following CM automation recipe to create virtual environments: ```bash -cmr "install python-venv" --name=mlperf -cm show cache "python-venv name-mlperf" +cm run script --tags=install,python-venv --name=mlperf +cm show cache --tags=python-venv,name-mlperf export CM_SCRIPT_EXTRA_CMD="--adr.python.name=mlperf" ``` @@ -180,7 +180,7 @@ while having a full and native access to change environment and run the final co You can try it as follows on Linux, MacOS, Windows or other operating system as follows: ```bash -cmr "python app image-classification onnx _cpu" --input=computer_mouse.jpg --debug +cm run script --tags=python,app,image-classification,onnx,_cpu --input=computer_mouse.jpg --debug ``` From 9fca4eb72a010738de45e05d594444eccdc0f6c4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:49:30 +0000 Subject: [PATCH 397/658] Update getting-started.md --- docs/getting-started.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 88b4b6425c..f5b621790e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -104,7 +104,8 @@ computer mouse image to the current directory: ```bash -cm run script --tags=download,file,_wget --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e +cm run script --tags=download,file,_wget --url=https://cKnowledge.org/ai/data/computer_mouse.jpg \ +--verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e ``` From 6b6f2ab8a43da82358dd84ed6df3b4df465306cb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:50:28 +0000 Subject: [PATCH 398/658] Update getting-started.md --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index f5b621790e..f1fb3f6f5a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -126,7 +126,7 @@ Furthermore, it is possible to reuse large cached files in other projects that c You can see cache entries and find a specific one as follows: ```bash -cmr "get ml-model resnet50 _onnx" -j +cm run script --tags=get,ml-model,resnet50,_onnx -j cm show cache cm show cache "get ml-model resnet50 _onnx" From 8d1e631b5113f1afc4eb837651d07fd87397bc2a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:51:00 +0000 Subject: [PATCH 399/658] Update getting-started.md --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index f1fb3f6f5a..623a6da414 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -202,7 +202,7 @@ To run a given script in an automatically-generated container, you can simply su with `cm docker script` or `cmr` with `cmrd`: ```bash -cm docker script --tags=python,app,image-classification,onnx,_cpu" +cm docker script --tags=python,app,image-classification,onnx,_cpu ``` CM will automatically generate a Dockerfile with Ubuntu 22.04 in the `dockerfiles` From 47502346ef987880a9fb02125ef378eda9a02d8a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 14:56:26 +0000 Subject: [PATCH 400/658] Update getting-started.md --- docs/getting-started.md | 235 ++++++++-------------------------------- 1 file changed, 47 insertions(+), 188 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 623a6da414..baed31eea3 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,15 +1,18 @@ -# Getting Started -## Simple script automation execution in cm +# Getting Started with CM Script Automation -A simple script automation execution in CM is as follows: +## Running CM Scripts -``` +To execute a simple script in CM that captures OS details, use the following command: + +```bash cm run script --tags=detect,os -j ``` -This would capture the OS details of the system which it is run. Example details could be found here: -``` +This command gathers details about the system on which it's run, such as: + +```json +{ "CM_HOST_OS_TYPE": "linux", "CM_HOST_OS_BITS": "64", "CM_HOST_OS_FLAVOR": "ubuntu", @@ -38,23 +41,21 @@ This would capture the OS details of the system which it is run. Example details "CM_HOST_PLATFORM_FLAVOR": "x86_64", "CM_HOST_PYTHON_BITS": "64", "CM_HOST_SYSTEM_NAME": "intel-spr-i9" +} ``` +For more details on CM scripts, see the [CM documentation](index.md). +### Adding New CM Scripts -### How to add new CM scripts? - -One of the main requirement for CM was to provide a very light-weight connectors -between existing automation scripts and tools rather than substituting them. - -You can add your own scripts and tools to CM using the following command -that will create a ready-to-use dummy CM script named `hello-world`: +CM aims to provide lightweight connectors between existing automation scripts and tools without substituting them. You can add your own scripts to CM with the following command, which creates a script named `hello-world`: ```bash cm add script hello-world --tags=hello-world,display,test ``` -This creates a bare cm script inside the local repo. The new folder structure would look something like this: +This command initializes a CM script in the local repository with the following structure: + ``` └── CM ├── index.json @@ -64,213 +65,71 @@ This creates a bare cm script inside the local repo. The new folder structure wo │ │ ├── cache │ │ ├── cmr.yaml │ │ └── script - │ │ └── hello-world - │ │ ├── _cm.yaml - │ │ ├── customize.py - │ │ ├── README-extra.md - │ │ ├── run.bat - │ │ └── run.sh + │ │ └── hello-world + │ │ ├── _cm.yaml + │ │ ├── customize.py + │ │ ├── README-extra.md + │ │ ├── run.bat + │ │ └── run.sh │ └── mlcommons@cm4mlops └── repos.json ``` -You can also run it from python as follows: -```bash +You can also execute the script from Python as follows: + +```python import cmind -output=cmind.access({'action':'run', - 'automation':'script', - 'tags':'hello-world,display,test`}) -if output['return']==0: print (output) +output = cmind.access({'action':'run', 'automation':'script', 'tags':'hello-world,display,test'}) +if output['return'] == 0: + print(output) ``` -If you find that the script you are newly creating is similar to any existing scripts in any cm repository, -you could create a new script by copying another script entirely using the following command: +If you discover that your new script is similar to an existing script in any CM repository, you can clone an existing script using the following command: -``` +```bash cm copy script .: ``` -The `source_script` contains the name of the script that you want to make a copy and `target_script` contains the name of the new script that will be created as a copy of the `source_script`. -The existing script names in `cm4mlops` repo could be found [here](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). +Here, `` is the name of the existing script, and `` is the name of the new script you're creating. Existing script names in the `cm4mlops` repository can be found [here](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script). +## Caching and Reusing CM Script Outputs -## How to cache and reuse CM scripts' output? - -By default, CM scripts run in the current directory and record all new files there. - -For example, the following universal download script will download -computer mouse image to the current directory: - - +By default, CM scripts run in the current directory and record all new files there. For example, a universal download script might download an image to the current directory: ```bash -cm run script --tags=download,file,_wget --url=https://cKnowledge.org/ai/data/computer_mouse.jpg \ ---verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e +cm run script --tags=download,file,_wget --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e ``` - - -In some cases, we want to cache and reuse the output of automation recipes (such as downloading models, preprocessing data sets or building some applications) -rather than just downloading it to the current directory. +To cache and reuse the output of scripts, CM offers a `cache` automation feature similar to `script`. When `"cache":true` is specified in a script's metadata, CM will create a `cache` directory in `$HOME/CM/repos/local` with a unique ID and the same tags as `script`, and execute the script there. -Following the feedback from our users, we implemented a `cache` automation in CM similar to `script`. -Whenever CM encounters `"cache":true` in a meta description of a given script, it will create -a `cache` directory in `$HOME/CM/repos/local` with some unique ID and the same tags as `script`, -and will execute that script there to record all the data in cache. +Subsequent executions of the same script will reuse files from the cache, avoiding redundancy. This is especially useful for large files or data sets. -Whenever the same CM script is executed and CM finds an associated cache entry, -it will skip execution and will reuse files from that entry. - -Furthermore, it is possible to reuse large cached files in other projects that call the same CM scripts! - -You can see cache entries and find a specific one as follows: +You can manage cache entries and find specific ones using commands like: ```bash -cm run script --tags=get,ml-model,resnet50,_onnx -j - cm show cache -cm show cache "get ml-model resnet50 _onnx" -cm find cache "download file ml-model resnet50 _onnx" -cm info cache "download file ml-model resnet50 _onnx" -``` - -You can clean some cache entries as follows: -```bash -cm rm cache --tags=ml-model,resnet50 -``` - -You can also clean all CM `cache` entries and start from scratch as follows: -```bash -cm rm cache -f +cm show cache --tags=get,ml-model,resnet50,_onnx +cm find cache --tags=download,file,ml-model,resnet50,_onnx +cm info cache --tags=download,file,ml-model,resnet50,_onnx ``` -In fact, you can remove `$HOME/CM` to reset CM framework completely -and remove all downloaded repositories and cached entries. - - -## How to use CM with Python virtual environments? - - -Using CM `cache` makes it possible to run CM automations for multiple virtual environments -installed inside CM `cache` entries. It is possible to run CM automations with different Python -virtual environments transparently to users while avoiding messing up native user environment. - -We created the following CM automation recipe to create virtual environments: +To clean cache entries: ```bash -cm run script --tags=install,python-venv --name=mlperf -cm show cache --tags=python-venv,name-mlperf -export CM_SCRIPT_EXTRA_CMD="--adr.python.name=mlperf" -``` - -If you now run our image classification automation recipe, -it will reuse model and dataset from the cache, but will -use the newly created virtual environment `mlperf` for running the script. - - -## How to debug CM scripts? - -One of the requirements from CM users was to avoid new and/or complex ways to debug CM automations. -Using native scripts and Python code makes it possible to apply standard techniques and tools to debug CM automations. - -We were also asked to add `--debug` flag to open a shell after the last native script is executed - -this allows users to rerun the last command line with all environment variables and paths assembled by CM -while having a full and native access to change environment and run the final command -(such as pinning threads, changing batch sizes, modifying files, etc). - -You can try it as follows on Linux, MacOS, Windows or other operating system as follows: - -```bash -cm run script --tags=python,app,image-classification,onnx,_cpu --input=computer_mouse.jpg --debug - +cm rm cache --tags=ml-model,resnet50 +cm rm cache -f # Clean all entries ``` -You can also use GDB via environment variable `--env.CM_RUN_PREFIX="gdb --args "` -to run the final command via GDB. - -## How to use CM with containers? - -One of the key requirements for CM was to run automation natively or inside containers in the same way. +You can completely reset the CM framework by removing the `$HOME/CM` directory, which deletes all downloaded repositories and cached entries. -We want CM scripts to adapt to the current/latest environment natively or run in the -container automatically generated on the fly when requested by user for more stability and determinism. +## Integration with Containers -In such case, we can get rid of separate development of native scripts/workflows and Dockerfile -and use the same CM commands instead. - -To run a given script in an automatically-generated container, you can simply substitute `cm run script` -with `cm docker script` or `cmr` with `cmrd`: +CM scripts are designed to run natively or inside containers with the same commands. You can substitute `cm run script` with `cm docker script` to execute a script inside an automatically-generated container: ```bash cm docker script --tags=python,app,image-classification,onnx,_cpu ``` -CM will automatically generate a Dockerfile with Ubuntu 22.04 in the `dockerfiles` -directory of a given script, will build container with the same CM command -and will run it inside container. - -* If you want to stay in the container, you can add flag `--docker_it`. -* You can change OS inside container using `--docker_base_image`, `--docker_os` and `--docker_os_version`. - -The tricky part is when we want to use host files and directories with a given CM script inside container. -To make it easier for users, we have implemented automatic detection and mounting of files and directories -in CM script. - -Developers of a CM script just need to specify which flags and environment variables are local files or directories -using `input_paths` in `docker` dictionary of the meta-description of this script: - -```yaml -docker: - skip_run_cmd: 'no' - all_gpus: 'yes' - input_paths: - - input - - env.CM_IMAGE - - output - skip_input_for_fake_run: - - input - - env.CM_IMAGE - - output - - j - pre_run_cmds: - - echo \"CM pre run commands\" -``` - -When you run the same script via container with the local computer_mouse.jpg file as an input, -CM will automatically mount current directory and will update the input to the CM script -inside container with the internal path: - - - -```bash -cm docker script --tags=python,app,image-classification,onnx,_cpu --input=computer_mouse.jpg - -... - -docker build -f D:\Work1\CM\ck\cm-mlops\script\app-image-classification-onnx-py\dockerfiles\ubuntu_22.04.Dockerfile \ - -t cknowledge/cm-script-app-image-classification-onnx-py:ubuntu-22.04-latest . - -... - -Container launch command: -docker run --entrypoint "" --gpus=all -v D:\Work1\CM\ck\docs\computer_mouse.jpg:/cm-mount/Work1/CM/ck/docs/computer_mouse.jpg - cknowledge/cm-script-app-image-classification-onnx-py:ubuntu-22.04-latest - bash -c "echo \"CM pre run commands\" && - cm run script --tags=python,app,image-classification,onnx,_cpu - --input=/cm-mount/Work1/CM/ck/docs/computer_mouse.jpg " - -CM pre run commands - - -``` - - - -It is now possible to download large data sets and models to the host from CM containers -or pass host scratch pads and data to CM containers transparently to a user! - -## How to run MLPerf benchmarks via CM? - -Please check this [documentation](https://docs.mlcommons.org/inference/) for more details. - +CM automatically handles the generation of Dockerfiles, building of containers, and execution within containers, providing a seamless experience whether running scripts natively or in containers. +This approach simplifies the development process by eliminating the need for separate Dockerfile maintenance and allows for the use of native scripts and workflows directly within containers. From 4bb1f3738ba0830f113d7af7d6a6db3dec12405c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 15:05:59 +0000 Subject: [PATCH 401/658] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index be8fe1bf8c..843040705d 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,7 @@ [![Test QAIC Software kit Compilation](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml/badge.svg)](https://github.com/mlcommons/cm4mlops/actions/workflows/test-qaic-software-kit.yml) -The `mlperf-branch` of the **cm4mlops** repository contains updated CM scripts specifically for MLPerf Inference, including support for Automotive. Please note that the general CM scripts in this branch may not be compatible with other projects. For more information on using CM for MLPerf Inference, visit the [MLPerf Inference Documentation site](https://docs.mlcommons.org/inference/). - -[![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) +Please see the [docs](https://docs.mlcommons.org/cm4mlops/) site for understanding CM scripts better. The `mlperf-branch` of the **cm4mlops** repository contains updated CM scripts specifically for MLPerf Inference. For more information on using CM for MLPerf Inference, visit the [MLPerf Inference Documentation site](https://docs.mlcommons.org/inference/). ## News From cf9c61eb444169c7cb9ae3e07178576d76011b5b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:57:37 +0530 Subject: [PATCH 402/658] Add getting started to cm4mlops docs --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 7eda0cd8aa..4cb2956917 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,6 +20,7 @@ theme: - toc.follow nav: - HOME: index.md + - Getting Started: getting-started.md - CM Scripts: - scripts/index.md - Python-automation: scripts/Python-automation/index.md From eaa793892ecceb969d2cd264114342396b622e2b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 01:45:54 +0530 Subject: [PATCH 403/658] Fix docker container reuse during docker_detached mode --- script/run-mlperf-inference-app/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 629bc3a254..f0e1c02f5f 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -202,7 +202,7 @@ def preprocess(i): docker_extra_input[k] = inp[k] inp = {} if str(docker_dt).lower() in ["yes", "true", "1"]: - env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'yes' + env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'no' # turning it off for the first run and after that we turn it on if env.get('CM_DOCKER_IMAGE_NAME', '') != '': docker_extra_input['docker_image_name'] = env['CM_DOCKER_IMAGE_NAME'] @@ -256,6 +256,7 @@ def preprocess(i): print(f"\nStop Running loadgen scenario: {scenario} and mode: {mode}") return {'return': 0} # We run commands interactively inside the docker container else: + env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'yes' container_id = env_copy['CM_DOCKER_CONTAINER_ID'] env['CM_DOCKER_CONTAINER_ID'] = container_id if state.get('docker', {}): From 018d09c32f2456bebac968a03975337def717dc6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 20:17:23 +0000 Subject: [PATCH 404/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index e51634c468..f9fc94d7be 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "04 18 * * *" #to be adjusted + - cron: "23 20 * * *" #to be adjusted jobs: build_nvidia: From 52dae77db81de4f60a972311d223e952ce23fb9a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 02:35:35 +0530 Subject: [PATCH 405/658] Fix docker container reuse during docker_detached mode --- script/run-docker-container/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 20f8138bad..35d3fdf1e4 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -62,6 +62,7 @@ def preprocess(i): if len(output_split) > 1 and str(env.get('CM_DOCKER_REUSE_EXISTING_CONTAINER', '')).lower() in [ "1", "true", "yes" ]: #container exists out = output_split[1].split(" ") existing_container_id = out[0] + print(f"Reusing existing container {existing_container_id}") env['CM_DOCKER_CONTAINER_ID'] = existing_container_id else: From 0a7423b8b651228c6cf952e3957ed6a8df68ae8b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 02:48:20 +0530 Subject: [PATCH 406/658] By default use existing docker image --- automation/script/module_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 92c8a5f717..c4e508c352 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1692,7 +1692,7 @@ def docker(i): env=i.get('env', {}) noregenerate_docker_file = i.get('docker_noregenerate', False) - norecreate_docker_image = i.get('docker_norecreate', False) + norecreate_docker_image = i.get('docker_norecreate', True) if i.get('docker_skip_build', False): noregenerate_docker_file = True From fecec8587e4d61692d03354e4e095ddf31165c4e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Oct 2024 21:19:39 +0000 Subject: [PATCH 407/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index f9fc94d7be..b1cfb11d0a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "23 20 * * *" #to be adjusted + - cron: "25 21 * * *" #to be adjusted jobs: build_nvidia: From 25b7043d3863fcdaf7cf8a52f6e192f2dfe0a658 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:48:34 +0530 Subject: [PATCH 408/658] Create test-submission-generation-non-cm-based-benchmarks.yml --- ...ion-generation-non-cm-based-benchmarks.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/test-submission-generation-non-cm-based-benchmarks.yml diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml new file mode 100644 index 0000000000..0e0e401f3d --- /dev/null +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -0,0 +1,38 @@ +# This workflow will test the submission generation capability of CM for non-CM based benchmarks + +name: Submission generation tests for non-CM based MLPerf Inference Benchmarks + +on: + pull_request_target: + branches: [ "main", "dev", "mlperf-inference" ] + paths: + - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' + - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + - '!**.md' +jobs: + Case-3: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + exclude: + - os: macos-latest + - os: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Pull repo where test cases are uploaded + run: | + cm pull repo anandhu-eng@externalSubmissionPOC + - name: Submission generation - datacenter open + run: | + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repo/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From b11a57b52bf7f30b81bc4b0d698d3e2c133848ab Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:51:53 +0530 Subject: [PATCH 409/658] Update run conditions for testing --- .../test-submission-generation-non-cm-based-benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 0e0e401f3d..eb8837be4a 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -7,8 +7,8 @@ on: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' - - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - - '!**.md' + #- '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + #- '!**.md' jobs: Case-3: runs-on: ${{ matrix.os }} From 0b9532f892bee2cbb26e985a5444e7616f06e030 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:03:54 +0530 Subject: [PATCH 410/658] test commit --- .../test-submission-generation-non-cm-based-benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index eb8837be4a..0e0e401f3d 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -7,8 +7,8 @@ on: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' - #- '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - #- '!**.md' + - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + - '!**.md' jobs: Case-3: runs-on: ${{ matrix.os }} From cb09d504c7b4d64a463491722ed510309f1062bf Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:23:59 +0530 Subject: [PATCH 411/658] Update test-submission-generation-non-cm-based-benchmarks.yml --- .../test-submission-generation-non-cm-based-benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 0e0e401f3d..ad1d662378 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -1,14 +1,14 @@ # This workflow will test the submission generation capability of CM for non-CM based benchmarks -name: Submission generation tests for non-CM based MLPerf Inference Benchmarks +name: CM based Submission Generation on: pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' - - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - - '!**.md' + # - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + # - '!**.md' jobs: Case-3: runs-on: ${{ matrix.os }} From 265224afa0ec2a05e0a6a4ea4edc8606126a83d3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:00:12 +0530 Subject: [PATCH 412/658] Update test-submission-generation-non-cm-based-benchmarks.yml --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index ad1d662378..dd5332d97c 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -3,7 +3,7 @@ name: CM based Submission Generation on: - pull_request_target: + pull_request: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' From 3fcfca22d6294e3823f17516e5b45f9422da73c3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:03:27 +0530 Subject: [PATCH 413/658] update results dir --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index dd5332d97c..e78d3c3c58 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -35,4 +35,4 @@ jobs: cm pull repo anandhu-eng@externalSubmissionPOC - name: Submission generation - datacenter open run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repo/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repoS/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From 5354309ea3338f491d60d31493096fa1699eabdd Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:04:29 +0530 Subject: [PATCH 414/658] Update test-submission-generation-non-cm-based-benchmarks.yml --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index e78d3c3c58..9d0d1e9bd4 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -35,4 +35,4 @@ jobs: cm pull repo anandhu-eng@externalSubmissionPOC - name: Submission generation - datacenter open run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repoS/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From c89dfa1d3896e7a6c8ab34ef1739ef6df20d3b74 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:11:35 +0530 Subject: [PATCH 415/658] division and category made matrix --- ...test-submission-generation-non-cm-based-benchmarks.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 9d0d1e9bd4..8e7e7f6668 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -17,9 +17,13 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: [ "3.12" ] + division: ["closed", "open"] + category: ["datacenter", "edge"] exclude: - os: macos-latest - os: windows-latest + - division: "open" + - division: "edge" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -33,6 +37,6 @@ jobs: - name: Pull repo where test cases are uploaded run: | cm pull repo anandhu-eng@externalSubmissionPOC - - name: Submission generation - datacenter open + - name: Submission generation - ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From 02675cb6f1399f1097abea22d0b3d6bf3a396684 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:15:42 +0530 Subject: [PATCH 416/658] fix typo --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 8e7e7f6668..2ae4e84250 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -23,7 +23,7 @@ jobs: - os: macos-latest - os: windows-latest - division: "open" - - division: "edge" + - category: "edge" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 75f8ea13f3783629b8cac1e9300b66cbe780a2a6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:28:00 +0530 Subject: [PATCH 417/658] added redhat os to test coverage --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 2ae4e84250..5b6b453477 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest, rhel-latest] python-version: [ "3.12" ] division: ["closed", "open"] category: ["datacenter", "edge"] From 4b5b6c978c3fa0861273df19528d44a426564182 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 10:24:46 +0000 Subject: [PATCH 418/658] Don't fail on dmidecode command in get-platform-details --- script/get-platform-details/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index d3fc7e0d37..acac8e0b23 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -120,7 +120,7 @@ echo "------------------------------------------------------------" >> $OUTPUT_F echo "21. dmidecode" >> $OUTPUT_FILE if [[ ${CM_SUDO_USER} == "yes" ]]; then eval "${CM_SUDO} dmidecode" >> $OUTPUT_FILE - test $? -eq 0 || exit $? + test $? -eq 0 || echo "FAILED: dmidecode" >> $OUTPUT_FILE else echo "Requires SUDO permission" >> $OUTPUT_FILE fi @@ -129,7 +129,7 @@ echo "------------------------------------------------------------" >> $OUTPUT_F echo "22. BIOS" >> $OUTPUT_FILE if [[ ${CM_SUDO_USER} == "yes" ]]; then eval "${CM_SUDO} dmidecode -t bios" >> $OUTPUT_FILE - test $? -eq 0 || exit $? + test $? -eq 0 || echo "FAILED: dmidecode -t bios" >> $OUTPUT_FILE else echo "Requires SUDO permission" >> $OUTPUT_FILE fi From a5eff8e6f4e0f1d0c59f4118f3a7c62b3f5a6b30 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 16:13:02 +0530 Subject: [PATCH 419/658] Added fail safe option for get-generic-sys-util --- script/get-generic-sys-util/run.sh | 23 ++++++++++++++++++----- script/get-platform-details/_cm.json | 5 ++++- script/get-platform-details/run.sh | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/run.sh index 27c2f62867..5074e756b8 100644 --- a/script/get-generic-sys-util/run.sh +++ b/script/get-generic-sys-util/run.sh @@ -1,6 +1,19 @@ -#!/bin/bash +# Safe execution of a command stored in a variable +cmd="${CM_SYS_UTIL_INSTALL_CMD}" +echo "$cmd" -cmd=${CM_SYS_UTIL_INSTALL_CMD} -echo $cmd -eval $cmd -test $? -eq 0 || exit $? +# Execute the command and capture the exit status directly +if ! eval "$cmd"; then + echo "Command failed with status $?" + if [[ "${CM_TMP_FAIL_SAFE}" == 'yes' ]]; then + # Exit safely if fail-safe is enabled + echo "Fail-safe is enabled, exiting with status 0" + exit 0 + else + # Otherwise exit with the actual error status + exit $? + fi +else + #echo "Command succeeded" + exit 0 +fi diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json index 11c57baa18..b02576ad9e 100644 --- a/script/get-platform-details/_cm.json +++ b/script/get-platform-details/_cm.json @@ -66,7 +66,10 @@ "linux" ] }, - "tags": "get,sys-util,generic,_linux-tools" + "tags": "get,sys-util,generic,_linux-tools", + "env": { + "CM_TMP_FAIL_SAFE": "yes" + } } ], "tags": [ diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index acac8e0b23..ba2194e669 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -79,7 +79,7 @@ echo "------------------------------------------------------------" >> $OUTPUT_F echo "14. cpupower frequency-info" >> $OUTPUT_FILE eval "cpupower frequency-info" >> $OUTPUT_FILE -test $? -eq 0 || exit $? +test $? -eq 0 || echo "FAILED: cpupower frequency-info" >> $OUTPUT_FILE echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "15. sysctl" >> $OUTPUT_FILE From 2ccf3737b339a1acb03b2a69676576ce859130d0 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:32:42 +0530 Subject: [PATCH 420/658] removed rhel latest --- ...ed-benchmarks.yml => test-cm-based-ubmission-generation.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{test-submission-generation-non-cm-based-benchmarks.yml => test-cm-based-ubmission-generation.yml} (96%) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-cm-based-ubmission-generation.yml similarity index 96% rename from .github/workflows/test-submission-generation-non-cm-based-benchmarks.yml rename to .github/workflows/test-cm-based-ubmission-generation.yml index 5b6b453477..2ae4e84250 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-cm-based-ubmission-generation.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest, rhel-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python-version: [ "3.12" ] division: ["closed", "open"] category: ["datacenter", "edge"] From f14ae8e20e90ff6230d76dbdd1d7172342dd793b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 16:40:59 +0530 Subject: [PATCH 421/658] Added test for testing all-variations (WIP) --- automation/script/module.py | 19 ++++++++++++++++++- script/get-generic-sys-util/_cm.json | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 183f7c48c0..d5388846f5 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2428,7 +2428,24 @@ def test(self, i): if console: logging.info(path) - logging.info(' Test: TBD') + test_config = meta.get('tests', '') + if test_config: + log.info(test_config) + test_all_variations = test_config.get('test-all-variations', False) + if test_all_variations: + variations = meta.get("variations") + individual_variations = [ v for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] + tags_string = ",".join(meta.get("tags")) + for variation in individual_variations: + run_tags = f"{tags_string},_{variation}" + r = self.cmind.access({'action':'run', + 'automation':'script', + 'tags': run_tags, + 'quiet': i.get('quiet') }) + if r['return'] > 0: + return r + + logging.info(' Test: WIP') return {'return':0, 'list': lst} diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a1f9d09918..e0db7c4944 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -721,5 +721,8 @@ } } } + }, + "tests": { + "test-all-variations": "yes" } } From 5a4771b5e77f23ed0cbd2fb6d1fce9c09f7521ed Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 11:13:12 +0000 Subject: [PATCH 422/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index b1cfb11d0a..0691a58604 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "25 21 * * *" #to be adjusted + - cron: "19 11 * * *" #to be adjusted jobs: build_nvidia: From 0aba91ee687e76997c6abb1fcc5a1e01f0068254 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:16:35 +0530 Subject: [PATCH 423/658] Added github action for individual CM script tests --- .../workflows/run-individual-script-tests.yml | 31 +++++++++++++++++++ tests/script/process_tests.py | 24 ++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 .github/workflows/run-individual-script-tests.yml create mode 100644 tests/script/process_tests.py diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml new file mode 100644 index 0000000000..8a4add3442 --- /dev/null +++ b/.github/workflows/run-individual-script-tests.yml @@ -0,0 +1,31 @@ +# This workflow will add/update the README.md files for any updated CM scripts +name: Readme update for CM scripts + +on: + pull_request: + branches: [ "main", "mlperf-inference", "dev" ] + paths: + - 'script/**_cm.json' + - 'script/**_cm.yml' + +jobs: + run-script-tests: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v3 + with: + fetch-depth: 2 + - name: Get changed files + id: getfile + run: | + echo "files=$(git diff --name-only ${{ github.event.before }} | xargs)" >> $GITHUB_OUTPUT + - name: RUN Script Tests + run: | + echo ${{ steps.getfile.outputs.files }} + for file in ${{ steps.getfile.outputs.files }}; do + echo $file + done + python3 -m pip install cmind + cm pull repo --url=https://github.com/${{ github.repository }} --checkout=${{ github.ref_name }} + python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py new file mode 100644 index 0000000000..386110edf4 --- /dev/null +++ b/tests/script/process_tests.py @@ -0,0 +1,24 @@ +import sys +import os +import cmind as cm +import check as checks +import json +import yaml + +files=sys.argv[1:] + +for file in files: + if not os.path.isfile(file): + continue + if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): + continue + script_path = os.path.dirname(file) + f = open(file) + if file.endswith(".json"): + data = json.load(f) + elif file.endswith(".yaml"): + data = yaml.safe_load(f) + uid = data['uid'] + + r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + checks.check_return(r) From c12942fd0a8a9dfa6e50cfd5fcc565d10e21705a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:20:45 +0530 Subject: [PATCH 424/658] Updated the source test repo --- .github/workflows/test-cm-based-ubmission-generation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-cm-based-ubmission-generation.yml b/.github/workflows/test-cm-based-ubmission-generation.yml index 2ae4e84250..4fc6c0bc45 100644 --- a/.github/workflows/test-cm-based-ubmission-generation.yml +++ b/.github/workflows/test-cm-based-ubmission-generation.yml @@ -1,4 +1,4 @@ -# This workflow will test the submission generation capability of CM for non-CM based benchmarks +# This workflow will test the submission generation capability of CM f name: CM based Submission Generation @@ -36,7 +36,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Pull repo where test cases are uploaded run: | - cm pull repo anandhu-eng@externalSubmissionPOC - - name: Submission generation - ${{ matrix.category }} ${{ matrix.division }} + cm pull repo anandhu-eng@inference --checkout=submission-generation-tests + - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From 00bfb46e3028c8e4e1cd30b30bd2cb5abd7dc942 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:25:33 +0530 Subject: [PATCH 425/658] Fix test name --- .github/workflows/run-individual-script-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index 8a4add3442..97d40d4154 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -1,5 +1,5 @@ -# This workflow will add/update the README.md files for any updated CM scripts -name: Readme update for CM scripts +# This workflow will run configured tests for any updated CM scripts +name: Individual CM script Tests on: pull_request: From f1001126b2cfe73b19f7b6b6ccc0ab1dd6160d71 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:29:38 +0530 Subject: [PATCH 426/658] Fix gh action for individual CM sript tests --- .github/workflows/run-individual-script-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index 97d40d4154..1ecd058af8 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -19,7 +19,9 @@ jobs: - name: Get changed files id: getfile run: | - echo "files=$(git diff --name-only ${{ github.event.before }} | xargs)" >> $GITHUB_OUTPUT + git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} + git fetch upstream + echo "files=$(git diff \"upstream\" \"${{ github.event.pull_request.base.ref }}\" --name-only | xargs)" >> $GITHUB_OUTPUT - name: RUN Script Tests run: | echo ${{ steps.getfile.outputs.files }} From e064bb9d256550c3859b7b86d4487064edcf7410 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:36:10 +0530 Subject: [PATCH 427/658] Fix gh action for individual CM sript tests --- .github/workflows/run-individual-script-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index 1ecd058af8..d8abb96d5c 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -13,15 +13,15 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - fetch-depth: 2 + fetch-depth: 0 - name: Get changed files id: getfile run: | git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} git fetch upstream - echo "files=$(git diff \"upstream\" \"${{ github.event.pull_request.base.ref }}\" --name-only | xargs)" >> $GITHUB_OUTPUT + echo "files=$(git diff upstream ${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT - name: RUN Script Tests run: | echo ${{ steps.getfile.outputs.files }} @@ -29,5 +29,5 @@ jobs: echo $file done python3 -m pip install cmind - cm pull repo --url=https://github.com/${{ github.repository }} --checkout=${{ github.ref_name }} + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} From e279a2faa59bea89ac1d553ab6f300af34aca911 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:39:06 +0530 Subject: [PATCH 428/658] Fix gh action for individual CM sript tests --- .github/workflows/run-individual-script-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index d8abb96d5c..d06696f746 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -21,7 +21,7 @@ jobs: run: | git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} git fetch upstream - echo "files=$(git diff upstream ${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT + echo "files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT - name: RUN Script Tests run: | echo ${{ steps.getfile.outputs.files }} From 566e103446d50c653d4c8fa399a53d80aadb56c6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:42:37 +0530 Subject: [PATCH 429/658] Fix gh action for individual CM sript tests --- .github/workflows/run-individual-script-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index d06696f746..04a709c974 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -15,7 +15,7 @@ jobs: - name: 'Checkout' uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 2 - name: Get changed files id: getfile run: | From e186af8d1a46a91bd7d0536b8988a6019dc694c4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:47:44 +0530 Subject: [PATCH 430/658] Added test case 7 --- ...> test-cm-based-submission-generation.yml} | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) rename .github/workflows/{test-cm-based-ubmission-generation.yml => test-cm-based-submission-generation.yml} (56%) diff --git a/.github/workflows/test-cm-based-ubmission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml similarity index 56% rename from .github/workflows/test-cm-based-ubmission-generation.yml rename to .github/workflows/test-cm-based-submission-generation.yml index 4fc6c0bc45..dcee3341dd 100644 --- a/.github/workflows/test-cm-based-ubmission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -40,3 +40,33 @@ jobs: - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} run: | cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + Case-7: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + division: ["closed", "open"] + category: ["datacenter", "edge"] + exclude: + - os: macos-latest + - os: windows-latest + - division: "open" + - category: "edge" + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Pull repo where test cases are uploaded + run: | + cm pull repo anandhu-eng@inference --checkout=submission-generation-tests + - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} + run: | + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-7/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From fccc9ca8e3846a1231b6f4254dc3f797c85a79ee Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:49:07 +0530 Subject: [PATCH 431/658] Updated test case description --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index dcee3341dd..c55525f62d 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -67,6 +67,6 @@ jobs: - name: Pull repo where test cases are uploaded run: | cm pull repo anandhu-eng@inference --checkout=submission-generation-tests - - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} + - name: Submission generation(sut_info.json is not completely filled but the SUT folder name is in required format(hardware_name-implementation-device-framework-run_config)) - ${{ matrix.category }} ${{ matrix.division }} run: | cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-7/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From 0694dce0a02b0c0c19668f953272eb2bebfd3b5e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 17:49:57 +0530 Subject: [PATCH 432/658] Fix gh action for individual CM sript tests --- automation/script/module.py | 3 +-- tests/script/process_tests.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index d5388846f5..7be9aa8554 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2425,12 +2425,11 @@ def test(self, i): alias = meta.get('alias','') uid = meta.get('uid','') - if console: logging.info(path) test_config = meta.get('tests', '') if test_config: - log.info(test_config) + logging.info(test_config) test_all_variations = test_config.get('test-all-variations', False) if test_all_variations: variations = meta.get("variations") diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 386110edf4..558c47a2ee 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -8,6 +8,7 @@ files=sys.argv[1:] for file in files: + print(file) if not os.path.isfile(file): continue if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): @@ -20,5 +21,5 @@ data = yaml.safe_load(f) uid = data['uid'] - r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con'}) checks.check_return(r) From 5ff6a68ba8c13fecc22d71ca6df43db935159c80 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:57:37 +0530 Subject: [PATCH 433/658] bug fix - test not running --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index c55525f62d..eeb2ed6968 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -6,7 +6,7 @@ on: pull_request: branches: [ "main", "dev", "mlperf-inference" ] paths: - - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' + - '.github/workflows/test-cm-based-submission-generation.yml' # - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also # - '!**.md' jobs: From cafec0bd36567983d55b471c7e7066530df50464 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 18:16:08 +0530 Subject: [PATCH 434/658] Enable docker run for individual CM script tests --- automation/script/module.py | 32 +++++++++++++++++++----- script/get-generic-sys-util/_cm.json | 6 ++++- script/run-docker-container/customize.py | 2 +- tests/script/process_tests.py | 2 +- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 7be9aa8554..ce509b75cc 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2431,18 +2431,38 @@ def test(self, i): if test_config: logging.info(test_config) test_all_variations = test_config.get('test-all-variations', False) + use_docker = test_config.get('use_docker', False) if test_all_variations: variations = meta.get("variations") individual_variations = [ v for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] tags_string = ",".join(meta.get("tags")) for variation in individual_variations: run_tags = f"{tags_string},_{variation}" - r = self.cmind.access({'action':'run', - 'automation':'script', - 'tags': run_tags, - 'quiet': i.get('quiet') }) - if r['return'] > 0: - return r + if use_docker: + docker_images = test_config.get('docker_images', [ "ubuntu-22.04" ]) + for docker_image in docker_images: + ii = {'action':'docker', + 'automation':'script', + 'tags': run_tags, + 'quiet': i.get('quiet'), + 'docker_image': docker_image, + 'docker_image_name': alias + } + if i.get('docker_cm_repo', '') != '': + ii['docker_cm_repo'] = i['docker_cm_repo'] + if i.get('docker_cm_repo_branch', '') != '': + ii['docker_cm_repo_branch'] = i['docker_cm_repo_branch'] + + r = self.cmind.access(ii) + if r['return'] > 0: + return r + else: + r = self.cmind.access({'action':'run', + 'automation':'script', + 'tags': run_tags, + 'quiet': i.get('quiet') }) + if r['return'] > 0: + return r logging.info(' Test: WIP') diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e0db7c4944..08df073dfa 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -723,6 +723,10 @@ } }, "tests": { - "test-all-variations": "yes" + "test-all-variations": "yes", + "use_docker": "yes", + "docker_images": [ + "ubuntu-22.04" + ] } } diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 35d3fdf1e4..db7ef766ab 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -303,7 +303,7 @@ def update_docker_info(env): if env.get('CM_DOCKER_IMAGE_NAME', '') != '': docker_image_name = env['CM_DOCKER_IMAGE_NAME'] else: - docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-') + docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-').replace('+','plus') env['CM_DOCKER_IMAGE_NAME'] = docker_image_name docker_image_tag_extra = env.get('CM_DOCKER_IMAGE_TAG_EXTRA', '-latest') diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 558c47a2ee..59c3b67694 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -9,7 +9,7 @@ for file in files: print(file) - if not os.path.isfile(file): + if not os.path.isfile(file) or not "script" in file: continue if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): continue From dd40f0a61224881934cd88989cc1719f7dec0496 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:21:37 +0530 Subject: [PATCH 435/658] capture framework version from cm_sut_info.json --- script/generate-mlperf-inference-submission/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 34d6c97b95..032a68618d 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -209,6 +209,7 @@ def generate_submission(i): implementation = sut_info["implementation"] device = sut_info["device"] framework = sut_info["framework"].replace(" ","_") + framework_version = sut_info["framework_version"] run_config = sut_info["run_config"] new_res = f"{system}-{implementation}-{device}-{framework}-{run_config}" else: @@ -234,7 +235,7 @@ def generate_submission(i): system_meta_default['framework'] = framework + " " + framework_version else: print(parts) - return {'return': 1, 'error': f"The required details for generating the inference submission:\n1.system_name\n2.implementation\n3.framework\n4.run_config\nInclude a cm-sut-info.json file with the above content in {result_path}"} + return {'return': 1, 'error': f"The required details for generating the inference submission:\n1.hardware_name\n2.implementation\n3.Device\n4.framework\n5.framework_version\n6.run_config\nInclude a cm-sut-info.json or sut-info.json file with the above content in {result_path}"} platform_prefix = inp.get('platform_prefix', '') if platform_prefix: From 9e3392bee828e1fc7803fb81be12f2aae6860a82 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:12:47 +0530 Subject: [PATCH 436/658] Enable default values to required env keys --- script/generate-mlperf-inference-submission/customize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 032a68618d..f82893e39a 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -15,7 +15,7 @@ def fill_from_json(file_path, keys, sut_info): with open(file_path, 'r') as f: data = json.load(f) for key in keys: - if key in data and sut_info[key] is None: + if key in data and (sut_info[key] is None or sut_info[key] == "default"): sut_info[key] = data[key] elif key in data and sut_info[key] != data[key]: return -1 # error saying there is a mismatch in the value of a key @@ -149,7 +149,8 @@ def generate_submission(i): "implementation": None, "device": None, "framework": None, - "run_config": None + "framework_version": "default", + "run_config": "default" } # variable to store the system meta model_mapping_combined = {} # to store all the model mapping related to an SUT From 7f58d670f7f6afae953d56b4649ff88afcc3663d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 19:17:51 +0530 Subject: [PATCH 437/658] Fix huggingface downloader - fixes #437 --- ...lperf-loadgen-onnx-huggingface-bert-fp32-squad.yml | 7 +++---- script/get-ml-model-huggingface-zoo/download_model.py | 11 ++++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml index ed73c7ad03..597121fb0d 100644 --- a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml +++ b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml @@ -5,7 +5,7 @@ name: MLPerf loadgen with HuggingFace bert onnx fp32 squad model on: pull_request: - branches: [ "main", "dev" ] + branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml' - '**' @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.12" ] + python-version: [ "3.10", "3.12" ] steps: - uses: actions/checkout@v3 @@ -30,7 +30,6 @@ jobs: run: | python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - cm run script --quiet --tags=get,sys-utils-cm - name: Test MLPerf loadgen with HuggingFace bert onnx fp32 squad model run: | - cmr "python app loadgen-generic _onnxruntime _custom _huggingface _model-stub.ctuning/mlperf-inference-bert-onnx-fp32-squad-v1.1" --adr.hf-downloader.model_filename=model.onnx --quiet + cm run script --tags=python,app,loadgen-generic,_onnxruntime,_custom,_huggingface,_model-stub.ctuning/mlperf-inference-bert-onnx-fp32-squad-v1.1 --quiet diff --git a/script/get-ml-model-huggingface-zoo/download_model.py b/script/get-ml-model-huggingface-zoo/download_model.py index 4e6e9c86e8..87f9b25aeb 100644 --- a/script/get-ml-model-huggingface-zoo/download_model.py +++ b/script/get-ml-model-huggingface-zoo/download_model.py @@ -28,8 +28,7 @@ model_filenames = model_filename.split(',') if ',' in model_filename else [model_filename] - # First must be model - base_model_filename = model_filenames[0] + base_model_filepath = None files = [] if full_subfolder!='': @@ -93,15 +92,17 @@ def list_hf_files(path): xrevision = None if revision == '' else revision xsubfolder = None if subfolder == '' else subfolder - hf_hub_download(repo_id=model_stub, + downloaded_path = hf_hub_download(repo_id=model_stub, subfolder=xsubfolder, filename=model_filename, - force_filename=model_filename, revision=xrevision, cache_dir=os.getcwd()) + print(downloaded_path) + if not base_model_filepath: + base_model_filepath = downloaded_path print ('') with open('tmp-run-env.out', 'w') as f: - f.write(f"CM_ML_MODEL_FILE_WITH_PATH={os.path.join(os.getcwd(),base_model_filename)}") + f.write(f"CM_ML_MODEL_FILE_WITH_PATH={base_model_filepath}") From a0a0df0007c3ff6f6bb6b862d1aa7298d6711dbb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 15:16:14 +0000 Subject: [PATCH 438/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 0691a58604..1a99e95b5a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "19 11 * * *" #to be adjusted + - cron: "22 15 * * *" #to be adjusted jobs: build_nvidia: From abbbbac00649cad8d3828d41af9ceaa01629b661 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 21:42:23 +0530 Subject: [PATCH 439/658] Support docker repo/branch for cm test --- automation/script/module.py | 10 ++++++---- script/get-generic-sys-util/_cm.json | 5 ++++- script/run-docker-container/_cm.yaml | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index ce509b75cc..7316328b9f 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2439,18 +2439,20 @@ def test(self, i): for variation in individual_variations: run_tags = f"{tags_string},_{variation}" if use_docker: - docker_images = test_config.get('docker_images', [ "ubuntu-22.04" ]) + docker_images = i.get("docker_images", test_config.get('docker_images', [ "ubuntu-22.04" ])) for docker_image in docker_images: ii = {'action':'docker', 'automation':'script', 'tags': run_tags, 'quiet': i.get('quiet'), - 'docker_image': docker_image, + 'docker_base_image': docker_image, 'docker_image_name': alias } - if i.get('docker_cm_repo', '') != '': + docker_cm_repo = i.get('docker_cm_repo', test_config.get('docker_cm_repo', '')) + docker_cm_repo_branch = i.get('docker_cm_repo_branch', test_config.get('docker_cm_repo_branch', '')) + if docker_cm_repo != '': ii['docker_cm_repo'] = i['docker_cm_repo'] - if i.get('docker_cm_repo_branch', '') != '': + if docker_cm_repo_branch != '': ii['docker_cm_repo_branch'] = i['docker_cm_repo_branch'] r = self.cmind.access(ii) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 08df073dfa..cdbf2f5b7a 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -726,7 +726,10 @@ "test-all-variations": "yes", "use_docker": "yes", "docker_images": [ - "ubuntu-22.04" + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", + "registry.access.redhat.com/ubi9" ] } } diff --git a/script/run-docker-container/_cm.yaml b/script/run-docker-container/_cm.yaml index 38a8c794bd..5a3d2f48c5 100644 --- a/script/run-docker-container/_cm.yaml +++ b/script/run-docker-container/_cm.yaml @@ -26,6 +26,7 @@ input_mapping: detached: CM_DOCKER_DETACHED_MODE device: CM_DOCKER_ADD_DEVICE docker_image_base: CM_DOCKER_IMAGE_BASE + docker_base_image: CM_DOCKER_IMAGE_BASE keep_detached: CM_KEEP_DETACHED_CONTAINER docker_os: CM_DOCKER_OS docker_os_version: CM_DOCKER_OS_VERSION From b1fef06ac04452388b49b52357238f7a81f119d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 21:53:32 +0530 Subject: [PATCH 440/658] Fix docker_base_image for CM tests --- .github/workflows/run-individual-script-tests.yml | 2 +- tests/script/process_tests.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index 04a709c974..963d13a57a 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -30,4 +30,4 @@ jobs: done python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} + DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref } python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 59c3b67694..6069a6a9cd 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -19,7 +19,18 @@ data = json.load(f) elif file.endswith(".yaml"): data = yaml.safe_load(f) + if data.get('uid', '') == '': + continue #not a CM script meta uid = data['uid'] - r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con'}) + ii = { + 'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con' + } + if os.environ.get('CM_DOCKER_REPO', '') != '': + ii['cm_docker_repo'] = env['CM_DOCKER_REPO'] + if os.environ.get('CM_DOCKER_REPO_BRANCH', '') != '': + ii['cm_docker_repo_branch'] = env['CM_DOCKER_REPO_BRANCH'] + + r = cm.access(ii) + checks.check_return(r) From 576691bcde94b6741538e0eb3d4e496b89f0fc64 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 21:59:10 +0530 Subject: [PATCH 441/658] Fix docker_base_image for CM tests --- script/get-generic-sys-util/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/run.sh index 5074e756b8..7e8afaf10a 100644 --- a/script/get-generic-sys-util/run.sh +++ b/script/get-generic-sys-util/run.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Safe execution of a command stored in a variable cmd="${CM_SYS_UTIL_INSTALL_CMD}" echo "$cmd" From c151278e1917feee53619c1e9ec953e589c0f5b1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 22:22:39 +0530 Subject: [PATCH 442/658] Fix passing of CM_TMP_FAIL_SAFE in get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index cdbf2f5b7a..161bc1bfe8 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -40,6 +40,9 @@ "prehook_deps": [ { "tags": "get,generic-sys-util,_install", + "force_env_keys": [ + "CM_TMP_FAIL_SAFE" + ], "inherit_variation_tags": true, "names": [ "install-sys-util" From f3106ddf234ea857ea8130aa9b5fe71f7ba1f8d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 22:23:26 +0530 Subject: [PATCH 443/658] Fix typo in GH action --- .github/workflows/run-individual-script-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index 963d13a57a..ab2f5b3abe 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -30,4 +30,4 @@ jobs: done python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref } python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} + DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} From 57d7e71ce35429db1d6a3d9e0d5447d6aa58db78 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 16:54:07 +0000 Subject: [PATCH 444/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 1a99e95b5a..cb0b549780 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "22 15 * * *" #to be adjusted + - cron: "59 16 * * *" #to be adjusted jobs: build_nvidia: From 9f8ea72b375ed669be6c85d2013262bd59ff7877 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 22:28:51 +0530 Subject: [PATCH 445/658] Fix typo in CM test --- script/get-generic-sys-util/_cm.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 161bc1bfe8..cc22ccf3d0 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -729,9 +729,9 @@ "test-all-variations": "yes", "use_docker": "yes", "docker_images": [ - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu:20.04", + "ubuntu:22.04", + "ubuntu:24.04", "registry.access.redhat.com/ubi9" ] } From e9fc19791f6a2bb340997ac1b9f624f90440fca7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 23:12:50 +0530 Subject: [PATCH 446/658] clone command changed + clones to custom folder --- .github/workflows/test-cm-based-submission-generation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index eeb2ed6968..7b7d138ae6 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -36,10 +36,10 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Pull repo where test cases are uploaded run: | - cm pull repo anandhu-eng@inference --checkout=submission-generation-tests + git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet Case-7: runs-on: ${{ matrix.os }} strategy: @@ -66,7 +66,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Pull repo where test cases are uploaded run: | - cm pull repo anandhu-eng@inference --checkout=submission-generation-tests + git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests - name: Submission generation(sut_info.json is not completely filled but the SUT folder name is in required format(hardware_name-implementation-device-framework-run_config)) - ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-7/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/case-7/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From a11b0b6880a61ef60840c8dad502f783e4679a3f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 00:08:52 +0530 Subject: [PATCH 447/658] Improved docker input meta for cm test script --- automation/script/module.py | 56 +++++++++++++----------- script/build-dockerfile/customize.py | 2 +- script/get-generic-sys-util/_cm.json | 25 ++++++++--- script/run-docker-container/customize.py | 6 ++- 4 files changed, 56 insertions(+), 33 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 7316328b9f..537ae3ee60 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2436,33 +2436,37 @@ def test(self, i): variations = meta.get("variations") individual_variations = [ v for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] tags_string = ",".join(meta.get("tags")) - for variation in individual_variations: - run_tags = f"{tags_string},_{variation}" + run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) + ii = {'action': 'run', + 'automation':'script', + 'quiet': i.get('quiet'), + 'env': i.get('env') + } + i_env = i.get('env') + for run_input in run_inputs: + for key in run_input:#override meta with any user inputs like for docker_cm_repo + if i.get(key, '') != '': + if type(run_input[key]) == dict: + utils.merge_dicts({'dict1': run_input[key] , 'dict2':i[key], 'append_lists':True, 'append_unique':True}) + else: + run_input[key] = i[key] + ii = {**ii, **run_input} if use_docker: - docker_images = i.get("docker_images", test_config.get('docker_images', [ "ubuntu-22.04" ])) - for docker_image in docker_images: - ii = {'action':'docker', - 'automation':'script', - 'tags': run_tags, - 'quiet': i.get('quiet'), - 'docker_base_image': docker_image, - 'docker_image_name': alias - } - docker_cm_repo = i.get('docker_cm_repo', test_config.get('docker_cm_repo', '')) - docker_cm_repo_branch = i.get('docker_cm_repo_branch', test_config.get('docker_cm_repo_branch', '')) - if docker_cm_repo != '': - ii['docker_cm_repo'] = i['docker_cm_repo'] - if docker_cm_repo_branch != '': - ii['docker_cm_repo_branch'] = i['docker_cm_repo_branch'] - - r = self.cmind.access(ii) - if r['return'] > 0: - return r - else: - r = self.cmind.access({'action':'run', - 'automation':'script', - 'tags': run_tags, - 'quiet': i.get('quiet') }) + ii['action'] = "docker" + for key in i: + if key.startswith("docker_"): + ii[key] = i[key] + + if ii.get('docker_image_name', '') == '': + ii['docker_image_name'] = alias + + for variation in individual_variations: + run_tags = f"{tags_string},_{variation}" + ii['tags'] = run_tags + if i_env: + import copy + ii['env'] = copy.deepcopy(i_env) + r = self.cmind.access(ii) if r['return'] > 0: return r diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 2a8c96b635..0534da4bf1 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -10,7 +10,7 @@ def preprocess(i): env = i['env'] if env["CM_DOCKER_OS"] not in [ "ubuntu", "rhel", "arch" ]: - return {'return': 1, 'error': "Currently only ubuntu, rhel and arch are supported in CM docker"} + return {'return': 1, 'error': f"Specified docker OS: {env['CM_DOCKER_OS']}. Currently only ubuntu, rhel and arch are supported in CM docker"} path = i['run_script_input']['path'] diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index cc22ccf3d0..fc54ce9254 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -728,11 +728,26 @@ "tests": { "test-all-variations": "yes", "use_docker": "yes", - "docker_images": [ - "ubuntu:20.04", - "ubuntu:22.04", - "ubuntu:24.04", - "registry.access.redhat.com/ubi9" + "run_inputs": [ + { + "docker_os": "rhel", + "docker_os_version": "9", + "env": { + "CM_TMP_FAIL_SAFE": "yes" + } + }, + { + "docker_os": "ubuntu", + "docker_os_version": "20.04" + }, + { + "docker_os": "ubuntu", + "docker_os_version": "22.04" + }, + { + "docker_os": "ubuntu", + "docker_os_version": "24.04" + } ] } } diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index db7ef766ab..5edfe7ecad 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -220,6 +220,8 @@ def postprocess(i): print ('') docker_out = subprocess.check_output(CMD, shell=True).decode("utf-8") + if docker_out != 0: + return {'return': docker_out, 'error': 'docker run failed'} lines = docker_out.split("\n") @@ -256,6 +258,8 @@ def postprocess(i): print ('') docker_out = os.system(CMD) + if docker_out != 0: + return {'return': docker_out, 'error': 'docker run failed'} return {'return':0} @@ -308,7 +312,7 @@ def update_docker_info(env): docker_image_tag_extra = env.get('CM_DOCKER_IMAGE_TAG_EXTRA', '-latest') - docker_image_tag = env.get('CM_DOCKER_IMAGE_TAG', docker_image_base.replace(':','-').replace('_','') + docker_image_tag_extra) + docker_image_tag = env.get('CM_DOCKER_IMAGE_TAG', docker_image_base.replace(':','-').replace('_','').replace("/","-") + docker_image_tag_extra) env['CM_DOCKER_IMAGE_TAG'] = docker_image_tag return From 106234c51b062c3b887eb0a7592148dabc103a49 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 00:15:39 +0530 Subject: [PATCH 448/658] Fix CM_MLPERF_DEVICE for nvidia implementation in process-accuracy --- script/process-mlperf-accuracy/_cm.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index efbf9b901e..2d24cc9261 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -288,7 +288,8 @@ "tags": "get,generic-python-lib,_torch", "skip_if_env": { "CM_MLPERF_DEVICE": [ - "cuda" + "cuda", + "gpu" ] } }, @@ -296,7 +297,8 @@ "tags": "get,generic-python-lib,_torch_cuda", "enable_if_env": { "CM_MLPERF_DEVICE": [ - "cuda" + "cuda", + "gpu" ] } }, From 2458da2c1540fcc127889803b5615542b3708674 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 00:29:41 +0530 Subject: [PATCH 449/658] Fix env corruption in cm test script --- automation/script/module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 537ae3ee60..20b0a46726 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2442,7 +2442,6 @@ def test(self, i): 'quiet': i.get('quiet'), 'env': i.get('env') } - i_env = i.get('env') for run_input in run_inputs: for key in run_input:#override meta with any user inputs like for docker_cm_repo if i.get(key, '') != '': @@ -2451,6 +2450,7 @@ def test(self, i): else: run_input[key] = i[key] ii = {**ii, **run_input} + i_env = ii.get('env') if use_docker: ii['action'] = "docker" for key in i: @@ -2466,6 +2466,7 @@ def test(self, i): if i_env: import copy ii['env'] = copy.deepcopy(i_env) + logging.info(ii) r = self.cmind.access(ii) if r['return'] > 0: return r From 091094c6fdcdea444659d321777371305729346b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Oct 2024 19:19:52 +0000 Subject: [PATCH 450/658] Update customize.py --- script/run-docker-container/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 5edfe7ecad..00ab63c978 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -220,8 +220,8 @@ def postprocess(i): print ('') docker_out = subprocess.check_output(CMD, shell=True).decode("utf-8") - if docker_out != 0: - return {'return': docker_out, 'error': 'docker run failed'} + #if docker_out != 0: + # return {'return': docker_out, 'error': 'docker run failed'} lines = docker_out.split("\n") From b3ca14fc86397cc2cb142c677fb6a558f90bb2d0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 03:17:33 +0530 Subject: [PATCH 451/658] Fix cm4mlops repo in test --- script/test-cm-core/src/script/test_docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/test-cm-core/src/script/test_docker.py b/script/test-cm-core/src/script/test_docker.py index 0ec418f214..9473997ad5 100644 --- a/script/test-cm-core/src/script/test_docker.py +++ b/script/test-cm-core/src/script/test_docker.py @@ -9,7 +9,7 @@ 'add_deps_recursive': { 'compiler': {'tags': "gcc"} }, - 'docker_cm_repo': 'mlcommons@ck', + 'docker_cm_repo': 'mlcommons@cm4mlops', 'image_name':'cm-script-app-image-classification-onnx-py', 'env': { 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', @@ -26,7 +26,7 @@ 'add_deps_recursive': { 'compiler': {'tags': "gcc"} }, - 'docker_cm_repo': 'mlcommons@ck', + 'docker_cm_repo': 'mlcommons@cm4mlops', 'image_name':'cm-script-app-image-classification-onnx-py', 'env': { 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', From d844d344de21a237e8d4f9da92b5f6e0fab50c79 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 03:41:37 +0530 Subject: [PATCH 452/658] Generalize CM test script meta --- automation/script/module.py | 27 ++++++++++++++++++------ script/get-generic-sys-util/_cm.json | 25 ++++++++++++++++------ script/get-generic-sys-util/customize.py | 6 +++++- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 20b0a46726..41a2e5b063 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2430,11 +2430,8 @@ def test(self, i): test_config = meta.get('tests', '') if test_config: logging.info(test_config) - test_all_variations = test_config.get('test-all-variations', False) - use_docker = test_config.get('use_docker', False) + variations = meta.get("variations") if test_all_variations: - variations = meta.get("variations") - individual_variations = [ v for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] tags_string = ",".join(meta.get("tags")) run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) ii = {'action': 'run', @@ -2443,6 +2440,24 @@ def test(self, i): 'env': i.get('env') } for run_input in run_inputs: + test_all_variations = run_input.get('test-all-variations', False) + if test_all_variations: + run_variations = [ f"_{v}" for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] + else: + given_variations = run_input.get('variations_list', []) + if given_variations: + v_split = [] + run_variations = [] + for i, v in enumerate(given_variations): + v_split = v.split(",") + for t in v_split: + if not t.startswith("_"): + given_variations[i] = f"_{t}" #variations must begin with _. We support both with and without _ in the meta + if v_split: + run_variations.append(",".join(v_split)) + else: + run_variations = [ "" ] #run the test without any variations + use_docker = run_input.get('docker', False) for key in run_input:#override meta with any user inputs like for docker_cm_repo if i.get(key, '') != '': if type(run_input[key]) == dict: @@ -2460,8 +2475,8 @@ def test(self, i): if ii.get('docker_image_name', '') == '': ii['docker_image_name'] = alias - for variation in individual_variations: - run_tags = f"{tags_string},_{variation}" + for variation_tags in run_variations: + run_tags = f"{tags_string},{variation_tags}" ii['tags'] = run_tags if i_env: import copy diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index fc54ce9254..4ed05dfbe1 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -20,6 +20,9 @@ "CM_GENERIC_SYS_UTIL_INSTALL_NEEDED": "no", "CM_SYS_UTIL_VERSION_CMD": "" }, + "input_mapping": { + "ignore_missing": "CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE" + }, "tags": [ "get", "sys-util", @@ -726,27 +729,37 @@ } }, "tests": { - "test-all-variations": "yes", - "use_docker": "yes", "run_inputs": [ { + "docker": "yes", "docker_os": "rhel", "docker_os_version": "9", + "test-all-variations": "yes", "env": { "CM_TMP_FAIL_SAFE": "yes" - } + }, + "ignore_missing": "yes" }, { + "docker": "yes", "docker_os": "ubuntu", - "docker_os_version": "20.04" + "docker_os_version": "20.04", + "test-all-variations": "yes", + "ignore_missing": "yes" }, { + "docker": "yes", "docker_os": "ubuntu", - "docker_os_version": "22.04" + "docker_os_version": "22.04", + "test-all-variations": "yes", + "ignore_missing": "yes" }, { + "docker": "yes", "docker_os": "ubuntu", - "docker_os_version": "24.04" + "docker_os_version": "24.04", + "test-all-variations": "yes", + "ignore_missing": "yes" } ] } diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index d74fc58669..c163e50122 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -53,7 +53,11 @@ def preprocess(i): return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} if not package_name: - return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} + if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '').lower() in ["1", "true", "yes"]: + print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...") + return {'return': 0} + else: + return {'return': 1, 'error': f'No package name specified for {pm} and util name {util}'} if util == "libffi": if env.get("CM_HOST_OS_FLAVOR", "") == "ubuntu": From 159e82d1101e17639822e7ba28ef6260fe267c17 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 03:45:12 +0530 Subject: [PATCH 453/658] Generalize CM test script meta --- automation/script/module.py | 97 ++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 41a2e5b063..6a7c45f629 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2431,62 +2431,59 @@ def test(self, i): if test_config: logging.info(test_config) variations = meta.get("variations") - if test_all_variations: - tags_string = ",".join(meta.get("tags")) - run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) - ii = {'action': 'run', + tags_string = ",".join(meta.get("tags")) + run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) + ii = {'action': 'run', 'automation':'script', 'quiet': i.get('quiet'), 'env': i.get('env') } - for run_input in run_inputs: - test_all_variations = run_input.get('test-all-variations', False) - if test_all_variations: - run_variations = [ f"_{v}" for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] + for run_input in run_inputs: + test_all_variations = run_input.get('test-all-variations', False) + if test_all_variations: + run_variations = [ f"_{v}" for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] + else: + given_variations = run_input.get('variations_list', []) + if given_variations: + v_split = [] + run_variations = [] + for i, v in enumerate(given_variations): + v_split = v.split(",") + for t in v_split: + if not t.startswith("_"): + given_variations[i] = f"_{t}" #variations must begin with _. We support both with and without _ in the meta + if v_split: + run_variations.append(",".join(v_split)) else: - given_variations = run_input.get('variations_list', []) - if given_variations: - v_split = [] - run_variations = [] - for i, v in enumerate(given_variations): - v_split = v.split(",") - for t in v_split: - if not t.startswith("_"): - given_variations[i] = f"_{t}" #variations must begin with _. We support both with and without _ in the meta - if v_split: - run_variations.append(",".join(v_split)) + run_variations = [ "" ] #run the test without any variations + use_docker = run_input.get('docker', False) + for key in run_input:#override meta with any user inputs like for docker_cm_repo + if i.get(key, '') != '': + if type(run_input[key]) == dict: + utils.merge_dicts({'dict1': run_input[key] , 'dict2':i[key], 'append_lists':True, 'append_unique':True}) else: - run_variations = [ "" ] #run the test without any variations - use_docker = run_input.get('docker', False) - for key in run_input:#override meta with any user inputs like for docker_cm_repo - if i.get(key, '') != '': - if type(run_input[key]) == dict: - utils.merge_dicts({'dict1': run_input[key] , 'dict2':i[key], 'append_lists':True, 'append_unique':True}) - else: - run_input[key] = i[key] - ii = {**ii, **run_input} - i_env = ii.get('env') - if use_docker: - ii['action'] = "docker" - for key in i: - if key.startswith("docker_"): - ii[key] = i[key] - - if ii.get('docker_image_name', '') == '': - ii['docker_image_name'] = alias - - for variation_tags in run_variations: - run_tags = f"{tags_string},{variation_tags}" - ii['tags'] = run_tags - if i_env: - import copy - ii['env'] = copy.deepcopy(i_env) - logging.info(ii) - r = self.cmind.access(ii) - if r['return'] > 0: - return r - - logging.info(' Test: WIP') + run_input[key] = i[key] + ii = {**ii, **run_input} + i_env = ii.get('env') + if use_docker: + ii['action'] = "docker" + for key in i: + if key.startswith("docker_"): + ii[key] = i[key] + + if ii.get('docker_image_name', '') == '': + ii['docker_image_name'] = alias + + for variation_tags in run_variations: + run_tags = f"{tags_string},{variation_tags}" + ii['tags'] = run_tags + if i_env: + import copy + ii['env'] = copy.deepcopy(i_env) + logging.info(ii) + r = self.cmind.access(ii) + if r['return'] > 0: + return r return {'return':0, 'list': lst} From d2ff8dd9d2068fc0fba863c07d16114192c423fb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 03:47:02 +0530 Subject: [PATCH 454/658] Generalize CM test script meta --- script/get-generic-sys-util/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index c163e50122..32cbf37506 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -53,7 +53,7 @@ def preprocess(i): return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} if not package_name: - if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '').lower() in ["1", "true", "yes"]: + if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in ["1", "true", "yes"]: print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...") return {'return': 0} else: From c79b51847bd0c6b9c0001a59d188d19ad559f423 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 03:55:20 +0530 Subject: [PATCH 455/658] Fix typo in CM test --- tests/script/process_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 6069a6a9cd..895b7709b1 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -26,11 +26,11 @@ ii = { 'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con' } - if os.environ.get('CM_DOCKER_REPO', '') != '': - ii['cm_docker_repo'] = env['CM_DOCKER_REPO'] - if os.environ.get('CM_DOCKER_REPO_BRANCH', '') != '': - ii['cm_docker_repo_branch'] = env['CM_DOCKER_REPO_BRANCH'] - + if os.environ.get('DOCKER_CM_REPO', '') != '': + ii['docker_cm_repo'] = env['DOCKER_CM_REPO'] + if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '': + ii['docker_cm_repo_branch'] = env['DOCKER_CM_REPO_BRANCH'] + print(ii) r = cm.access(ii) checks.check_return(r) From a166057fa2be2070ea384682efe9bde6a1b3c918 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 03:56:38 +0530 Subject: [PATCH 456/658] Fix typo in CM test --- tests/script/process_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 895b7709b1..99a0ff7ca5 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -27,9 +27,9 @@ 'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con' } if os.environ.get('DOCKER_CM_REPO', '') != '': - ii['docker_cm_repo'] = env['DOCKER_CM_REPO'] + ii['docker_cm_repo'] = os.environ['DOCKER_CM_REPO'] if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '': - ii['docker_cm_repo_branch'] = env['DOCKER_CM_REPO_BRANCH'] + ii['docker_cm_repo_branch'] = os.environ['DOCKER_CM_REPO_BRANCH'] print(ii) r = cm.access(ii) From 0ddc2aae6f8bf35610c027ef72a8b3061579cf46 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 04:03:28 +0530 Subject: [PATCH 457/658] Fix regular expression to convert git repo url to repo@user format --- script/build-dockerfile/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 0534da4bf1..69a603b64f 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -56,7 +56,7 @@ def preprocess(i): if env.get("CM_MLOPS_REPO", "") != "": cm_mlops_repo = env["CM_MLOPS_REPO"] # the below pattern matches both the HTTPS and SSH git link formats - git_link_pattern = r'^(https?://github\.com/([^/]+)/([^/]+)\.git|git@github\.com:([^/]+)/([^/]+)\.git)$' + git_link_pattern = r'^(https?://github\.com/([^/]+)/([^/]+)(?:\.git)?|git@github\.com:([^/]+)/([^/]+)(?:\.git)?)$' if match := re.match(git_link_pattern, cm_mlops_repo): if match.group(2) and match.group(3): repo_owner = match.group(2) From da5a24f3a231a4a63bc38c7e5200eae3827ee880 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 04:11:47 +0530 Subject: [PATCH 458/658] Fix libffi meta --- script/get-generic-sys-util/_cm.json | 2 +- script/get-generic-sys-util/customize.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 4ed05dfbe1..76472e9c11 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -229,7 +229,7 @@ "CM_SYS_UTIL_NAME": "libffi" }, "state": { - "libffi7": { + "libffi": { "apt": "libffi" } } diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 32cbf37506..4996459351 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -50,10 +50,10 @@ def preprocess(i): if not package: - return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)} + return {'return': 1, 'error': f'No package name specified for {util} in the meta'} if not package_name: - if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in ["1", "true", "yes"]: + if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in [ "1", "true", "yes" ]: print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...") return {'return': 0} else: From 95618aa6171c42880b403fb360f9d028ddf31497 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 04:22:14 +0530 Subject: [PATCH 459/658] Fix env corruption in CM script test --- automation/script/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 6a7c45f629..1156339e6e 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2433,12 +2433,12 @@ def test(self, i): variations = meta.get("variations") tags_string = ",".join(meta.get("tags")) run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) - ii = {'action': 'run', + for run_input in run_inputs: + ii = {'action': 'run', 'automation':'script', 'quiet': i.get('quiet'), 'env': i.get('env') } - for run_input in run_inputs: test_all_variations = run_input.get('test-all-variations', False) if test_all_variations: run_variations = [ f"_{v}" for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ] From 7140915f55f8d05fbade243929afd36e85a367d9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 04:28:06 +0530 Subject: [PATCH 460/658] Fix env corruption in CM script test --- automation/script/module.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 1156339e6e..e383653d38 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2436,8 +2436,7 @@ def test(self, i): for run_input in run_inputs: ii = {'action': 'run', 'automation':'script', - 'quiet': i.get('quiet'), - 'env': i.get('env') + 'quiet': i.get('quiet'), } test_all_variations = run_input.get('test-all-variations', False) if test_all_variations: @@ -2464,7 +2463,7 @@ def test(self, i): else: run_input[key] = i[key] ii = {**ii, **run_input} - i_env = ii.get('env') + i_env = ii.get('env', i.get('env', {})) if use_docker: ii['action'] = "docker" for key in i: From 59f7addcbd449d955d1db2bc768a8fa65a69cabd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 04:35:17 +0530 Subject: [PATCH 461/658] Added fail_safe option for Ubuntu sys-util tests --- script/get-generic-sys-util/_cm.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 76472e9c11..c2731d15ab 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -21,7 +21,8 @@ "CM_SYS_UTIL_VERSION_CMD": "" }, "input_mapping": { - "ignore_missing": "CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE" + "ignore_missing": "CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE", + "fail_safe": "CM_TMP_FAIL_SAFE" }, "tags": [ "get", @@ -745,21 +746,24 @@ "docker_os": "ubuntu", "docker_os_version": "20.04", "test-all-variations": "yes", - "ignore_missing": "yes" + "ignore_missing": "yes", + "fail_safe": "yes" }, { "docker": "yes", "docker_os": "ubuntu", "docker_os_version": "22.04", "test-all-variations": "yes", - "ignore_missing": "yes" + "ignore_missing": "yes", + "fail_safe": "yes" }, { "docker": "yes", "docker_os": "ubuntu", "docker_os_version": "24.04", "test-all-variations": "yes", - "ignore_missing": "yes" + "ignore_missing": "yes", + "fail_safe": "yes" } ] } From 944d6a7659de0817f88a5682304a3eb1b0fc4b00 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 05:13:36 +0530 Subject: [PATCH 462/658] Added wheel deps for cm dockerfile --- script/build-dockerfile/dockerinfo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-dockerfile/dockerinfo.json b/script/build-dockerfile/dockerinfo.json index adc7cf269b..df9c6c90a7 100644 --- a/script/build-dockerfile/dockerinfo.json +++ b/script/build-dockerfile/dockerinfo.json @@ -1,6 +1,6 @@ { "python-packages": [ - "cmind", "requests", "giturlparse", "tabulate" + "wheel", "cmind", "requests", "giturlparse", "tabulate" ], "ARGS": [ "CM_GH_TOKEN" From f34ef4b0105a72b5f48446f9616d9aaccf8e3e88 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 00:19:32 +0000 Subject: [PATCH 463/658] Update build_wheel.yml | Use ssh key --- .github/workflows/build_wheel.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 049b498403..496aeda757 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -27,7 +27,10 @@ jobs: fail-fast: false steps: # Step 1: Checkout the code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + ssh-key: ${{ secrets.DEPLOY_KEY }} # Step 2: Set up Python - uses: actions/setup-python@v3 From 667f6fec735de33ec499ffa6bdb0e2c3112875bf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 00:54:55 +0000 Subject: [PATCH 464/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index cb0b549780..70bf00ccaf 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "59 16 * * *" #to be adjusted + - cron: "01 01 * * *" #to be adjusted jobs: build_nvidia: From 172e6d818ecd662deb9696af4e54337594c9e676 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 01:50:04 +0000 Subject: [PATCH 465/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 70bf00ccaf..cea871153c 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "01 01 * * *" #to be adjusted + - cron: "01 02 * * *" #to be adjusted jobs: build_nvidia: From 13c177a6fa78d98c9a70026d32bff73694efa395 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 02:22:52 +0000 Subject: [PATCH 466/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index cea871153c..032cb4624a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "01 02 * * *" #to be adjusted + - cron: "31 2 * * *" #to be adjusted jobs: build_nvidia: From df77b511db1b6988923592a386085d7b47d864b2 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:17:46 +0530 Subject: [PATCH 467/658] Test commit - Dynamic step name --- .../test-cm-based-submission-generation.yml | 48 ++++++------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 7b7d138ae6..7eb6b3df24 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -10,7 +10,7 @@ on: # - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also # - '!**.md' jobs: - Case-3: + submission_generation: runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -19,6 +19,7 @@ jobs: python-version: [ "3.12" ] division: ["closed", "open"] category: ["datacenter", "edge"] + case: ["case-3", "case-7"] exclude: - os: macos-latest - os: windows-latest @@ -37,36 +38,17 @@ jobs: - name: Pull repo where test cases are uploaded run: | git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests - - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} + - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet - Case-7: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [ "3.12" ] - division: ["closed", "open"] - category: ["datacenter", "edge"] - exclude: - - os: macos-latest - - os: windows-latest - - division: "open" - - category: "edge" - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - - name: Pull repo where test cases are uploaded - run: | - git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests - - name: Submission generation(sut_info.json is not completely filled but the SUT folder name is in required format(hardware_name-implementation-device-framework-run_config)) - ${{ matrix.category }} ${{ matrix.division }} - run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/case-7/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + if [ "${{ matrix.case }}" == "case-3" ]; then + #results_dir="submission_generation_tests/case-3/" + description="Submission generation (model_mapping.json not present but model name matches with official one)" + elif [ "${{ matrix.case }}" == "case-7" ]; then + #results_dir="submission_generation_tests/case-7/" + description="Submission generation (sut_info.json incomplete, SUT folder name in required format)" + fi + # Dynamically set the log group to simulate a dynamic step name + echo "::group::$description" + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + echo "::endgroup::" + From c59608eb4e42410f5eeffd3630af110749659b00 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:07:08 +0530 Subject: [PATCH 468/658] enable docker-submission generation --- .../_cm.json | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 3827af24a4..cd82a9d422 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -120,5 +120,40 @@ "mlperf-inference-submission", "mlcommons-inference-submission" ], - "uid": "5f8ab2d0b5874d53" + "uid": "5f8ab2d0b5874d53", + "docker": { + "use_host_group_id": true, + "use_host_user_id": true, + "deps": [ + { + "tags": "get,mlperf,inference,results,dir,local", + "names": "get-mlperf-inference-results-dir", + "skip_if_env": { + "OUTPUT_BASE_DIR": [ + "on" + ] + } + }, + { + "tags": "get,mlperf,inference,submission,dir,local", + "names": "get-mlperf-inference-submission-dir", + "skip_if_env": { + "CM_MLPERF_INFERENCE_SUBMISSION_DIR": [ + "on" + ] + } + } + ], + "pre_run_cmds": [ + "cm pull repo" + ], + "mounts": [ + "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", + "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" + ], + "os": "ubuntu", + "cm_repo": "mlcommons@cm4mlops", + "cm_repo_branch": "mlperf-inference", + "os_version": "22.04" + } } From 28ce998f3292d497bb5481f3c2b4fd4638d9c164 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:19:22 +0530 Subject: [PATCH 469/658] real run set to false --- script/generate-mlperf-inference-submission/_cm.json | 1 + 1 file changed, 1 insertion(+) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index cd82a9d422..d6acd726b9 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -124,6 +124,7 @@ "docker": { "use_host_group_id": true, "use_host_user_id": true, + "real_run": false, "deps": [ { "tags": "get,mlperf,inference,results,dir,local", From f467f5c99848b221dcf312d5832ce9b6a41c0995 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 31 Oct 2024 17:24:34 +0530 Subject: [PATCH 470/658] Support test_input_id and test_input_index in CM tests --- .../workflows/run-individual-script-tests.yml | 6 ++++- automation/script/module.py | 26 +++++++++++++++++++ tests/script/process_tests.py | 2 ++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index ab2f5b3abe..e6c3db4b5a 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -11,6 +11,10 @@ on: jobs: run-script-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-input-index: [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11+" ] steps: - name: 'Checkout' uses: actions/checkout@v4 @@ -30,4 +34,4 @@ jobs: done python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} + DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} diff --git a/automation/script/module.py b/automation/script/module.py index e383653d38..64226f5742 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2432,8 +2432,34 @@ def test(self, i): logging.info(test_config) variations = meta.get("variations") tags_string = ",".join(meta.get("tags")) + test_input_index = i.get('test_input_index') + test_input_id = i.get('test_input_id') run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) + if test_input_index: + index_plus = False + try: + if test_input_index.endswith("+"): + input_index = int(test_input_index[:-1]) + index_plus = True + else: + input_index = int(test_input_index) + except ValueError as e: + print(e) + return {'return': 1, 'error': f'Invalid test_input_index: {test_input_index}. Must be an integer or an integer followed by a +'} + if input_index > len(run_inputs): + run_inputs = [] + else: + if index_plus: + run_inputs = run_inputs[index_index-1:] + else: + run_inputs = [ run_inputs[input_index - 1] ] + for run_input in run_inputs: + if test_input_id: + if run_input.get('id', '') != test_input_id: + continue + + ii = {'action': 'run', 'automation':'script', 'quiet': i.get('quiet'), diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 99a0ff7ca5..a9a7d0e555 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -30,6 +30,8 @@ ii['docker_cm_repo'] = os.environ['DOCKER_CM_REPO'] if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '': ii['docker_cm_repo_branch'] = os.environ['DOCKER_CM_REPO_BRANCH'] + if os.environ.get('TEST_INPUT_INDEX', '') != '': + ii['test_input_index'] = os.environ['TEST_INPUT_INDEX'] print(ii) r = cm.access(ii) From 2b35b792951856803a0d60a0deaf98fa89fabadc Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 19:00:54 +0530 Subject: [PATCH 471/658] docker run set to false --- script/get-mlperf-inference-sut-description/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/get-mlperf-inference-sut-description/_cm.json b/script/get-mlperf-inference-sut-description/_cm.json index e5b8723c49..d350ee2eaa 100644 --- a/script/get-mlperf-inference-sut-description/_cm.json +++ b/script/get-mlperf-inference-sut-description/_cm.json @@ -4,6 +4,9 @@ "automation_uid": "5b4e0237da074764", "cache": false, "category": "MLPerf benchmark support", + "docker": { + "run": false + }, "deps": [ { "tags": "detect,os" From 3309a3535d45a67437de9b5e4aac1980a5dbb6d2 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 19:05:43 +0530 Subject: [PATCH 472/658] Update results dir env variable --- script/generate-mlperf-inference-submission/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index d6acd726b9..10e24bb874 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -130,7 +130,7 @@ "tags": "get,mlperf,inference,results,dir,local", "names": "get-mlperf-inference-results-dir", "skip_if_env": { - "OUTPUT_BASE_DIR": [ + "CM_MLPERF_INFERENCE_RESULTS_DIR_": [ "on" ] } From 8ffa471eb235ef04f4b2dbcd159c103d4e575d7d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 19:58:38 +0530 Subject: [PATCH 473/658] updated docker permissions --- script/generate-mlperf-inference-submission/_cm.json | 1 + 1 file changed, 1 insertion(+) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 10e24bb874..79bddc8ea7 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -152,6 +152,7 @@ "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" ], + "extra_run_args": " --cap-add SYS_ADMIN --security-opt apparmor=unconfined --security-opt seccomp=unconfined", "os": "ubuntu", "cm_repo": "mlcommons@cm4mlops", "cm_repo_branch": "mlperf-inference", From 23d692b7a3176498109fabd73574326dd1680ed1 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Thu, 31 Oct 2024 23:06:52 +0800 Subject: [PATCH 474/658] - fixed regex for md5sum, pstree, and bzcat which caused failed tests --- script/get-generic-sys-util/_cm.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 39e1322d8e..a95d618b3b 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -237,7 +237,7 @@ "env": { "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", - "CM_SYS_UTIL_VERSION_RE": "Version ([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "libbz2_dev": { @@ -554,7 +554,7 @@ "env": { "CM_SYS_UTIL_NAME": "md5sha1sum", "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", - "CM_SYS_UTIL_VERSION_RE": "md5sum \\(GNU coreutils\\) ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "md5sha1sum": { @@ -661,7 +661,7 @@ "env": { "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "psmisc": { From 5ced2d23ec2e3045ea7973c9d4ea6cf9e69ce2b5 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:42:39 +0530 Subject: [PATCH 475/658] update docker permissions --- script/generate-mlperf-inference-submission/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 79bddc8ea7..354d453b1f 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -152,7 +152,7 @@ "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" ], - "extra_run_args": " --cap-add SYS_ADMIN --security-opt apparmor=unconfined --security-opt seccomp=unconfined", + "extra_run_args": " --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined", "os": "ubuntu", "cm_repo": "mlcommons@cm4mlops", "cm_repo_branch": "mlperf-inference", From 6723d2c2f69329cb84654e38c8670db7ee2a0b93 Mon Sep 17 00:00:00 2001 From: Oseltamvir Date: Thu, 31 Oct 2024 17:24:15 +0000 Subject: [PATCH 476/658] - fixed an annoying psmisc/pstree bug --- script/get-generic-sys-util/_cm.json | 12 ++++++------ script/get-generic-sys-util/customize.py | 23 +++++++++++++++-------- script/get-generic-sys-util/detect.sh | 9 ++++++++- 3 files changed, 29 insertions(+), 15 deletions(-) mode change 100644 => 100755 script/get-generic-sys-util/detect.sh diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a95d618b3b..51fe526c2a 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -89,7 +89,7 @@ "env": { "CM_SYS_UTIL_NAME": "dmidecode", "CM_SYS_UTIL_VERSION_CMD": "dmidecode --version", - "CM_SYS_UTIL_VERSION_RE": "version\\s+([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "dmidecode": { @@ -411,8 +411,8 @@ "libnuma-dev": { "env": { "CM_SYS_UTIL_NAME": "libnuma-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "libnuma-dev": { @@ -629,8 +629,8 @@ ], "env": { "CM_SYS_UTIL_NAME": "numactl", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "numactl": { @@ -661,7 +661,7 @@ "env": { "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) (\\d+\\.\\d+(?:\\.\\d+)?)" }, "state": { "psmisc": { diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 4996459351..5121708343 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -100,25 +100,32 @@ def preprocess(i): return {'return':0} + + def detect_version(i): env = i['env'] version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '') + # Confirm that the regex pattern and file are present if version_check_re == '' or not os.path.exists("tmp-ver.out"): version = "undetected" - else: - r = i['automation'].parse_version({'match_text': version_check_re, - 'group_number': 1, - 'env_key': version_env_key, - 'which_env': env}) - if r['return'] >0: return r + r = i['automation'].parse_version({ + 'match_text': version_check_re, + 'group_number': 1, + 'env_key': version_env_key, + 'which_env': env, + }) + + if r['return'] > 0: + return r version = r['version'] + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + + return {'return': 0, 'version': version} - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} def postprocess(i): env = i['env'] diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh old mode 100644 new mode 100755 index 6f2e3759bd..42de864cde --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,7 +1,14 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out + if [[ "${CM_SYS_UTIL_VERSION_CMD}" == *"pstree --version"* ]]; then + # Redirect both stdout and stderr to tmp-ver.out + ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1 + else + ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out + fi + echo ${CM_SYS_UTIL_VERSION_CMD} + echo $PWD fi test $? -eq 0 || exit $? From 3cabfa1386bf23de8e55724d479c52e391d66c23 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 23:10:29 +0530 Subject: [PATCH 477/658] Add docker input mapping --- script/generate-mlperf-inference-submission/_cm.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 354d453b1f..8a755379a2 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -156,6 +156,10 @@ "os": "ubuntu", "cm_repo": "mlcommons@cm4mlops", "cm_repo_branch": "mlperf-inference", - "os_version": "22.04" + "os_version": "22.04", + "docker_input_mapping": { + "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", + "results_dir": "CM_MLPERF_INFERENCE_RESULTS_DIR_" + } } } From 5149a0b269c0fb4cfdc11c453ad6dfbe33654758 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Oct 2024 23:19:31 +0530 Subject: [PATCH 478/658] enabled modifying input mapping env inside docker --- automation/script/module_misc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index c4e508c352..eb4f37f3a1 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1917,10 +1917,11 @@ def docker(i): if host_env_key: container_env_string += " --env.{}={} ".format(host_env_key, container_env_key) # check if the below lines are needed when inputs are mapped to container paths - '''for v in docker_input_mapping: + + for v in docker_input_mapping: if docker_input_mapping[v] == host_env_key: - i[v] = container_env_key - ''' + i[v] = container_env_key + mounts = list(filter(lambda item: item is not None, mounts)) mount_string = "" if len(mounts)==0 else ",".join(mounts) From 44be9bd3fb9948d8be29908be26eecd8ee24e2b6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 00:45:48 +0530 Subject: [PATCH 479/658] update the env key to run command --- automation/script/module_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index eb4f37f3a1..0a5f283ea8 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1916,11 +1916,11 @@ def docker(i): mounts[index] = new_host_mount+":"+new_container_mount if host_env_key: container_env_string += " --env.{}={} ".format(host_env_key, container_env_key) - # check if the below lines are needed when inputs are mapped to container paths for v in docker_input_mapping: if docker_input_mapping[v] == host_env_key: i[v] = container_env_key + i_run_cmd[v] = container_env_key mounts = list(filter(lambda item: item is not None, mounts)) From dc2d75ca2c006664b0818d3854f36a335676bf60 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:47:16 +0530 Subject: [PATCH 480/658] pass uid and gid args to docker run --- script/run-docker-container/customize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 00ab63c978..2e55978613 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -120,6 +120,8 @@ def postprocess(i): port_map_cmds = [] run_opts = '' + CM_DOCKER_BUILD_ARGS = env.get('+ CM_DOCKER_BUILD_ARGS', []) + if env.get('CM_DOCKER_PRE_RUN_COMMANDS', []): for pre_run_cmd in env['CM_DOCKER_PRE_RUN_COMMANDS']: run_cmds.append(pre_run_cmd) @@ -246,7 +248,7 @@ def postprocess(i): x2 = " && bash ) || bash" - CONTAINER="docker run " + x1 + " --entrypoint " + x + x + " " + run_opts + " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + CONTAINER="docker run " + x1 + " --entrypoint " + x + x + " " + CM_DOCKER_BUILD_ARGS + " " + run_opts + " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag CMD = CONTAINER + " bash -c " + x + run_cmd_prefix + run_cmd + x2 + x print ('') From 97ccbc9baf67d2cf353cd040bc6064f7cdd32434 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:54:20 +0530 Subject: [PATCH 481/658] added docker run args --- automation/script/module_misc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 0a5f283ea8..6479125fcd 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1941,10 +1941,12 @@ def docker(i): docker_use_host_group_id = i.get('docker_use_host_group_id', docker_settings.get('use_host_group_id')) if str(docker_use_host_group_id).lower() not in ['false', 'no', '0'] and os.name != 'nt': env['+ CM_DOCKER_BUILD_ARGS'].append("{}={}".format('GID', '\\" $(id -g $USER) \\"')) + env['+ CM_DOCKER_RUN_ARGS'].append("{}={}".format('GID', '\\" $(id -g $USER) \\"')) docker_use_host_user_id = i.get('docker_use_host_user_id', docker_settings.get('use_host_user_id')) if str(docker_use_host_user_id).lower() not in ['false', 'no', '0'] and os.name != 'nt': env['+ CM_DOCKER_BUILD_ARGS'].append("{}={}".format('UID', '\\" $(id -u $USER) \\"')) + env['+ CM_DOCKER_RUN_ARGS'].append("{}={}".format('UID', '\\" $(id -u $USER) \\"')) docker_base_image = i.get('docker_base_image', docker_settings.get('base_image')) docker_os = i.get('docker_os', docker_settings.get('os', 'ubuntu')) From 1634d68f8f57b0f1845dadc6453cacd71366ecb8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:55:54 +0530 Subject: [PATCH 482/658] Updated build to run args --- script/run-docker-container/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 2e55978613..5b4a1b1c05 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -120,7 +120,7 @@ def postprocess(i): port_map_cmds = [] run_opts = '' - CM_DOCKER_BUILD_ARGS = env.get('+ CM_DOCKER_BUILD_ARGS', []) + CM_DOCKER_RUN_ARGS = env.get('+ CM_DOCKER_RUN_ARGS', []) if env.get('CM_DOCKER_PRE_RUN_COMMANDS', []): for pre_run_cmd in env['CM_DOCKER_PRE_RUN_COMMANDS']: @@ -248,7 +248,7 @@ def postprocess(i): x2 = " && bash ) || bash" - CONTAINER="docker run " + x1 + " --entrypoint " + x + x + " " + CM_DOCKER_BUILD_ARGS + " " + run_opts + " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + CONTAINER="docker run " + x1 + " --entrypoint " + x + x + " " + CM_DOCKER_RUN_ARGS + " " + run_opts + " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag CMD = CONTAINER + " bash -c " + x + run_cmd_prefix + run_cmd + x2 + x print ('') From 5795ff365dd1d8f2f2275ccd25d221996d0b2409 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:57:49 +0530 Subject: [PATCH 483/658] Update module_misc.py --- automation/script/module_misc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 6479125fcd..210557af03 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1932,6 +1932,9 @@ def docker(i): if env.get('+ CM_DOCKER_BUILD_ARGS', []) == []: env['+ CM_DOCKER_BUILD_ARGS'] = [] + if env.get('+ CM_DOCKER_RUN_ARGS', []) == []: + env['+ CM_DOCKER_RUN_ARGS'] = [] + for key in proxy_keys: if os.environ.get(key, '') != '': value = os.environ[key] From 737024bd447cfdbc7644d9057ad092138b0eef03 Mon Sep 17 00:00:00 2001 From: Oseltamvir Date: Fri, 1 Nov 2024 08:00:45 +0000 Subject: [PATCH 484/658] - changed to use env var to detect stderr, fixed some regexes --- automation/script/module.py | 8 ++++++-- script/get-generic-sys-util/_cm.json | 12 +++++++----- script/get-generic-sys-util/customize.py | 2 +- script/get-generic-sys-util/detect.sh | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 64226f5742..4a1b91f233 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -8,7 +8,7 @@ # TBD: when we have bandwidth and resources, we should refactor it # and make it cleaner and simpler while keeping full backwards compatibility. # - +import re import os import logging @@ -3979,7 +3979,11 @@ def parse_version(self, i): return r string = r['string'] - + match = re.search(match_text, string) + if debug: + print(f"Regex Pattern: {match_text}") + print(f"Matched String: {string}") + print(f"Match Groups: {match.groups()}") version = r['match'].group(group_number) which_env[env_key] = version diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 51fe526c2a..345673d364 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -235,9 +235,10 @@ }, "libbz2-dev": { "env": { + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" }, "state": { "libbz2_dev": { @@ -351,7 +352,7 @@ "env": { "CM_SYS_UTIL_NAME": "liblzma_dev", "CM_SYS_UTIL_VERSION_CMD": "xz --version", - "CM_SYS_UTIL_VERSION_RE": "[A-Za-z]+\\s\\d\\.\\d\\.\\d" + "CM_SYS_UTIL_VERSION_RE": "(\\d(\\.\\d)+)" }, "state": { "liblzma_dev": { @@ -475,8 +476,8 @@ "libreadline-dev": { "env": { "CM_SYS_UTIL_NAME": "libreadline_dev", - "CM_SYS_UTIL_VERSION_CMD": "readline --version", - "CM_SYS_UTIL_VERSION_RE": "readline, version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion readline", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libreadline_dev": { @@ -490,7 +491,7 @@ "libsqlite3-dev": { "env": { "CM_SYS_UTIL_NAME": "libsqlite3_dev", - "CM_SYS_UTIL_VERSION_CMD": "sqlite3 --version", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion sqlite3", "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { @@ -659,6 +660,7 @@ }, "psmisc": { "env": { + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) (\\d+\\.\\d+(?:\\.\\d+)?)" diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 5121708343..7480c77991 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -115,7 +115,7 @@ def detect_version(i): 'match_text': version_check_re, 'group_number': 1, 'env_key': version_env_key, - 'which_env': env, + 'which_env': env, #'debug': 1 }) if r['return'] > 0: diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 42de864cde..dff0d91fd7 100755 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,7 +1,7 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - if [[ "${CM_SYS_UTIL_VERSION_CMD}" == *"pstree --version"* ]]; then + if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then # Redirect both stdout and stderr to tmp-ver.out ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1 else From bc3294608bceed0419d22cad3088ea6b010ba57c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:04:37 +0000 Subject: [PATCH 485/658] Improvements to version detect for get-generic-sys-util --- automation/script/module.py | 5 ++++- script/get-generic-sys-util/_cm.json | 6 +++++- script/get-generic-sys-util/customize.py | 10 ++++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 64226f5742..f5e89b362a 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3980,7 +3980,10 @@ def parse_version(self, i): string = r['string'] - version = r['match'].group(group_number) + if r['match'].lastindex and r['match'].lastindex >= group_number: + version = r['match'].group(group_number) + else: + return {'return':1, 'error': 'Invalid version detection group number. Version was not detected. Last index of match = {}. Given group number = {}'.format(r['match'].lastindex, group_number)} which_env[env_key] = version which_env['CM_DETECTED_VERSION'] = version # to be recorded in the cache meta diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index c2731d15ab..8db37db16e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -457,8 +457,12 @@ }, "md5sha1sum": { "env": { - "CM_SYS_UTIL_NAME": "md5sha1sum" + "CM_SYS_UTIL_NAME": "md5sha1sum", + "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, + "new_env_keys": ["CM_MD5SHA1SUM_VERSION"], "state": { "md5sha1sum": { "apt": "", diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 4996459351..28bc35b3ec 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -17,7 +17,8 @@ def preprocess(i): if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '': r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0: #detection failed, do install via prehook_deps + if r['return'] != 0: #detection failed, do install via prehook_deps + print("detection failed, going for installation") env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" return {'return': 0} else: #detection is successful, no need to install @@ -104,13 +105,14 @@ def detect_version(i): env = i['env'] version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '') + group_number = env.get('CM_TMP_VERSION_DETECT_GROUP_NUMBER', 1) if version_check_re == '' or not os.path.exists("tmp-ver.out"): version = "undetected" else: r = i['automation'].parse_version({'match_text': version_check_re, - 'group_number': 1, + 'group_number': group_number, 'env_key': version_env_key, 'which_env': env}) if r['return'] >0: return r @@ -128,8 +130,8 @@ def postprocess(i): if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') : automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0: - return r + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: + return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) From 4ea53419d3dc632f569db5a7613aff8b2818dbc3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:09:59 +0000 Subject: [PATCH 486/658] Improvements to version detect for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 3 ++- script/get-generic-sys-util/detect.sh | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 8db37db16e..bdf81d8d6e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -460,7 +460,8 @@ "CM_SYS_UTIL_NAME": "md5sha1sum", "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes" }, "new_env_keys": ["CM_MD5SHA1SUM_VERSION"], "state": { diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 6f2e3759bd..4095d0fbdf 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,7 +1,14 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out + if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then + # Redirect both stdout and stderr to tmp-ver.out + cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1" + else + cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" + fi + echo $cmd + eval $cmd + test $? -eq 0 || exit $? fi -test $? -eq 0 || exit $? From ee098c9a1968e7d05540a206dbde87f85591df6f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:28:04 +0000 Subject: [PATCH 487/658] Ignore version detect error for md5sum on macos --- script/get-generic-sys-util/_cm.json | 2 +- script/get-generic-sys-util/detect.sh | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index bdf81d8d6e..7fc3361930 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -461,7 +461,7 @@ "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes" + "CM_SYS_UTIL_VERSION_CMD_IGNORE_ERROR": "yes" }, "new_env_keys": ["CM_MD5SHA1SUM_VERSION"], "state": { diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 4095d0fbdf..c983aca149 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -8,7 +8,12 @@ if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" fi echo $cmd - eval $cmd - test $? -eq 0 || exit $? + if [[ "${CM_SYS_UTIL_VERSION_CMD_IGNORE_ERROR}" == "yes" ]]; then + eval $cmd + exit 0 + else + eval $cmd + test $? -eq 0 || exit $? + fi fi From 4de4c7d7be3169c132f35045ce0af8f86ba9bf74 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:37:39 +0000 Subject: [PATCH 488/658] Ignore version detect error for md5sum on macos --- script/get-generic-sys-util/_cm.json | 5 ++--- script/get-generic-sys-util/detect.sh | 9 ++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 7fc3361930..85a5b84503 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -458,10 +458,9 @@ "md5sha1sum": { "env": { "CM_SYS_UTIL_NAME": "md5sha1sum", - "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", + "CM_SYS_UTIL_VERSION_CMD": "md5sum --version | grep sha1sum", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, - "CM_SYS_UTIL_VERSION_CMD_IGNORE_ERROR": "yes" + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "new_env_keys": ["CM_MD5SHA1SUM_VERSION"], "state": { diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index c983aca149..4095d0fbdf 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -8,12 +8,7 @@ if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" fi echo $cmd - if [[ "${CM_SYS_UTIL_VERSION_CMD_IGNORE_ERROR}" == "yes" ]]; then - eval $cmd - exit 0 - else - eval $cmd - test $? -eq 0 || exit $? - fi + eval $cmd + test $? -eq 0 || exit $? fi From 5d7fab818354f232a809875a67d4a1a64fb34027 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:43:37 +0000 Subject: [PATCH 489/658] Ignore version detect failure if FAIL_SAFE is on --- script/get-generic-sys-util/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 28bc35b3ec..79e3f32f60 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -130,7 +130,7 @@ def postprocess(i): if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') : automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '').lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) From 86125c5f8cef42d5f3e5f9f525a8a426756f7af3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:45:05 +0000 Subject: [PATCH 490/658] Ignore version detect failure if FAIL_SAFE is on --- script/get-generic-sys-util/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 79e3f32f60..edc09f22a6 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -130,7 +130,7 @@ def postprocess(i): if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') : automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '').lower() not in [ "1", "yes", "true" ]: + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) From cc82180ff033a90036e08b41bdcb0e82e0f616d4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 12:51:12 +0000 Subject: [PATCH 491/658] Ignore version detect failure if FAIL_SAFE is on --- script/get-generic-sys-util/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index edc09f22a6..b020841eac 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -56,6 +56,7 @@ def preprocess(i): if not package_name: if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in [ "1", "true", "yes" ]: print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...") + env['CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED'] = 'yes' return {'return': 0} else: return {'return': 1, 'error': f'No package name specified for {pm} and util name {util}'} @@ -127,7 +128,7 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') : + if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: From ffa3966c10932a5868c5876b36fc50edc8c8cfc7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 14:55:14 +0000 Subject: [PATCH 492/658] Use raw string in build-docker-image --- script/build-docker-image/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-docker-image/customize.py b/script/build-docker-image/customize.py index e66eddbd8b..a3a3bc8df7 100644 --- a/script/build-docker-image/customize.py +++ b/script/build-docker-image/customize.py @@ -54,7 +54,7 @@ def preprocess(i): image_name = get_image_name(env) if build_dockerfile: - dockerfile_path = "\${CM_DOCKERFILE_WITH_PATH}" + dockerfile_path = r"\${CM_DOCKERFILE_WITH_PATH}" # Write .dockerignore with open('.dockerignore', 'w') as f: From aca9abdd3e65e45d37d672cd260f688881a685a4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 15:31:13 +0000 Subject: [PATCH 493/658] Update VERSION | Test pypi release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a1dad2aa79..cfe389e945 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.23 +0.3.24 From 0529eb0a281d413c6b45d6eccb36f9f62385be33 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 15:55:24 +0000 Subject: [PATCH 494/658] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48e844e593..47afb3fb19 100644 --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ def read_file(file_name, default=""): return default long_description = read_file("README.md", "No description available.") -version_ = read_file("VERSION", "0.3.0") +version_ = read_file("VERSION", "0.3.1") setup( name='cm4mlops', From 7d6ee2bbaffd444f55a28135d7ff1bc207454878 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:38:28 +0530 Subject: [PATCH 495/658] clean the contents rather than the main submission folder --- script/generate-mlperf-inference-submission/customize.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index f82893e39a..665a889088 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -70,7 +70,13 @@ def generate_submission(i): print ('=================================================') print ('Cleaning {} ...'.format(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'])) if os.path.exists(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): - shutil.rmtree(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']) + # delete the content keeping the main folder intact + for items in os.listdir(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): + item_path = os.path.join(folder_path, item) + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) print ('=================================================') if not os.path.isdir(submission_dir): From cb818165307f84f6ce05be70757317b3075e0701 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 22:02:59 +0530 Subject: [PATCH 496/658] update-prevent deletion of entire directory --- .../customize.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index 03bca7cd9b..216d7e77bf 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -37,7 +37,22 @@ def postprocess(i): submission_processed = submission_dir + "_processed" shutil.copytree(submission_dir, submission_backup) - shutil.rmtree(submission_dir) - os.rename(submission_processed, submission_dir) - + + # clean the submission directory + for item in os.listdir(submission_dir): + item_path = os.path.join(submission_dir, item) + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + # copy the files from submission processed directory to submission directory + for item in os.listdir(submission_processed): + source = os.path.join(submission_processed, item) + destination = os.path.join(submission_dir, item) + if os.path.isdir(source): + shutil.copytree(source, destination) # Copy entire directory + else: + shutil.copy2(source, destination) # Copy individual files + return {'return':0} From fc4fea3ccb27e5a0a673ee81e2945aeaa99ffd86 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 22:20:59 +0530 Subject: [PATCH 497/658] test commit --- script/preprocess-mlperf-inference-submission/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index 216d7e77bf..b370370f50 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -53,6 +53,6 @@ def postprocess(i): if os.path.isdir(source): shutil.copytree(source, destination) # Copy entire directory else: - shutil.copy2(source, destination) # Copy individual files + shutil.copy(source, destination) # Copy individual files return {'return':0} From fd70fcfe0c98900fce91fe7004ddd5ffec486d1c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Nov 2024 23:56:00 +0530 Subject: [PATCH 498/658] bug fix --- script/generate-mlperf-inference-submission/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 665a889088..ef6e97c7b2 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -72,7 +72,7 @@ def generate_submission(i): if os.path.exists(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): # delete the content keeping the main folder intact for items in os.listdir(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): - item_path = os.path.join(folder_path, item) + item_path = os.path.join(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'], item) if os.path.isdir(item_path): shutil.rmtree(item_path) else: From 8cf8d56dd67339bac93c021a416dc3958e32773e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 2 Nov 2024 00:11:02 +0530 Subject: [PATCH 499/658] bug fix --- script/generate-mlperf-inference-submission/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index ef6e97c7b2..10fde422a2 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -71,7 +71,7 @@ def generate_submission(i): print ('Cleaning {} ...'.format(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'])) if os.path.exists(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): # delete the content keeping the main folder intact - for items in os.listdir(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): + for item in os.listdir(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): item_path = os.path.join(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'], item) if os.path.isdir(item_path): shutil.rmtree(item_path) From 941efef8f8e4c2f5249cfca48f7e04a11a8ad4c4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 2 Nov 2024 00:50:14 +0530 Subject: [PATCH 500/658] added mount for submission processed --- script/generate-mlperf-inference-submission/_cm.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 8a755379a2..d4859f475a 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -72,7 +72,8 @@ }, "default_env": { "CM_RUN_MLPERF_ACCURACY": "on", - "CM_MLPERF_RUN_STYLE": "valid" + "CM_MLPERF_RUN_STYLE": "valid", + "CM_MLPERF_INFERENCE_SUBMISSION_PROCESSED_DIR": "<<>>_processed" }, "post_deps": [ { @@ -150,7 +151,8 @@ ], "mounts": [ "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" + "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}", + "${{ CM_MLPERF_INFERENCE_SUBMISSION_PROCESSED_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_PROCESSED_DIR }}" ], "extra_run_args": " --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined", "os": "ubuntu", From 54afe334666c8ab9383d5b98a194a6a638de581e Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Fri, 1 Nov 2024 17:27:03 -0500 Subject: [PATCH 501/658] Update CODEOWNERS --- .github/CODEOWNERS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 06ce1b0991..21b575640a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,8 @@ # Unless a later match takes precedence,they will be requested for review when someone opens a pull request. * @mlcommons/wg-ck -/CODEOWNERS @mlcommons/staff +/.github/CODEOWNERS @mlcommons/systems + +/.github/workflows/cla.yml @mlcommons/systems + +/LICENSE.md @mlcommons/systems From 6d9d67571f2a5451e8d50f08ffb6c0b6ab06d0a6 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 00:58:31 +0800 Subject: [PATCH 502/658] Change numactl command to be more inclusive libnuma-dev => libnuma --- script/get-generic-sys-util/_cm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 2246bc6a2b..fb1fd4c86c 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -412,7 +412,7 @@ "libnuma-dev": { "env": { "CM_SYS_UTIL_NAME": "libnuma-dev", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { @@ -632,7 +632,7 @@ ], "env": { "CM_SYS_UTIL_NAME": "numactl", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { From f1f084d71d588f1fa62b00242f929cafa3d01582 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 2 Nov 2024 23:26:26 +0530 Subject: [PATCH 503/658] Added sympy dependency for retinanet --- script/app-mlperf-inference-nvidia/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 3c3ea70733..a997ef8156 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -436,6 +436,7 @@ variations: - tags: get,generic-python-lib,_onnx-graphsurgeon - tags: get,generic-python-lib,_package.onnx version: 1.13.1 + - tags: get,generic-python-lib,_package.sympy sdxl: new_env_keys: From b9eb3d7db08e890b5b8e75759cfd8e65f758cf9c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 2 Nov 2024 18:04:13 +0000 Subject: [PATCH 504/658] Update _cm.json | Fix libnuma re --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index fb1fd4c86c..2579713152 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -413,7 +413,8 @@ "env": { "CM_SYS_UTIL_NAME": "libnuma-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libnuma-dev": { From 3486ef92ab4488ec76ec5edd8d84f3a92c79a969 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 2 Nov 2024 18:45:50 +0000 Subject: [PATCH 505/658] Update default-config.yaml --- .../framework-version-default/default-config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml index d2c66a716c..cffda0ea0c 100644 --- a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml +++ b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml @@ -12,9 +12,13 @@ bert-99: Offline: target_qps: 8000 + Server: + target_qps: 6000 bert-99.9: Offline: - target_qps: 8000 + target_qps: 3500 + Server: + target_qps: 3000 3d-unet-99: Offline: target_qps: 8.0 From f1010d615bc8e7ad25b9c708ffe04c3f66cae8cd Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 02:52:49 +0800 Subject: [PATCH 506/658] pstree regex change to be more lax for brew version --- script/get-generic-sys-util/_cm.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 2579713152..ef849a3246 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -414,7 +414,7 @@ "CM_SYS_UTIL_NAME": "libnuma-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libnuma-dev": { @@ -559,7 +559,6 @@ "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, - "new_env_keys": ["CM_MD5SHA1SUM_VERSION"], "state": { "md5sha1sum": { "apt": "", @@ -663,10 +662,10 @@ }, "psmisc": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) (\\d+\\.\\d+(?:\\.\\d+)?)" + "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "psmisc": { From 8fef3edd4dda66661598a89500d075ce49be8756 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 18:35:54 +0800 Subject: [PATCH 507/658] Added CM_TMP_VERSION_DETECT_GROUP_NUMBER for all packages --- script/get-generic-sys-util/_cm.json | 151 ++++++++++++++++++--------- 1 file changed, 101 insertions(+), 50 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index ef849a3246..9f63ca943e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -73,7 +73,8 @@ "env": { "CM_SYS_UTIL_NAME": "cmake", "CM_SYS_UTIL_VERSION_CMD": "cmake --version", - "CM_SYS_UTIL_VERSION_RE": "cmake version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "cmake version ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "cmake": { @@ -89,7 +90,8 @@ "env": { "CM_SYS_UTIL_NAME": "dmidecode", "CM_SYS_UTIL_VERSION_CMD": "dmidecode --version", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "dmidecode": { @@ -105,7 +107,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++11": { @@ -119,7 +122,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++12": { @@ -133,7 +137,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++9", "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++9": { @@ -147,7 +152,8 @@ "env": { "CM_SYS_UTIL_NAME": "gcc11", "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "gcc11": { @@ -160,7 +166,8 @@ "env": { "CM_SYS_UTIL_NAME": "gcc9", "CM_SYS_UTIL_VERSION_CMD": "gcc-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "gcc9": { @@ -173,7 +180,9 @@ "env": { "CM_SYS_UTIL_NAME": "gflags-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gflags", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + }, "state": { "gflags-dev": { @@ -189,7 +198,8 @@ "env": { "CM_SYS_UTIL_NAME": "git-lfs", "CM_SYS_UTIL_VERSION_CMD": "git-lfs --version", - "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "git-lfs": { @@ -205,7 +215,8 @@ "env": { "CM_SYS_UTIL_NAME": "glog-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libglog", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "glog-dev": { @@ -221,7 +232,8 @@ "env": { "CM_SYS_UTIL_NAME": "libboost-all-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libboost-dev | grep 'Version'", - "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libboost-all-dev": { @@ -238,7 +250,8 @@ "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", - "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libbz2_dev": { @@ -255,7 +268,8 @@ "env": { "CM_SYS_UTIL_NAME": "libev_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libev-dev | grep 'Version'", - "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libev_dev": { @@ -268,7 +282,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi": { @@ -281,7 +296,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi_dev": { @@ -297,7 +313,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi7", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l libffi7 2>/dev/null | grep '^ii' | awk '{print $3}' || rpm -q libffi7 2>/dev/null || pacman -Q libffi7 2>/dev/null", - "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d-[0-9]+" + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d-[0-9]+", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi7": { @@ -310,7 +327,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi8", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi8", - "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d" + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi8": { @@ -323,7 +341,8 @@ "env": { "CM_SYS_UTIL_NAME": "libgdbm_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libgdbm-dev | grep 'Version'", - "CM_SYS_UTIL_VERSION_RE": "([\\d]+\\.[\\d\\.-]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d]+\\.[\\d\\.-]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libgdbm_dev": { @@ -336,7 +355,8 @@ "env": { "CM_SYS_UTIL_NAME": "libgmock-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gmock", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libgmock-dev": { @@ -352,7 +372,8 @@ "env": { "CM_SYS_UTIL_NAME": "liblzma_dev", "CM_SYS_UTIL_VERSION_CMD": "xz --version", - "CM_SYS_UTIL_VERSION_RE": "(\\d(\\.\\d)+)" + "CM_SYS_UTIL_VERSION_RE": "(\\d(\\.\\d)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "liblzma_dev": { @@ -365,7 +386,8 @@ "env": { "CM_SYS_UTIL_NAME": "libmkl-dev", "CM_SYS_UTIL_VERSION_CMD": "ldconfig -p | grep libmkl", - "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libmkl-dev": { @@ -381,7 +403,8 @@ "env": { "CM_SYS_UTIL_NAME": "libmpfr-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion mpfr", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libmpfr-dev": { @@ -398,7 +421,8 @@ "env": { "CM_SYS_UTIL_NAME": "libncurses_dev", "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", - "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libncurses_dev": { @@ -430,7 +454,8 @@ "env": { "CM_SYS_UTIL_NAME": "libpci-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpci", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libpci-dev": { @@ -446,7 +471,8 @@ "env": { "CM_SYS_UTIL_NAME": "libpng-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpng", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libpng-dev": { @@ -462,7 +488,8 @@ "env": { "CM_SYS_UTIL_NAME": "libre2-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion re2", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libre2-dev": { @@ -478,7 +505,8 @@ "env": { "CM_SYS_UTIL_NAME": "libreadline_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion readline", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libreadline_dev": { @@ -493,7 +521,8 @@ "env": { "CM_SYS_UTIL_NAME": "libsqlite3_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion sqlite3", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libsqlite3_dev": { @@ -506,7 +535,8 @@ "env": { "CM_SYS_UTIL_NAME": "libssl_dev", "CM_SYS_UTIL_VERSION_CMD": "openssl version", - "CM_SYS_UTIL_VERSION_RE": "OpenSSL\\s+([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "OpenSSL\\s+([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libssl_dev": { @@ -522,7 +552,8 @@ "env": { "CM_SYS_UTIL_NAME": "libudev-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libudev", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libudev-dev": { @@ -543,7 +574,8 @@ "env": { "CM_SYS_UTIL_NAME": "linux-tools", "CM_SYS_UTIL_VERSION_CMD": "linux-tools --version", - "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "linux-tools": { @@ -571,7 +603,8 @@ "env": { "CM_SYS_UTIL_NAME": "ninja-build", "CM_SYS_UTIL_VERSION_CMD": "ninja --version", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "ninja-build": { @@ -588,7 +621,8 @@ "env": { "CM_SYS_UTIL_NAME": "nlohmann_json3_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion nlohmann_json", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "nlohmann_json3_dev": { @@ -602,7 +636,8 @@ "env": { "CM_SYS_UTIL_NAME": "ntpdate", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'", - "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "ntpdate": { @@ -633,7 +668,8 @@ "env": { "CM_SYS_UTIL_NAME": "numactl", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "numactl": { @@ -648,7 +684,8 @@ "env": { "CM_SYS_UTIL_NAME": "nvidia-cuda-toolkit", "CM_SYS_UTIL_VERSION_CMD": "nvcc --version", - "CM_SYS_UTIL_VERSION_RE": "release ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "release ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "nvidia-cuda-toolkit": { @@ -665,7 +702,8 @@ "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "psmisc": { @@ -681,7 +719,8 @@ "env": { "CM_SYS_UTIL_NAME": "rapidjson-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion RapidJSON", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "rapidjson-dev": { @@ -697,7 +736,8 @@ "env": { "CM_SYS_UTIL_NAME": "rsync", "CM_SYS_UTIL_VERSION_CMD": "rsync --version", - "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "rsync": { @@ -714,7 +754,8 @@ "env": { "CM_SYS_UTIL_NAME": "screen", "CM_SYS_UTIL_VERSION_CMD": "screen --version", - "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "screen": { @@ -731,7 +772,8 @@ "env": { "CM_SYS_UTIL_NAME": "sox", "CM_SYS_UTIL_VERSION_CMD": "sox --version", - "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "sox": { @@ -746,7 +788,8 @@ "env": { "CM_SYS_UTIL_NAME": "systemd", "CM_SYS_UTIL_VERSION_CMD": "systemctl --version", - "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)" + "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "systemd": { @@ -762,7 +805,8 @@ "env": { "CM_SYS_UTIL_NAME": "tk_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s tk-dev | grep Version", - "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "tk_dev": { @@ -775,7 +819,8 @@ "env": { "CM_SYS_UTIL_NAME": "transmission", "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", - "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "transmission": { @@ -791,7 +836,8 @@ "env": { "CM_SYS_UTIL_NAME": "wget", "CM_SYS_UTIL_VERSION_CMD": "wget --version", - "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "wget": { @@ -807,7 +853,8 @@ "env": { "CM_SYS_UTIL_NAME": "wkhtmltopdf", "CM_SYS_UTIL_VERSION_CMD": "wkhtmltopdf --version", - "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "wkhtmltopdf": { @@ -821,7 +868,8 @@ "env": { "CM_SYS_UTIL_NAME": "vim_common", "CM_SYS_UTIL_VERSION_CMD": "vim --version", - "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "vim_common": { @@ -838,7 +886,8 @@ "env": { "CM_SYS_UTIL_NAME": "xz", "CM_SYS_UTIL_VERSION_CMD": "xz --version", - "CM_SYS_UTIL_VERSION_RE": "xz \\(XZ Utils\\) ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "xz \\(XZ Utils\\) ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "xz": { @@ -855,7 +904,8 @@ "env": { "CM_SYS_UTIL_NAME": "zlib", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "zlib": { @@ -869,7 +919,8 @@ "env": { "CM_SYS_UTIL_NAME": "zlib1g_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "zlib1g_dev": { From 7da8b3ec9038199237e3704e71295260549579de Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 14:25:18 +0000 Subject: [PATCH 508/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 9f63ca943e..49ad809357 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -668,7 +668,7 @@ "env": { "CM_SYS_UTIL_NAME": "numactl", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { @@ -699,10 +699,10 @@ }, "psmisc": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { @@ -737,7 +737,7 @@ "CM_SYS_UTIL_NAME": "rsync", "CM_SYS_UTIL_VERSION_CMD": "rsync --version", "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "rsync": { @@ -755,7 +755,7 @@ "CM_SYS_UTIL_NAME": "screen", "CM_SYS_UTIL_VERSION_CMD": "screen --version", "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "screen": { @@ -773,7 +773,7 @@ "CM_SYS_UTIL_NAME": "sox", "CM_SYS_UTIL_VERSION_CMD": "sox --version", "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "sox": { @@ -789,7 +789,7 @@ "CM_SYS_UTIL_NAME": "systemd", "CM_SYS_UTIL_VERSION_CMD": "systemctl --version", "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "systemd": { @@ -806,7 +806,7 @@ "CM_SYS_UTIL_NAME": "tk_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s tk-dev | grep Version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "tk_dev": { @@ -820,7 +820,7 @@ "CM_SYS_UTIL_NAME": "transmission", "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "transmission": { @@ -837,7 +837,7 @@ "CM_SYS_UTIL_NAME": "wget", "CM_SYS_UTIL_VERSION_CMD": "wget --version", "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "wget": { @@ -854,7 +854,7 @@ "CM_SYS_UTIL_NAME": "wkhtmltopdf", "CM_SYS_UTIL_VERSION_CMD": "wkhtmltopdf --version", "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "wkhtmltopdf": { @@ -869,7 +869,7 @@ "CM_SYS_UTIL_NAME": "vim_common", "CM_SYS_UTIL_VERSION_CMD": "vim --version", "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "vim_common": { From 53b1f7a87969464e1a475d5378d6444a841a9c57 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 14:30:59 +0000 Subject: [PATCH 509/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 49ad809357..2ce5249c41 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -108,7 +108,7 @@ "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "g++11": { @@ -123,7 +123,7 @@ "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "g++12": { @@ -138,7 +138,7 @@ "CM_SYS_UTIL_NAME": "g++9", "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "g++9": { @@ -153,7 +153,7 @@ "CM_SYS_UTIL_NAME": "gcc11", "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "gcc11": { @@ -199,7 +199,7 @@ "CM_SYS_UTIL_NAME": "git-lfs", "CM_SYS_UTIL_VERSION_CMD": "git-lfs --version", "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "git-lfs": { @@ -233,7 +233,7 @@ "CM_SYS_UTIL_NAME": "libboost-all-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libboost-dev | grep 'Version'", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libboost-all-dev": { @@ -249,7 +249,7 @@ "env": { "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", + "CM_SYS_UTIL_VERSION_CMD": "bzcat --version | grep Version", "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, From 9e3964226a195755307b17e112e6904e953c4873 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 14:34:14 +0000 Subject: [PATCH 510/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 2ce5249c41..e6984d6ab6 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -250,8 +250,8 @@ "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version | grep Version", - "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libbz2_dev": { From 68596062b031f127b5df5283f035d09797ba2b05 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 23:02:17 +0800 Subject: [PATCH 511/658] change g++/gcc regex --- script/get-generic-sys-util/_cm.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e6984d6ab6..0dd6dfc55e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -107,8 +107,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++11": { @@ -122,8 +122,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++12": { @@ -137,8 +137,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++9", "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++9": { @@ -152,8 +152,8 @@ "env": { "CM_SYS_UTIL_NAME": "gcc11", "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "gcc11": { @@ -166,7 +166,7 @@ "env": { "CM_SYS_UTIL_NAME": "gcc9", "CM_SYS_UTIL_VERSION_CMD": "gcc-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { From 646252a5f5dc20c34c1200437c361fe5c08198d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 15:24:55 +0000 Subject: [PATCH 512/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 0dd6dfc55e..f2205b16ca 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -249,7 +249,7 @@ "env": { "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD": "bzcat --version | grep Version", + "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, From 90d17f00f5dc77ce01b2b1786049761c309b8ee8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 16:20:41 +0000 Subject: [PATCH 513/658] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 85a5b84503..9c7e7d559c 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -11,6 +11,14 @@ "deps": [ { "tags": "detect,os" + }, + { + "tags": "get,generic-sys-util,_package.pkg-config", + "skip_if_env": { + "CM_SYS_UTIL_NAME": [ + "pkg-config" + ] + } } ], "new_env_keys": [ @@ -548,6 +556,19 @@ } } }, + "pkg-config": { + "env": { + "CM_SYS_UTIL_NAME": "pkg_config" + }, + "state": { + "pkg_config": { + "apt": "pkg-config", + "brew": "pkg-config", + "dnf": "pkg-config", + "yum": "pkg-config" + } + } + }, "psmisc": { "env": { "CM_SYS_UTIL_NAME": "psmisc" From 1f961608298db9df3e25b28e6c31772dfa376ecf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 16:23:38 +0000 Subject: [PATCH 514/658] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 9c7e7d559c..0944a82ab1 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -13,7 +13,7 @@ "tags": "detect,os" }, { - "tags": "get,generic-sys-util,_package.pkg-config", + "tags": "get,generic-sys-util,_pkg-config", "skip_if_env": { "CM_SYS_UTIL_NAME": [ "pkg-config" From 26f9eb332aa2ba200a19cc3316085bc1b0730170 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 16:28:38 +0000 Subject: [PATCH 515/658] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 0944a82ab1..941ebfb9e6 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -16,7 +16,7 @@ "tags": "get,generic-sys-util,_pkg-config", "skip_if_env": { "CM_SYS_UTIL_NAME": [ - "pkg-config" + "pkg_config" ] } } From 57b659a416815990b19f3269af64bd04e36fa6da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 17:58:20 +0000 Subject: [PATCH 516/658] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 4 +++- script/get-generic-sys-util/detect.sh | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 941ebfb9e6..be44441a9a 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -558,7 +558,9 @@ }, "pkg-config": { "env": { - "CM_SYS_UTIL_NAME": "pkg_config" + "CM_SYS_UTIL_NAME": "pkg_config", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "pkg_config": { diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 4095d0fbdf..330a10806f 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,14 +1,18 @@ #!/bin/bash - -if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then - # Redirect both stdout and stderr to tmp-ver.out - cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1" - else - cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" - fi - echo $cmd - eval $cmd +if [[ -n "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" ]]; then + eval "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" test $? -eq 0 || exit $? +else + if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then + if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then + # Redirect both stdout and stderr to tmp-ver.out + cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1" + else + cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" + fi + echo $cmd + eval $cmd + test $? -eq 0 || exit $? + fi fi From f0497658b825e69def4baf5648c66382bc351ccf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 18:10:13 +0000 Subject: [PATCH 517/658] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index be44441a9a..6fc9c004ee 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -13,7 +13,7 @@ "tags": "detect,os" }, { - "tags": "get,generic-sys-util,_pkg-config", + "tags": "get,generic-sys-util,_pkg-config,_detect", "skip_if_env": { "CM_SYS_UTIL_NAME": [ "pkg_config" From 5e02bad6c89399f1b0778a9bb5c3d2589a7775c3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 18:25:58 +0000 Subject: [PATCH 518/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 38907e7a59..dfabebeb9f 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -255,9 +255,8 @@ }, "libbz2-dev": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", + "CM_SYS_UTIL_VERSION_CMD_OVERRIDE": "bzcat --version > tmp_ver.out 2>&1 | grep bzip", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, From b1f06fd9a35128791ca0573157cf8460df1f2e38 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Mon, 4 Nov 2024 02:41:49 +0800 Subject: [PATCH 519/658] Implements #455 uses CM_REPO_PATH to set local repo to copy to CM/repos/mlcommons@cm4mlops, currently DOES NOT USE docker_cm_repo --- script/build-dockerfile/customize.py | 93 ++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 20 deletions(-) diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 69a603b64f..de96626070 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -3,6 +3,7 @@ import os import json import re +import shutil def preprocess(i): @@ -53,21 +54,64 @@ def preprocess(i): if not docker_image_base: return {'return': 1, 'error': f"Version \"{env['CM_DOCKER_OS_VERSION']}\" is not supported yet for \"{env['CM_DOCKER_OS']}\" "} - if env.get("CM_MLOPS_REPO", "") != "": - cm_mlops_repo = env["CM_MLOPS_REPO"] - # the below pattern matches both the HTTPS and SSH git link formats - git_link_pattern = r'^(https?://github\.com/([^/]+)/([^/]+)(?:\.git)?|git@github\.com:([^/]+)/([^/]+)(?:\.git)?)$' - if match := re.match(git_link_pattern, cm_mlops_repo): - if match.group(2) and match.group(3): - repo_owner = match.group(2) - repo_name = match.group(3) - elif match.group(4) and match.group(5): - repo_owner = match.group(4) - repo_name = match.group(5) - cm_mlops_repo = f"{repo_owner}@{repo_name}" - print(f"Converted repo format from {env['CM_MLOPS_REPO']} to {cm_mlops_repo}") + # Handle cm_mlops Repository + if env.get("CM_REPO_PATH", "") != "": + use_copy_repo = True + cm_repo_path = os.path.abspath(env["CM_REPO_PATH"]) + + if not os.path.exists(cm_repo_path): + return {'return': 1, 'error': f"Specified CM_REPO_PATH does not exist: {cm_repo_path}"} + + cmr_yml_path = os.path.join(cm_repo_path, "cmr.yaml") + if not os.path.isfile(cmr_yml_path): + return {'return': 1, 'error': f"cmr.yaml not found in CM_REPO_PATH: {cm_repo_path}"} + + # Define the build context directory (where the Dockerfile will be) + build_context_dir = os.path.dirname(env.get('CM_DOCKERFILE_WITH_PATH', os.path.join(os.getcwd(), "Dockerfile"))) + os.makedirs(build_context_dir, exist_ok=True) + + # Create cm_repo directory relative to the build context + repo_build_context_path = os.path.join(build_context_dir, "cm_repo") + + # Remove existing directory if it exists + if os.path.exists(repo_build_context_path): + shutil.rmtree(repo_build_context_path) + + try: + print(f"Copying repository from {cm_repo_path} to {repo_build_context_path}") + shutil.copytree(cm_repo_path, repo_build_context_path) + except Exception as e: + return {'return': 1, 'error': f"Failed to copy repository to build context: {str(e)}"} + + if not os.path.isdir(repo_build_context_path): + return {'return': 1, 'error': f"Repository was not successfully copied to {repo_build_context_path}"} + + # (Optional) Verify the copy + if not os.path.isdir(repo_build_context_path): + return {'return': 1, 'error': f"cm_repo was not successfully copied to the build context at {repo_build_context_path}"} + else: + print(f"cm_repo is present in the build context at {repo_build_context_path}") + + relative_repo_path = os.path.relpath(repo_build_context_path, build_context_dir) else: - cm_mlops_repo = "mlcommons@cm4mlops" + # CM_REPO_PATH is not set; use cm pull repo as before + use_copy_repo = False + + if env.get("CM_MLOPS_REPO", "") != "": + cm_mlops_repo = env["CM_MLOPS_REPO"] + # the below pattern matches both the HTTPS and SSH git link formats + git_link_pattern = r'^(https?://github\.com/([^/]+)/([^/]+)(?:\.git)?|git@github\.com:([^/]+)/([^/]+)(?:\.git)?)$' + if match := re.match(git_link_pattern, cm_mlops_repo): + if match.group(2) and match.group(3): + repo_owner = match.group(2) + repo_name = match.group(3) + elif match.group(4) and match.group(5): + repo_owner = match.group(4) + repo_name = match.group(5) + cm_mlops_repo = f"{repo_owner}@{repo_name}" + print(f"Converted repo format from {env['CM_MLOPS_REPO']} to {cm_mlops_repo}") + else: + cm_mlops_repo = "mlcommons@cm4mlops" cm_mlops_repo_branch_string = f" --branch={env['CM_MLOPS_REPO_BRANCH']}" @@ -125,7 +169,6 @@ def preprocess(i): f.write(EOL) copy_cmds = [] if 'CM_DOCKER_COPY_FILES' in env: - import shutil for copy_file in env['CM_DOCKER_COPY_FILES']: copy_split = copy_file.split(":") if len(copy_split) != 2: @@ -199,11 +242,21 @@ def preprocess(i): f.write(EOL+'# Download CM repo for scripts' + EOL) - # Add possibility to force rebuild with some extra flag for the repository - x = env.get('CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO','') - if x!='': x=' '+x - - f.write('RUN cm pull repo ' + cm_mlops_repo + cm_mlops_repo_branch_string + x + EOL) + if use_copy_repo: + docker_repo_dest = "/home/cmuser/CM/repos/mlcommons@cm4mlops" + f.write(f'COPY --chown=cmuser:cm {relative_repo_path} {docker_repo_dest}' + EOL) + + f.write(EOL + '# Register CM repository' + EOL) + f.write('RUN cm pull repo --url={} --quiet'.format(docker_repo_dest) + EOL) + f.write(EOL) + + + else: + # Use cm pull repo as before + x = env.get('CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO','') + if x!='': x=' '+x + + f.write('RUN cm pull repo ' + cm_mlops_repo + cm_mlops_repo_branch_string + x + EOL) # Check extra repositories x = env.get('CM_DOCKER_EXTRA_CM_REPOS','') From f7ccd2cb5a81dc88764816924849d789000ef66f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 18:43:10 +0000 Subject: [PATCH 520/658] Fix pstree version detection on macos --- script/get-generic-sys-util/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index b020841eac..c429066b6e 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -10,6 +10,8 @@ def preprocess(i): state = i['state'] automation = i['automation'] + if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree" #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] From c0e92ac1ee924f75895b9aec40bb4460b32626db Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:11:54 +0000 Subject: [PATCH 521/658] Fix pstree version detection on macos --- script/get-generic-sys-util/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index c429066b6e..8e3598f1ec 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,7 +11,8 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree" + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree:" + env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] From 5df4e4a54a4405c685d1a4c8614b41e65bfe059f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:16:30 +0000 Subject: [PATCH 522/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index dfabebeb9f..6b671db54f 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -579,10 +579,7 @@ } ], "env": { - "CM_SYS_UTIL_NAME": "linux-tools", - "CM_SYS_UTIL_VERSION_CMD": "linux-tools --version", - "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_NAME": "linux-tools" }, "state": { "linux-tools": { From 93e8d947d81dbbfa24c3cc15de00cd7be499accb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:23:48 +0000 Subject: [PATCH 523/658] Update _cm.json | Dont do version check for numactl --- script/get-generic-sys-util/_cm.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 6b671db54f..f0a79e3b22 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -671,8 +671,6 @@ ], "env": { "CM_SYS_UTIL_NAME": "numactl", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { From 68fec0794f91aa2e912c4036d7e357144b1339e1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:36:39 +0000 Subject: [PATCH 524/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index f0a79e3b22..d0a1320eca 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -75,7 +75,10 @@ "group": "mode", "env": { "CM_GENERIC_SYS_UTIL_RUN_MODE": "install" - } + }, + "new_env_keys": [ + "CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED" + ] }, "cmake": { "env": { @@ -332,10 +335,7 @@ }, "libffi8": { "env": { - "CM_SYS_UTIL_NAME": "libffi8", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi8", - "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libffi8" }, "state": { "libffi8": { From 00f3d26e85182fa584dd535035900013f7d93a0d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:40:49 +0000 Subject: [PATCH 525/658] Remove version detection for libffi --- script/get-generic-sys-util/_cm.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index d0a1320eca..e63a05ebba 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -290,10 +290,7 @@ }, "libffi": { "env": { - "CM_SYS_UTIL_NAME": "libffi", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libffi" }, "state": { "libffi": { From bc9e102ee7cbf31b16af33c3c947fd2b4e587ba5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 01:11:59 +0530 Subject: [PATCH 526/658] Fix onnx version for nvidia mlperf inference retinanet --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index a997ef8156..9102732e31 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -435,7 +435,7 @@ variations: - tags: get,generic-python-lib,_pycocotools - tags: get,generic-python-lib,_onnx-graphsurgeon - tags: get,generic-python-lib,_package.onnx - version: 1.13.1 + version: 1.14.1 - tags: get,generic-python-lib,_package.sympy sdxl: From fa071a00d2fa1fa1a1846b62a92f22b1c18ba8da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:56:25 +0000 Subject: [PATCH 527/658] Fix pstree version detect on macos --- script/get-generic-sys-util/customize.py | 5 +++-- script/get-generic-sys-util/detect.sh | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 8e3598f1ec..65c41eaae1 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -13,12 +13,13 @@ def preprocess(i): if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree:" env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" + #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": - if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '': + if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '': r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] != 0: #detection failed, do install via prehook_deps print("detection failed, going for installation") @@ -131,7 +132,7 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 330a10806f..90ef1312ba 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,6 +1,8 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" ]]; then - eval "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" + cmd="${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" + echo $cmd + eval $cmd test $? -eq 0 || exit $? else if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then From 62fe3a6bbfcd7c1d92231d03e99724fa87cb35b7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 20:04:58 +0000 Subject: [PATCH 528/658] Fix version check for get-generic-sys-util install failures --- script/get-generic-sys-util/customize.py | 4 ++-- script/get-generic-sys-util/run.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 65c41eaae1..0d15e14d2c 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,7 +11,7 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree:" + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1 | grep pstree:" env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set @@ -132,7 +132,7 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/run.sh index 7e8afaf10a..de14fe9d03 100644 --- a/script/get-generic-sys-util/run.sh +++ b/script/get-generic-sys-util/run.sh @@ -8,6 +8,7 @@ if ! eval "$cmd"; then echo "Command failed with status $?" if [[ "${CM_TMP_FAIL_SAFE}" == 'yes' ]]; then # Exit safely if fail-safe is enabled + echo "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED=yes" > tmp-run.out echo "Fail-safe is enabled, exiting with status 0" exit 0 else From 12b70e646c7fdacd7fec035eb5998e141153b3ce Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 20:26:57 +0000 Subject: [PATCH 529/658] Fix version detection only for install mode --- script/get-generic-sys-util/customize.py | 7 ++++--- script/get-generic-sys-util/detect.sh | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 0d15e14d2c..df852f624d 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,8 +11,9 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1 | grep pstree:" + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1" env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" + env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 0 #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': @@ -132,10 +133,10 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 90ef1312ba..2c35837992 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,4 +1,5 @@ #!/bin/bash + if [[ -n "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" ]]; then cmd="${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" echo $cmd From 9711d26db222fc254aa4ff9cc755c47d9e70ea1e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 02:11:58 +0530 Subject: [PATCH 530/658] Support detached mode for nvidia-mlperf-inference-gptj --- script/get-ml-model-gptj/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index ab475f0454..b3c61086d1 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -24,6 +24,8 @@ def preprocess(i): elif env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': i['run_script_input']['script_name'] = 'run-nvidia' + if str(env.get('CM_DOCKER_DETACHED_MODE','')).lower() in ['yes', 'true', "1"]: + env['DOCKER_RUN_OPTS'] = "--rm -dt --ipc=host --ulimit memlock=-1 --ulimit stack=67108864" gpu_arch = int(float(env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * 10) env['CM_GPU_ARCH'] = gpu_arch env['CM_TMP_REQUIRE_DOWNLOAD'] = 'no' From 4d7195b8f664ac52c64b5fa81b7ffbfa2f63ff1b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 21:54:55 +0000 Subject: [PATCH 531/658] Cleanup for get-generic-sys-util --- script/get-generic-sys-util/customize.py | 22 +++++++++++++------ .../{run.sh => install.sh} | 0 2 files changed, 15 insertions(+), 7 deletions(-) rename script/get-generic-sys-util/{run.sh => install.sh} (100%) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index df852f624d..25ba9ec575 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,14 +11,17 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1" - env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" - env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 0 + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree | grep pstree: > tmp-ver.out 2>&1" + env['CM_SYS_UTIL_VERSION_RE'] = r"(?:stable\s+)(\d+\.\d+(?:\.\d+)?)" + env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 1 #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] + if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "install": + i['run_script_input']['name'] = "install" + if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '': r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) @@ -27,9 +30,11 @@ def preprocess(i): env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" return {'return': 0} else: #detection is successful, no need to install + #print("detection success") env['CM_SYS_UTIL_INSTALL_CMD'] = "" return {'return': 0} else: #No detction command available, just install + #print("No detection possible, going for installation") env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" return {'return': 0} @@ -121,6 +126,7 @@ def detect_version(i): 'group_number': group_number, 'env_key': version_env_key, 'which_env': env}) + if r['return'] >0: return r version = r['version'] @@ -133,11 +139,13 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] - r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: - return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} + + if (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == ''): + r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: + return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/install.sh similarity index 100% rename from script/get-generic-sys-util/run.sh rename to script/get-generic-sys-util/install.sh From 92391b3a8960b3833c74c9e1986cf08d6847bfc5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:14:36 +0000 Subject: [PATCH 532/658] Cleanup for get-generic-sys-util --- script/get-generic-sys-util/customize.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 25ba9ec575..e6aac8b763 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -20,7 +20,7 @@ def preprocess(i): env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "install": - i['run_script_input']['name'] = "install" + i['run_script_input']['script_name'] = "install" if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '': @@ -139,13 +139,12 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and env.get(version_env_key, '') == '' and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] - if (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == ''): - r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: - return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} + r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: + return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) From 196c9b800fd9e035e4b864918a5311688858b986 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:21:50 +0000 Subject: [PATCH 533/658] No version detection for psmisc --- script/get-generic-sys-util/customize.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index e6aac8b763..673cb7f89f 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -10,11 +10,6 @@ def preprocess(i): state = i['state'] automation = i['automation'] - if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree | grep pstree: > tmp-ver.out 2>&1" - env['CM_SYS_UTIL_VERSION_RE'] = r"(?:stable\s+)(\d+\.\d+(?:\.\d+)?)" - env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 1 - #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] From f5251950d17991440b7b0448836b57dd1dbb533e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:23:02 +0000 Subject: [PATCH 534/658] No version detection for psmisc --- script/get-generic-sys-util/_cm.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e63a05ebba..d27ea3720e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -713,11 +713,7 @@ }, "psmisc": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", - "CM_SYS_UTIL_NAME": "psmisc", - "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "psmisc" }, "state": { "psmisc": { From 94ac51c299af8fd49581f354eebea836615d062e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:30:52 +0000 Subject: [PATCH 535/658] Fix tmp-run-env.out name --- script/get-generic-sys-util/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/install.sh b/script/get-generic-sys-util/install.sh index de14fe9d03..145d0b7451 100644 --- a/script/get-generic-sys-util/install.sh +++ b/script/get-generic-sys-util/install.sh @@ -8,7 +8,7 @@ if ! eval "$cmd"; then echo "Command failed with status $?" if [[ "${CM_TMP_FAIL_SAFE}" == 'yes' ]]; then # Exit safely if fail-safe is enabled - echo "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED=yes" > tmp-run.out + echo "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED=yes" > tmp-run-env.out echo "Fail-safe is enabled, exiting with status 0" exit 0 else From 2da6bef79767e66ef19917b537dc88981d34e998 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:50:00 +0000 Subject: [PATCH 536/658] Update _cm.json | Fix bzip2 version cmd --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index d27ea3720e..1d5f590b2b 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -259,7 +259,7 @@ "libbz2-dev": { "env": { "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD_OVERRIDE": "bzcat --version > tmp_ver.out 2>&1 | grep bzip", + "CM_SYS_UTIL_VERSION_CMD_OVERRIDE": "bzcat --version 2>&1 | grep bzip > tmp-ver.out", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, From 000a799893e63514f80101891c4cdd1daaa5ee68 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:05:22 +0000 Subject: [PATCH 537/658] Fix tmp-run-env.out name --- script/get-generic-sys-util/_cm.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 6fc9c004ee..415def53ca 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -43,9 +43,6 @@ "detect": { "group": "mode", "default": true, - "names": [ - "detect-sys-util" - ], "env": { "CM_GENERIC_SYS_UTIL_RUN_MODE": "detect" }, @@ -75,7 +72,10 @@ "group": "mode", "env": { "CM_GENERIC_SYS_UTIL_RUN_MODE": "install" - } + }, + "new_env_keys": [ + "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED" + ] }, "cmake": { "env": { @@ -105,7 +105,10 @@ }, "g++-11": { "env": { - "CM_SYS_UTIL_NAME": "g++11" + "CM_SYS_UTIL_NAME": "g++11", + "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++11": { From 4795f0b651487829156da64673c73e50c736cd2e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:14:36 +0000 Subject: [PATCH 538/658] Skip g++11 version detection for ubuntu 20.04 --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 415def53ca..90f7107955 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -108,7 +108,8 @@ "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE": "yes" }, "state": { "g++11": { From dd2c715011007c856025287029495c3a99559026 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:20:24 +0000 Subject: [PATCH 539/658] Skip g++11 version detection for ubuntu 20.04 --- script/get-generic-sys-util/customize.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 673cb7f89f..80cdd1e96b 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -141,16 +141,17 @@ def postprocess(i): if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} - r = detect_version(i) + elif r['return'] == 0:: + r = detect_version(i) - if r['return'] >0: return r + if r['return'] >0: return r - version = r['version'] + version = r['version'] - env[version_env_key] = version + env[version_env_key] = version - #Not used now - env['CM_GENERIC_SYS_UTIL_'+env['CM_SYS_UTIL_NAME'].upper()+'_CACHE_TAGS'] = 'version-'+version + #Not used now + env['CM_GENERIC_SYS_UTIL_'+env['CM_SYS_UTIL_NAME'].upper()+'_CACHE_TAGS'] = 'version-'+version if env.get(version_env_key, '') == '': env[version_env_key] = "undetected" From f1c9e31e2e8dd41dbbd63ee44c4bda90157e1f2c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:22:34 +0000 Subject: [PATCH 540/658] Skip g++11 version detection for ubuntu 20.04 --- script/get-generic-sys-util/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 80cdd1e96b..ae707684d5 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -141,7 +141,7 @@ def postprocess(i): if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} - elif r['return'] == 0:: + elif r['return'] == 0: r = detect_version(i) if r['return'] >0: return r From 9990d9886bc7ab6bdec05c0bcd840ab3b50b2a7c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:40:08 +0000 Subject: [PATCH 541/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index b0b561d2bf..afdf341dd9 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -133,7 +133,8 @@ "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE": "yes" }, "state": { "g++12": { From 432a6ed4811a5d3d5aa6c241850abfba471d903b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:47:22 +0000 Subject: [PATCH 542/658] Remove version detect for libnuma-dev --- script/get-generic-sys-util/_cm.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index afdf341dd9..98d07f5c0c 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -388,10 +388,7 @@ }, "libmkl-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmkl-dev", - "CM_SYS_UTIL_VERSION_CMD": "ldconfig -p | grep libmkl", - "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_NAME": "libmkl-dev" }, "state": { "libmkl-dev": { @@ -439,10 +436,7 @@ }, "libnuma-dev": { "env": { - "CM_SYS_UTIL_NAME": "libnuma-dev", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libnuma-dev" }, "state": { "libnuma-dev": { From 4478fde1ed0f508f86b67cbb5f91087f488cfc76 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:52:57 +0000 Subject: [PATCH 543/658] Use berkeley link for imagenet-aux by default --- script/get-dataset-imagenet-aux/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-imagenet-aux/_cm.json b/script/get-dataset-imagenet-aux/_cm.json index 97e10571cd..7093e8b2b6 100644 --- a/script/get-dataset-imagenet-aux/_cm.json +++ b/script/get-dataset-imagenet-aux/_cm.json @@ -39,6 +39,7 @@ }, "from.berkeleyvision": { "group": "download-source", + "default": true, "base": [ "2012" ], @@ -51,7 +52,6 @@ }, "from.dropbox": { "group": "download-source", - "default": true, "base": [ "2012" ], From d5554b021ce155dc98260eaa04981b511a75daf1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:09:46 +0000 Subject: [PATCH 544/658] Update docker repo and branch in test --- tests/script/test_docker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/test_docker.py b/tests/script/test_docker.py index 5a02c932ed..6b95143aae 100644 --- a/tests/script/test_docker.py +++ b/tests/script/test_docker.py @@ -12,7 +12,8 @@ 'image_name':'cm-script-app-image-classification-onnx-py', 'env': { 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', - 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', + 'CM_MLOPS_REPO': 'mlcommons@cm4mlops', + 'CM_MLOPS_REPO_BRANCH': 'mlperf-inference', 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04' }, 'quiet': 'yes' From c54eaaa0c48692edaa418e5fbc2615c61fa72693 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:12:40 +0000 Subject: [PATCH 545/658] Remove ntpdate version detect --- script/get-generic-sys-util/_cm.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 98d07f5c0c..538851ac04 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -629,10 +629,7 @@ }, "ntpdate": { "env": { - "CM_SYS_UTIL_NAME": "ntpdate", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'", - "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "ntpdate" }, "state": { "ntpdate": { From 9739fafe5f5eaf1dab8c0fc35e75fd6f045c62a3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:19:01 +0000 Subject: [PATCH 546/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 538851ac04..a569e6e1b9 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -423,7 +423,7 @@ "CM_SYS_UTIL_NAME": "libncurses_dev", "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libncurses_dev": { From 3c7904bdff12ddef9ba34e88c95ac1464ee67a1f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:24:44 +0000 Subject: [PATCH 547/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a569e6e1b9..e231f49bce 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -423,7 +423,8 @@ "CM_SYS_UTIL_NAME": "libncurses_dev", "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1, + "CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE": "yes" }, "state": { "libncurses_dev": { From 9501b18db588b779ba4372c88015942ec3c6c342 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:33:17 +0000 Subject: [PATCH 548/658] Update _cm.json | Remove failing version detects --- script/get-generic-sys-util/_cm.json | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e231f49bce..3135863a59 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -11,14 +11,6 @@ "deps": [ { "tags": "detect,os" - }, - { - "tags": "get,generic-sys-util,_pkg-config,_detect", - "skip_if_env": { - "CM_SYS_UTIL_NAME": [ - "pkg_config" - ] - } } ], "new_env_keys": [ @@ -615,10 +607,7 @@ }, "nlohmann-json3-dev": { "env": { - "CM_SYS_UTIL_NAME": "nlohmann_json3_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion nlohmann_json", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "nlohmann_json3_dev" }, "state": { "nlohmann_json3_dev": { @@ -719,10 +708,7 @@ }, "rapidjson-dev": { "env": { - "CM_SYS_UTIL_NAME": "rapidjson-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion RapidJSON", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "rapidjson-dev" }, "state": { "rapidjson-dev": { @@ -904,10 +890,7 @@ }, "zlib": { "env": { - "CM_SYS_UTIL_NAME": "zlib", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "zlib" }, "state": { "zlib": { @@ -919,10 +902,7 @@ }, "zlib1g-dev": { "env": { - "CM_SYS_UTIL_NAME": "zlib1g_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "zlib1g_dev" }, "state": { "zlib1g_dev": { From c9edadaa7c7e104258628178d1d807dd5093310e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:40:09 +0000 Subject: [PATCH 549/658] Removed pkg detection from pkg_config --- script/get-generic-sys-util/_cm.json | 56 ++++++---------------------- 1 file changed, 11 insertions(+), 45 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 3135863a59..023bfabb01 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -181,11 +181,7 @@ }, "gflags-dev": { "env": { - "CM_SYS_UTIL_NAME": "gflags-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gflags", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 - + "CM_SYS_UTIL_NAME": "gflags-dev" }, "state": { "gflags-dev": { @@ -216,10 +212,7 @@ }, "glog-dev": { "env": { - "CM_SYS_UTIL_NAME": "glog-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libglog", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "glog-dev" }, "state": { "glog-dev": { @@ -293,10 +286,7 @@ }, "libffi-dev": { "env": { - "CM_SYS_UTIL_NAME": "libffi_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libffi_dev" }, "state": { "libffi_dev": { @@ -349,10 +339,7 @@ }, "libgmock-dev": { "env": { - "CM_SYS_UTIL_NAME": "libgmock-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gmock", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libgmock-dev" }, "state": { "libgmock-dev": { @@ -394,10 +381,7 @@ }, "libmpfr-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmpfr-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion mpfr", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libmpfr-dev" }, "state": { "libmpfr-dev": { @@ -443,10 +427,7 @@ }, "libpci-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpci-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpci", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libpci-dev" }, "state": { "libpci-dev": { @@ -460,10 +441,7 @@ }, "libpng-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpng-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpng", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libpng-dev" }, "state": { "libpng-dev": { @@ -477,10 +455,7 @@ }, "libre2-dev": { "env": { - "CM_SYS_UTIL_NAME": "libre2-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion re2", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libre2-dev" }, "state": { "libre2-dev": { @@ -494,10 +469,7 @@ }, "libreadline-dev": { "env": { - "CM_SYS_UTIL_NAME": "libreadline_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion readline", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libreadline_dev" }, "state": { "libreadline_dev": { @@ -510,10 +482,7 @@ }, "libsqlite3-dev": { "env": { - "CM_SYS_UTIL_NAME": "libsqlite3_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion sqlite3", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libsqlite3_dev" }, "state": { "libsqlite3_dev": { @@ -541,10 +510,7 @@ }, "libudev-dev": { "env": { - "CM_SYS_UTIL_NAME": "libudev-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libudev", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libudev-dev" }, "state": { "libudev-dev": { From 63859985ee17d2a9227345fd665fa528d5c562e2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 01:13:51 +0000 Subject: [PATCH 550/658] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 023bfabb01..100dfbc4b8 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -774,7 +774,8 @@ "CM_SYS_UTIL_NAME": "transmission", "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes" }, "state": { "transmission": { From cb0c19a777f5d4048d082dad3f4e3c5b480002ec Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:50:29 +0530 Subject: [PATCH 551/658] Generalised the code --- .../test-mlperf-inference-abtf-poc.yml | 67 ++++--------------- 1 file changed, 13 insertions(+), 54 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index bb8403ed9f..5c1b3c750b 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -16,13 +16,24 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, ubuntu-20.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-20.04, ubuntu-24.04, macos-latest, macos-13, windows-latest] python-version: [ "3.8", "3.12" ] backend: [ "pytorch" ] implementation: [ "python" ] + extra-args: [ "--adr.compiler.tags=gcc" ] exclude: - os: ubuntu-24.04 python-version: "3.8" + - os: windows-latest + python-version: "3.8" + extra-args: "--adr.compiler.tags=gcc" + - os: macos-latest + python-version: "3.8" + - os: macos-13 + python-version: "3.8" + include: + - os: windows-latest + extra-args: "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" steps: - uses: actions/checkout@v3 @@ -35,58 +46,6 @@ jobs: pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm pull repo mlcommons@cm4abtf --branch=poc - - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker - run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v - - build2: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest, macos-13] - python-version: [ "3.12" ] - backend: [ "pytorch" ] - implementation: [ "python" ] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - cm pull repo mlcommons@cm4abtf --branch=poc - - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} - run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v - - build3: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-latest] - python-version: [ "3.12" ] - backend: [ "pytorch" ] - implementation: [ "python" ] - exclude: - - python-version: "3.8" - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - cm pull repo mlcommons@cm4abtf --branch=poc - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 -v + cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} -v From b206baf2ea7cb1b260c43fcceb395b334aa7a49e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:58:18 +0530 Subject: [PATCH 552/658] split exclude condition --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 5c1b3c750b..8a4ef20171 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -26,6 +26,7 @@ jobs: python-version: "3.8" - os: windows-latest python-version: "3.8" + - os: windows-latest extra-args: "--adr.compiler.tags=gcc" - os: macos-latest python-version: "3.8" From 893530e7797f54d25fd516cfd22a1c94b35bbd97 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:30:16 +0530 Subject: [PATCH 553/658] code clean --- automation/script/module_misc.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 210557af03..0a5f283ea8 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1932,9 +1932,6 @@ def docker(i): if env.get('+ CM_DOCKER_BUILD_ARGS', []) == []: env['+ CM_DOCKER_BUILD_ARGS'] = [] - if env.get('+ CM_DOCKER_RUN_ARGS', []) == []: - env['+ CM_DOCKER_RUN_ARGS'] = [] - for key in proxy_keys: if os.environ.get(key, '') != '': value = os.environ[key] @@ -1944,12 +1941,10 @@ def docker(i): docker_use_host_group_id = i.get('docker_use_host_group_id', docker_settings.get('use_host_group_id')) if str(docker_use_host_group_id).lower() not in ['false', 'no', '0'] and os.name != 'nt': env['+ CM_DOCKER_BUILD_ARGS'].append("{}={}".format('GID', '\\" $(id -g $USER) \\"')) - env['+ CM_DOCKER_RUN_ARGS'].append("{}={}".format('GID', '\\" $(id -g $USER) \\"')) docker_use_host_user_id = i.get('docker_use_host_user_id', docker_settings.get('use_host_user_id')) if str(docker_use_host_user_id).lower() not in ['false', 'no', '0'] and os.name != 'nt': env['+ CM_DOCKER_BUILD_ARGS'].append("{}={}".format('UID', '\\" $(id -u $USER) \\"')) - env['+ CM_DOCKER_RUN_ARGS'].append("{}={}".format('UID', '\\" $(id -u $USER) \\"')) docker_base_image = i.get('docker_base_image', docker_settings.get('base_image')) docker_os = i.get('docker_os', docker_settings.get('os', 'ubuntu')) From de29f11529aa56c361cd0bfafcc68e61f21a7ee1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:31:31 +0530 Subject: [PATCH 554/658] code clean --- script/run-docker-container/customize.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 5b4a1b1c05..00ab63c978 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -120,8 +120,6 @@ def postprocess(i): port_map_cmds = [] run_opts = '' - CM_DOCKER_RUN_ARGS = env.get('+ CM_DOCKER_RUN_ARGS', []) - if env.get('CM_DOCKER_PRE_RUN_COMMANDS', []): for pre_run_cmd in env['CM_DOCKER_PRE_RUN_COMMANDS']: run_cmds.append(pre_run_cmd) @@ -248,7 +246,7 @@ def postprocess(i): x2 = " && bash ) || bash" - CONTAINER="docker run " + x1 + " --entrypoint " + x + x + " " + CM_DOCKER_RUN_ARGS + " " + run_opts + " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + CONTAINER="docker run " + x1 + " --entrypoint " + x + x + " " + run_opts + " " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag CMD = CONTAINER + " bash -c " + x + run_cmd_prefix + run_cmd + x2 + x print ('') From 16a88afb679d19fc21171c419bf82e171f42ef5a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 11:55:25 +0000 Subject: [PATCH 555/658] Rename README.md to README-extra.md --- script/get-generic-sys-util/{README.md => README-extra.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename script/get-generic-sys-util/{README.md => README-extra.md} (99%) diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README-extra.md similarity index 99% rename from script/get-generic-sys-util/README.md rename to script/get-generic-sys-util/README-extra.md index bf09f83053..d8f0015ae8 100644 --- a/script/get-generic-sys-util/README.md +++ b/script/get-generic-sys-util/README-extra.md @@ -422,4 +422,4 @@ wkhtmltopdf 0.12.6 `pkg-config --modversion zlib` -1.2.11 \ No newline at end of file +1.2.11 From 356387b1f8ee5afc00735c881797c2c0182b627f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 12:15:21 +0000 Subject: [PATCH 556/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 032cb4624a..c7639d989a 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 From a45cca362a03c90408b8a5d786c569371ff4bdd3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 18:48:18 +0530 Subject: [PATCH 557/658] Fix nvidia mlperf inference gptj detached mode --- script/run-mlperf-inference-app/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index f0e1c02f5f..0eb089abdd 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -203,6 +203,7 @@ def preprocess(i): inp = {} if str(docker_dt).lower() in ["yes", "true", "1"]: env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'no' # turning it off for the first run and after that we turn it on + env['CM_DOCKER_DETACHED_MODE'] = 'yes' if env.get('CM_DOCKER_IMAGE_NAME', '') != '': docker_extra_input['docker_image_name'] = env['CM_DOCKER_IMAGE_NAME'] From 36e9badf09850c349286c1c38328c397843e0777 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 13:19:06 +0000 Subject: [PATCH 558/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index c7639d989a..acf7aa43c1 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "31 2 * * *" #to be adjusted + - cron: "24 13 * * *" #to be adjusted jobs: build_nvidia: From ed2b48ab484057ab7099ff7dfd2fba9026723c69 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 19:38:59 +0530 Subject: [PATCH 559/658] Update pythonpath for coco2014 accuracy check --- script/process-mlperf-accuracy/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 72e7aec9e8..4a6ba1634c 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -92,7 +92,7 @@ def preprocess(i): elif dataset == "coco2014": - env['+PYTHONPATH'] = [ os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools") ] + env['+PYTHONPATH'] = [ os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools") , os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", "fid") ] extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': From 77c8d2586850cd5affa478fe6101712b1e1b235e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 14:13:50 +0000 Subject: [PATCH 560/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 40f563f460..698c0a2240 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -2,7 +2,7 @@ name: MLPerf inference SDXL (SCC) on: schedule: - - cron: "5 2 * * *" + - cron: "20 14 * * *" jobs: build_reference: From ecac7e90e4989ed31e3d01cc14a9674dbd8a0792 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 22:38:11 +0530 Subject: [PATCH 561/658] Added docker detatched option --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 8a4ef20171..56f5768e1b 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -20,6 +20,7 @@ jobs: python-version: [ "3.8", "3.12" ] backend: [ "pytorch" ] implementation: [ "python" ] + docker: [ "", " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" ] extra-args: [ "--adr.compiler.tags=gcc" ] exclude: - os: ubuntu-24.04 @@ -34,6 +35,7 @@ jobs: python-version: "3.8" include: - os: windows-latest + docker: "" extra-args: "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" steps: @@ -49,4 +51,4 @@ jobs: cm pull repo mlcommons@cm4abtf --branch=poc - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} -v + cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v From 468264d8e32215dd5f4937d481f53edb3d8bd3df Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 17:30:48 +0000 Subject: [PATCH 562/658] Update test-mlperf-inference-abtf-poc.yml | Enable docker on macos and windows --- .../test-mlperf-inference-abtf-poc.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 56f5768e1b..ea07391198 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -49,6 +49,45 @@ jobs: pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm pull repo mlcommons@cm4abtf --branch=poc + + - name: Install Docker on macos + if: runner.os == 'macOS' + run: | + brew update + brew install --cask docker + + - name: Start Docker Daemon on macos + if: runner.os == 'macOS' + run: | + open /Applications/Docker.app + # Wait until Docker daemon is running + while ! docker system info > /dev/null 2>&1; do + echo "Waiting for Docker to start..." + sleep 5 + done + echo "Docker is up and running" + + - name: Install Docker Desktop on Windows + if: runner.os == 'Windows' + run: | + choco install docker-desktop --no-progress -y + + - name: Start Docker Desktop on Windows + if: runner.os == 'Windows' + run: | + Start-Process 'C:\Program Files\Docker\Docker\Docker Desktop.exe' + # Wait until Docker daemon is running + $retryCount = 0 + while (!(docker info) -and ($retryCount -lt 10)) { + Write-Output "Waiting for Docker to start..." + Start-Sleep -Seconds 10 + $retryCount++ + } + if ($retryCount -ge 10) { + throw "Docker failed to start" + } + Write-Output "Docker is up and running" + - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v From a9375a41e83a23a8b279207c7bf779caf1211c13 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 17:37:44 +0000 Subject: [PATCH 563/658] Update test-mlperf-inference-abtf-poc.yml --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index ea07391198..23bf019098 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -21,7 +21,7 @@ jobs: backend: [ "pytorch" ] implementation: [ "python" ] docker: [ "", " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" ] - extra-args: [ "--adr.compiler.tags=gcc" ] + extra-args: [ "--adr.compiler.tags=gcc", "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" ] exclude: - os: ubuntu-24.04 python-version: "3.8" @@ -35,7 +35,6 @@ jobs: python-version: "3.8" include: - os: windows-latest - docker: "" extra-args: "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" steps: From e3e4f9f945df5311a677e97daa59c4b9aadd4e5f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 18:00:01 +0000 Subject: [PATCH 564/658] Update test-mlperf-inference-abtf-poc.yml --- .../test-mlperf-inference-abtf-poc.yml | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 23bf019098..2d586d2861 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -36,6 +36,8 @@ jobs: include: - os: windows-latest extra-args: "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" + docker: "" + # windows docker image is not supported in CM yet steps: - uses: actions/checkout@v3 @@ -59,11 +61,26 @@ jobs: if: runner.os == 'macOS' run: | open /Applications/Docker.app - # Wait until Docker daemon is running + echo "Starting Docker, this may take a while..." + + # Set max attempts and initial wait time + MAX_ATTEMPTS=20 + WAIT_TIME=5 + + # Loop until Docker daemon is up or max attempts reached + attempt=1 while ! docker system info > /dev/null 2>&1; do - echo "Waiting for Docker to start..." - sleep 5 + echo "Attempt $attempt: Waiting for Docker to start..." + sleep $WAIT_TIME + attempt=$((attempt + 1)) + WAIT_TIME=$((WAIT_TIME * 2)) # Exponential backoff + + if [ $attempt -gt $MAX_ATTEMPTS ]; then + echo "Docker failed to start within the timeout period" + exit 1 + fi done + echo "Docker is up and running" - name: Install Docker Desktop on Windows From 0a4fbdc8cadbd70b9ddf1577d7f03d71c9d02499 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 18:07:56 +0000 Subject: [PATCH 565/658] Update test-mlperf-inference-abtf-poc.yml --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 2d586d2861..356fd0913b 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -28,16 +28,13 @@ jobs: - os: windows-latest python-version: "3.8" - os: windows-latest - extra-args: "--adr.compiler.tags=gcc" + extra-args: "--adr.compiler.tags=gcc" + docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" + # windows docker image is not supported in CM yet - os: macos-latest python-version: "3.8" - os: macos-13 python-version: "3.8" - include: - - os: windows-latest - extra-args: "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" - docker: "" - # windows docker image is not supported in CM yet steps: - uses: actions/checkout@v3 From 8127ab51cec3ba8b2f1cc48cf536267bc4eff12f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 18:10:13 +0000 Subject: [PATCH 566/658] Update test-mlperf-inference-abtf-poc.yml --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 356fd0913b..8328d150ac 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -66,7 +66,7 @@ jobs: # Loop until Docker daemon is up or max attempts reached attempt=1 - while ! docker system info > /dev/null 2>&1; do + while ! docker --version > /dev/null 2>&1; do echo "Attempt $attempt: Waiting for Docker to start..." sleep $WAIT_TIME attempt=$((attempt + 1)) From 5aa7fb5349b9ad022dedb6c0a452d4b2a6f5955b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 18:19:34 +0000 Subject: [PATCH 567/658] Update test-mlperf-inference-abtf-poc.yml | Turn of docker runs on macos --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 8328d150ac..f6a13bd6d2 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -29,12 +29,17 @@ jobs: python-version: "3.8" - os: windows-latest extra-args: "--adr.compiler.tags=gcc" + - os: windows-latest docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" # windows docker image is not supported in CM yet - os: macos-latest python-version: "3.8" - os: macos-13 python-version: "3.8" + - os: macos-latest + docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" + - os: macos-13 + docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" steps: - uses: actions/checkout@v3 From 18f46a8763fe6e202601688ea43a8a5e23feec6a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 20:22:31 +0000 Subject: [PATCH 568/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index acf7aa43c1..3229f21511 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "24 13 * * *" #to be adjusted + - cron: "29 20 * * *" #to be adjusted jobs: build_nvidia: @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 From 232a22cd38eb268a9e6423ba3c5a7f00eff8a7f7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:41:35 +0530 Subject: [PATCH 569/658] initial commit - model download to host --- script/app-mlperf-inference/_cm.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 928709a8f4..a1b3420097 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -810,6 +810,15 @@ variations: CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: - 'yes' tags: get,ml-model,llama2 + + llama2-70b_,amd: + docker: + image_name: mlperf-inference-amd-python-implementation-llama2-70b + deps: + - enable_if_env: + CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: + - 'yes' + tags: get,ml-model,llama2,_amd,_pytorch mixtral-8x7b: group: From 0564d6b1c2c96531c4255665ca1502a803fadbbb Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:47:41 +0530 Subject: [PATCH 570/658] safe tensor path -> new env keys --- script/get-ml-model-llama2/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index 8c80521130..d980df8e0c 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -17,7 +17,8 @@ "new_env_keys": [ "CM_ML_MODEL_*", "LLAMA2_CHECKPOINT_PATH", - "CM_NVIDIA_TP_SIZE" + "CM_NVIDIA_TP_SIZE", + "CM_LLAMA2_FINAL_SAFE_TENSORS_PATH" ], "prehook_deps": [ { From f3b6b6f5027f29b1a4d034a9f074f6a0ea772951 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:54:29 +0530 Subject: [PATCH 571/658] updated mounts for amd-llama2 --- script/app-mlperf-inference/_cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index a1b3420097..9d1c37b5fa 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -814,6 +814,8 @@ variations: llama2-70b_,amd: docker: image_name: mlperf-inference-amd-python-implementation-llama2-70b + mounts: + - "${{ CM_LLAMA2_FINAL_SAFE_TENSORS_PATH }}:${{ CM_LLAMA2_FINAL_SAFE_TENSORS_PATH }" deps: - enable_if_env: CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: From e743110aeb1d5a6da9b1e38c0650eb58228652ad Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:55:22 +0530 Subject: [PATCH 572/658] Updated env - llama2 model --- script/app-mlperf-inference-amd/run-llama2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-amd/run-llama2.sh b/script/app-mlperf-inference-amd/run-llama2.sh index 4cf9f4f2be..10f36f8ca2 100644 --- a/script/app-mlperf-inference-amd/run-llama2.sh +++ b/script/app-mlperf-inference-amd/run-llama2.sh @@ -16,7 +16,7 @@ export VLLM_FP8_REDUCE_CONV=1 export HARNESS_DISABLE_VLLM_LOGS=1 export VLLM_LOGGING_LEVEL=ERROR -MODEL_PATH=${CM_ML_MODEL_LLAMA2_FILE_WITH_PATH:-/data/llm/llama2-70b-chat/} +MODEL_PATH=${LLAMA2_CHECKPOINT_PATH:-/data/llm/llama2-70b-chat/} DATASET_PATH=${CM_DATASET_OPENORCA_PREPROCESSED_PATH:-/data/open_orca/open_orca_gpt4_tokenized_llama.sampled_24576.pkl.gz} QUANTIZED_WEIGHTS_PATH=${CM_LLAMA2_FINAL_SAFE_TENSORS_PATH:-quantized/quark_share/modelzoo/llama2_70b_wfp8_afp8_ofp8_nomerge/json-safetensors/llama.safetensors} QUANTIZATION_PARAM_PATH=${QUANTIZATION_PARAM_PATH:-/app/kv_cache_scales.json} From f10267fdb358ef5681fcb1e9d9388e97c4ee73f0 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:58:52 +0530 Subject: [PATCH 573/658] handled model download in host --- script/app-mlperf-inference-amd/_cm.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-amd/_cm.yaml b/script/app-mlperf-inference-amd/_cm.yaml index 089a1be1eb..f05d4753e0 100644 --- a/script/app-mlperf-inference-amd/_cm.yaml +++ b/script/app-mlperf-inference-amd/_cm.yaml @@ -244,8 +244,10 @@ variations: llama2-70b_: deps: - tags: get,preprocessed,dataset,openorca,_mlc,_validation - - tags: get,ml-model,llama2,_fp32,_pytorch - tags: get,ml-model,llama2,_amd,_pytorch + skip_if_env: + CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: + - 'yes' - tags: get,preprocessed,dataset,openorca,_mlc,_validation - tags: download,file,_url.https://github.com/vllm-project/vllm/blob/38c4b7e863570a045308af814c72f4504297222e/tests/fp8_kv/llama2-70b-fp8-kv/kv_cache_scales.json extra_cache_tags: llama2-scales,kv-cache From c20c1759774ca36aaf05761d5e11d64a39da3876 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:09:56 +0530 Subject: [PATCH 574/658] added compressed tensors module --- script/app-mlperf-inference-amd/_cm.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/app-mlperf-inference-amd/_cm.yaml b/script/app-mlperf-inference-amd/_cm.yaml index f05d4753e0..e4982b74ff 100644 --- a/script/app-mlperf-inference-amd/_cm.yaml +++ b/script/app-mlperf-inference-amd/_cm.yaml @@ -243,6 +243,9 @@ variations: llama2-70b_: deps: + - tags: get,generic-python-lib,_package.compressed_tensors + names: + - compressed_tensors - tags: get,preprocessed,dataset,openorca,_mlc,_validation - tags: get,ml-model,llama2,_amd,_pytorch skip_if_env: From c60d7487860b96fb64f56dd37d0ef190bce138e1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:15:36 +0530 Subject: [PATCH 575/658] Added compressed tensors support --- script/get-ml-model-llama2/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index d980df8e0c..07332f09c9 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -189,6 +189,9 @@ }, { "tags": "get,generic-python-lib,_torch" + }, + { + "tags": "get,generic-python-lib,_compressed_tensors" } ] }, From 3228f881c157c11d6263b12cc067a89c5d886c46 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:16:14 +0530 Subject: [PATCH 576/658] bug fix --- script/get-ml-model-llama2/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index 07332f09c9..839cba1a0d 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -191,7 +191,7 @@ "tags": "get,generic-python-lib,_torch" }, { - "tags": "get,generic-python-lib,_compressed_tensors" + "tags": "get,generic-python-lib,_package.compressed_tensors" } ] }, From 82c6af20bf4b338f05c9b3289fb2524d31bac694 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 5 Nov 2024 16:22:49 +0530 Subject: [PATCH 577/658] Update PYTHONPATH for mlperf inference stable diffusion --- script/app-mlperf-inference-mlcommons-python/customize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index f227c07644..97e7000235 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -290,6 +290,10 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio elif "stable-diffusion-xl" in env['CM_MODEL']: env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image") + if env.get('+PYTHONPATH', '') == '': + env['+PYTHONPATH'] = [] + env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", "fid") ]) + backend = env['CM_MLPERF_BACKEND'] device = env['CM_MLPERF_DEVICE'] if env['CM_MLPERF_DEVICE'] not in [ "gpu", "rocm" ] else "cuda" max_batchsize = env.get('CM_MLPERF_LOADGEN_MAX_BATCHSIZE', '1') From 324df6cb56c463a4d0ff216076cfbc7ff13d2daf Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:16:51 +0530 Subject: [PATCH 578/658] code clean --- .../customize.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index b370370f50..a4bb876622 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -4,6 +4,14 @@ from os.path import exists import shutil +def clean_dir(dir_name): + for item in os.listdir(dir_name): + item_path = os.path.join(dir_name, item) + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + def preprocess(i): os_info = i['os_info'] @@ -18,7 +26,7 @@ def preprocess(i): submission_processed = submission_dir + "_processed" if os.path.exists(submission_processed): - shutil.rmtree(submission_processed) + clean_dir(submission_processed) os.system("rm -rf " + submission_dir + "_processed") @@ -37,14 +45,8 @@ def postprocess(i): submission_processed = submission_dir + "_processed" shutil.copytree(submission_dir, submission_backup) - - # clean the submission directory - for item in os.listdir(submission_dir): - item_path = os.path.join(submission_dir, item) - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) + + clean_dir(submission_dir) # copy the files from submission processed directory to submission directory for item in os.listdir(submission_processed): From 569dc1200bdd798b60050632519792bcc810eaf9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:18:09 +0530 Subject: [PATCH 579/658] Handle dynamic env variables --- automation/script/module.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/automation/script/module.py b/automation/script/module.py index e383653d38..addaddd2eb 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3982,6 +3982,19 @@ def update_deps(self, i): return {'return':0} + ############################################################################## + def update_state_from_meta(self, meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys, new_state_keys, i): + """ + Updates state and env from meta + Args: + """ + + r = update_state_from_meta(meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys, new_state_keys, i) + if r['return']>0: + return r + + return {'return':0} + ############################################################################## def get_default_path_list(self, i): default_path_env_key = i.get('default_path_env_key', '') @@ -5235,6 +5248,11 @@ def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps if input_mapping: update_env_from_input_mapping(env, i['input'], input_mapping) + # handle dynamic env values + r = update_env_with_values(env) + if r['return']>0: + return r + # Possibly restrict this to within docker environment add_deps_info = meta.get('ad', i.get('ad', {})) #we need to see input here if not add_deps_info: From 18c284cec72f95f4fffb426ce09ddafd94fecc4e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:20:31 +0530 Subject: [PATCH 580/658] update state from meta --- automation/script/module_misc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 0a5f283ea8..400021594b 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1802,6 +1802,10 @@ def docker(i): state['docker'] = docker_settings add_deps_recursive = i.get('add_deps_recursive', {}) + r = script_automation.update_state_from_meta(meta, env, state, const, const_state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys = [], new_state_keys = [], i = i) + if r['return'] > 0: + return r + r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False) if r['return'] > 0: return r From 39bedf2fcea0283493ba1f1bc3544b14f81cb0dd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 5 Nov 2024 18:24:54 +0530 Subject: [PATCH 581/658] Update PYTHONPATH for mlperf inference stable diffusion --- script/app-mlperf-inference-mlcommons-python/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 97e7000235..0272c516f2 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -292,7 +292,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image") if env.get('+PYTHONPATH', '') == '': env['+PYTHONPATH'] = [] - env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", "fid") ]) + env['+PYTHONPATH'].append(os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", "fid")) backend = env['CM_MLPERF_BACKEND'] device = env['CM_MLPERF_DEVICE'] if env['CM_MLPERF_DEVICE'] not in [ "gpu", "rocm" ] else "cuda" From 659aeccaf3b5f2df6f10bb0c480174379f91ab66 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:28:01 +0530 Subject: [PATCH 582/658] Added submission base directory --- script/generate-mlperf-inference-submission/_cm.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index d4859f475a..e72916df61 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -53,6 +53,7 @@ "run_style": "CM_MLPERF_RUN_STYLE", "skip_truncation": "CM_SKIP_TRUNCATE_ACCURACY", "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", + "submission_base_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", "clean": "CM_MLPERF_CLEAN_SUBMISSION_DIR", "hw_name": "CM_HW_NAME", "sw_notes_extra": "CM_MLPERF_SUT_SW_NOTES_EXTRA", @@ -72,8 +73,7 @@ }, "default_env": { "CM_RUN_MLPERF_ACCURACY": "on", - "CM_MLPERF_RUN_STYLE": "valid", - "CM_MLPERF_INFERENCE_SUBMISSION_PROCESSED_DIR": "<<>>_processed" + "CM_MLPERF_RUN_STYLE": "valid" }, "post_deps": [ { @@ -150,9 +150,8 @@ "cm pull repo" ], "mounts": [ - "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}", - "${{ CM_MLPERF_INFERENCE_SUBMISSION_PROCESSED_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_PROCESSED_DIR }}" + "${{ CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR }}", + "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" ], "extra_run_args": " --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined", "os": "ubuntu", From 536b9ffc401902833f7f15016ed785903dcb352e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:35:47 +0530 Subject: [PATCH 583/658] seperated inp through cm.access --- automation/script/module.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index addaddd2eb..7a47da86ce 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -5244,9 +5244,12 @@ def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps r4 = update_deps(posthook_deps, add_deps_info, True, env) if r1['return']>0 and r2['return']>0 and r3['return'] > 0 and r4['return'] > 0: return r1 + # i would have 'input' when called through cm.access + input_update_env = i.get('input', i) + input_mapping = meta.get('input_mapping', {}) if input_mapping: - update_env_from_input_mapping(env, i['input'], input_mapping) + update_env_from_input_mapping(env, input_update_env, input_mapping) # handle dynamic env values r = update_env_with_values(env) From 3f521c6435112d89aaeb402d8652ecbcc59ee6e0 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:37:49 +0530 Subject: [PATCH 584/658] fix typo --- script/generate-mlperf-inference-submission/_cm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index e72916df61..6f7e484bd0 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -53,7 +53,7 @@ "run_style": "CM_MLPERF_RUN_STYLE", "skip_truncation": "CM_SKIP_TRUNCATE_ACCURACY", "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", - "submission_base_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", + "submission_base_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", "clean": "CM_MLPERF_CLEAN_SUBMISSION_DIR", "hw_name": "CM_HW_NAME", "sw_notes_extra": "CM_MLPERF_SUT_SW_NOTES_EXTRA", @@ -140,7 +140,7 @@ "tags": "get,mlperf,inference,submission,dir,local", "names": "get-mlperf-inference-submission-dir", "skip_if_env": { - "CM_MLPERF_INFERENCE_SUBMISSION_DIR": [ + "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR": [ "on" ] } From 2377572b2ec9dd7e7082c6bf4c47122207df4623 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:46:50 +0530 Subject: [PATCH 585/658] Supports submission base dir --- .../customize.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 10fde422a2..3c03a2dda0 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -65,18 +65,19 @@ def generate_submission(i): env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] = os.path.join(user_home, "mlperf_submission") submission_dir = env.get('CM_MLPERF_INFERENCE_SUBMISSION_DIR', '') + if submission_dir == '': + submission_base_dir = env.get('CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR', '') + if submission_base_dir == '': + return {'return':1, 'error':f"Both CM_MLPERF_INFERENCE_SUBMISSION_DIR and CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR can not be empty!"} + else: + submission_dir = os.path.join(submission_base_dir, "mlperf_inference_submission") + env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'] = submission_dir if env.get('CM_MLPERF_CLEAN_SUBMISSION_DIR','')!='': print ('=================================================') print ('Cleaning {} ...'.format(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'])) - if os.path.exists(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): - # delete the content keeping the main folder intact - for item in os.listdir(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR']): - item_path = os.path.join(env['CM_MLPERF_INFERENCE_SUBMISSION_DIR'], item) - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) + if os.path.exists(submission_dir): + shutil.rmtree(submission_dir) print ('=================================================') if not os.path.isdir(submission_dir): From 5fabd94627dedabec5483c643b380db6a536ff11 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:50:04 +0530 Subject: [PATCH 586/658] code clean --- .../customize.py | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index a4bb876622..c5669338be 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -4,14 +4,6 @@ from os.path import exists import shutil -def clean_dir(dir_name): - for item in os.listdir(dir_name): - item_path = os.path.join(dir_name, item) - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - def preprocess(i): os_info = i['os_info'] @@ -26,7 +18,7 @@ def preprocess(i): submission_processed = submission_dir + "_processed" if os.path.exists(submission_processed): - clean_dir(submission_processed) + shutil.rmtree(submission_processed) os.system("rm -rf " + submission_dir + "_processed") @@ -45,16 +37,7 @@ def postprocess(i): submission_processed = submission_dir + "_processed" shutil.copytree(submission_dir, submission_backup) - - clean_dir(submission_dir) - - # copy the files from submission processed directory to submission directory - for item in os.listdir(submission_processed): - source = os.path.join(submission_processed, item) - destination = os.path.join(submission_dir, item) - if os.path.isdir(source): - shutil.copytree(source, destination) # Copy entire directory - else: - shutil.copy(source, destination) # Copy individual files + shutil.rmtree(submission_dir) + os.rename(submission_processed, submission_dir) return {'return':0} From c95819c7f807992990062cae70b56eb97ed74ef8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:16:46 +0530 Subject: [PATCH 587/658] consider base submission dir as output dir --- script/tar-my-folder/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/tar-my-folder/customize.py b/script/tar-my-folder/customize.py index d405e76fdb..3f8760d567 100644 --- a/script/tar-my-folder/customize.py +++ b/script/tar-my-folder/customize.py @@ -11,7 +11,7 @@ def preprocess(i): input_dir = env.get("CM_TAR_INPUT_DIR", "") if input_dir == "": return {'return': 1, 'error': 'Please set CM_TAR_INPUT_DIR'} - output_dir = env.get("CM_TAR_OUTPUT_DIR", "") + output_dir = env.get("CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", env.get("CM_TAR_OUTPUT_DIR", "")) if output_dir == "": output_dir = os.getcwd() output_file = env.get("CM_TAR_OUTFILE", "") From 60c02fa9a76c6bd23e8996e5924186bfed33f968 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 5 Nov 2024 19:16:56 +0530 Subject: [PATCH 588/658] Added google dns to mlperf-inference docker --- script/app-mlperf-inference/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 928709a8f4..58a675ba80 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -1675,7 +1675,7 @@ docker: skip_run_cmd: 'no' shm_size: '32gb' interactive: True - extra_run_args: ' --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined' + extra_run_args: ' --dns 8.8.8.8 --dns 8.8.4.4 --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined' os: ubuntu cm_repo: mlcommons@cm4mlops cm_repo_branch: mlperf-inference From 4ee37338c0a68a1b2533a862472e42f457308d33 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:21:23 +0530 Subject: [PATCH 589/658] fix typo --- script/generate-mlperf-inference-submission/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 6f7e484bd0..6cfb98ce41 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -159,7 +159,7 @@ "cm_repo_branch": "mlperf-inference", "os_version": "22.04", "docker_input_mapping": { - "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", + "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", "results_dir": "CM_MLPERF_INFERENCE_RESULTS_DIR_" } } From 1df3eb1ade2e5d5b7f173cb0e3a040f8ba2bf45a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:31:06 +0530 Subject: [PATCH 590/658] bug fix --- script/generate-mlperf-inference-submission/_cm.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 6cfb98ce41..d8f3293190 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -139,9 +139,12 @@ { "tags": "get,mlperf,inference,submission,dir,local", "names": "get-mlperf-inference-submission-dir", - "skip_if_env": { + "skip_if_any_env": { "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR": [ "on" + ], + "CM_MLPERF_INFERENCE_SUBMISSION_DIR": [ + "on" ] } } @@ -150,6 +153,7 @@ "cm pull repo" ], "mounts": [ + "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", "${{ CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR }}", "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" ], @@ -160,6 +164,7 @@ "os_version": "22.04", "docker_input_mapping": { "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", + "submission_base_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", "results_dir": "CM_MLPERF_INFERENCE_RESULTS_DIR_" } } From a96d211fd28cb1eae889461016d6ee8d0598050a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:33:25 +0530 Subject: [PATCH 591/658] fix typo --- script/generate-mlperf-inference-submission/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index d8f3293190..af5a61c399 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -163,7 +163,7 @@ "cm_repo_branch": "mlperf-inference", "os_version": "22.04", "docker_input_mapping": { - "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", + "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", "submission_base_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", "results_dir": "CM_MLPERF_INFERENCE_RESULTS_DIR_" } From 5a26cd58d0c7807a7e6fe9ebb55ac666f23bbf31 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:50:12 +0530 Subject: [PATCH 592/658] enable test globally --- .github/workflows/test-cm-based-submission-generation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 7eb6b3df24..cd5be0b4fc 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -7,8 +7,8 @@ on: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-cm-based-submission-generation.yml' - # - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - # - '!**.md' + - '**' + - '!**.md' jobs: submission_generation: runs-on: ${{ matrix.os }} From 87ccbe0383bf4e3ebfa6f947ace07af3fbd4e725 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:22:09 +0530 Subject: [PATCH 593/658] submission base dir -> tar output --- script/run-mlperf-inference-submission-checker/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/run-mlperf-inference-submission-checker/_cm.json b/script/run-mlperf-inference-submission-checker/_cm.json index ab57623799..53129a7eca 100644 --- a/script/run-mlperf-inference-submission-checker/_cm.json +++ b/script/run-mlperf-inference-submission-checker/_cm.json @@ -68,6 +68,9 @@ "yes" ] }, + "env": { + "CM_TAR_OUTPUT_DIR": "<<>>" + }, "tags": "run,tar" } ], From 152712fad8ab04a4793417c09f779de93c68f31c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:23:07 +0530 Subject: [PATCH 594/658] revert commit https://github.com/mlcommons/cm4mlops/pull/486/commits/c95819c7f807992990062cae70b56eb97ed74ef8 --- script/tar-my-folder/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/tar-my-folder/customize.py b/script/tar-my-folder/customize.py index 3f8760d567..d405e76fdb 100644 --- a/script/tar-my-folder/customize.py +++ b/script/tar-my-folder/customize.py @@ -11,7 +11,7 @@ def preprocess(i): input_dir = env.get("CM_TAR_INPUT_DIR", "") if input_dir == "": return {'return': 1, 'error': 'Please set CM_TAR_INPUT_DIR'} - output_dir = env.get("CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", env.get("CM_TAR_OUTPUT_DIR", "")) + output_dir = env.get("CM_TAR_OUTPUT_DIR", "") if output_dir == "": output_dir = os.getcwd() output_file = env.get("CM_TAR_OUTFILE", "") From a89fa43fb247621581af431845bb9f16a7867d20 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:27:14 +0530 Subject: [PATCH 595/658] get submission directory if base dir not specified - docker --- script/generate-mlperf-inference-submission/_cm.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index af5a61c399..96e9e6ca87 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -142,9 +142,6 @@ "skip_if_any_env": { "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR": [ "on" - ], - "CM_MLPERF_INFERENCE_SUBMISSION_DIR": [ - "on" ] } } From bfe6cb64a1dc75007e8f50798ba432a2dfae593d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:45:17 +0530 Subject: [PATCH 596/658] add tar output dir --- script/run-mlperf-inference-submission-checker/customize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 1781385c59..c0e943c75e 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -95,6 +95,9 @@ def postprocess(i): if x!='': env['CM_TAR_OUTFILE']=x + if env.get('CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR', '') != '': + env['CM_TAR_OUTPUT_DIR'] = env['CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR'] + x=env.get('MLPERF_INFERENCE_SUBMISSION_SUMMARY','') if x!='': for y in ['.csv', '.json', '.xlsx']: From 6e48bdd1d631b0258410c971393abe725e9c1f80 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:45:35 +0530 Subject: [PATCH 597/658] Update _cm.json --- script/run-mlperf-inference-submission-checker/_cm.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/run-mlperf-inference-submission-checker/_cm.json b/script/run-mlperf-inference-submission-checker/_cm.json index 53129a7eca..ab57623799 100644 --- a/script/run-mlperf-inference-submission-checker/_cm.json +++ b/script/run-mlperf-inference-submission-checker/_cm.json @@ -68,9 +68,6 @@ "yes" ] }, - "env": { - "CM_TAR_OUTPUT_DIR": "<<>>" - }, "tags": "run,tar" } ], From 94ef81901e042f197b395c13fc52a09b4aa71348 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 5 Nov 2024 15:35:16 +0000 Subject: [PATCH 598/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 698c0a2240..6516939f57 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -27,7 +27,7 @@ jobs: pip install --upgrade cm4mlops pip install tabulate cm pull repo - cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean + cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --pull_changes=yes --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions @@ -52,7 +52,7 @@ jobs: pip install --upgrade cm4mlops pip install tabulate cm pull repo - cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --pull_changes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes --clean + cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --pull_changes=yes --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --pull_changes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --hw_name=go-spr --custom_system_nvidia=yes --clean cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions From ab2f5283a08b01f1efc0cc45906c0d7b6376192e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 5 Nov 2024 22:21:34 +0530 Subject: [PATCH 599/658] code clean --- script/generate-mlperf-inference-submission/_cm.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/script/generate-mlperf-inference-submission/_cm.json b/script/generate-mlperf-inference-submission/_cm.json index 96e9e6ca87..2a51f764b3 100644 --- a/script/generate-mlperf-inference-submission/_cm.json +++ b/script/generate-mlperf-inference-submission/_cm.json @@ -150,17 +150,15 @@ "cm pull repo" ], "mounts": [ - "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}", "${{ CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR }}", "${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR_ }}" ], - "extra_run_args": " --ulimit memlock=-1 --cap-add SYS_ADMIN --cap-add SYS_TIME --security-opt apparmor=unconfined --security-opt seccomp=unconfined", + "extra_run_args": " --cap-add SYS_ADMIN", "os": "ubuntu", "cm_repo": "mlcommons@cm4mlops", "cm_repo_branch": "mlperf-inference", "os_version": "22.04", "docker_input_mapping": { - "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", "submission_base_dir": "CM_MLPERF_INFERENCE_SUBMISSION_BASE_DIR", "results_dir": "CM_MLPERF_INFERENCE_RESULTS_DIR_" } From e83e72cc951b67227d954d0c5af3bc95e3426528 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:55:40 +0530 Subject: [PATCH 600/658] test commit for macos --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index f6a13bd6d2..25275ac870 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -38,8 +38,8 @@ jobs: python-version: "3.8" - os: macos-latest docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" - - os: macos-13 - docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" + # - os: macos-13 + # docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" steps: - uses: actions/checkout@v3 @@ -85,6 +85,8 @@ jobs: echo "Docker is up and running" + cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v + - name: Install Docker Desktop on Windows if: runner.os == 'Windows' run: | @@ -108,4 +110,5 @@ jobs: - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v + echo "done" + # cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v From 903cd2a1fc8355b582479c42d1fcc35322ad44e6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:00:49 +0530 Subject: [PATCH 601/658] test commit - mac --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 25275ac870..fe364e55b1 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -71,6 +71,7 @@ jobs: # Loop until Docker daemon is up or max attempts reached attempt=1 + docker --version while ! docker --version > /dev/null 2>&1; do echo "Attempt $attempt: Waiting for Docker to start..." sleep $WAIT_TIME From 6861de2f665003aa84c5e9a2761924aa99569c3b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:12:59 +0530 Subject: [PATCH 602/658] check docker info --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index fe364e55b1..a47bed7506 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -72,6 +72,7 @@ jobs: # Loop until Docker daemon is up or max attempts reached attempt=1 docker --version + docker info while ! docker --version > /dev/null 2>&1; do echo "Attempt $attempt: Waiting for Docker to start..." sleep $WAIT_TIME From afc3a028ce3e2b7415208ba6604caa42e34bc379 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:18:22 +0530 Subject: [PATCH 603/658] test commit - mac --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index a47bed7506..1a3731835e 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -73,7 +73,7 @@ jobs: attempt=1 docker --version docker info - while ! docker --version > /dev/null 2>&1; do + while ! docker info > /dev/null 2>&1; do echo "Attempt $attempt: Waiting for Docker to start..." sleep $WAIT_TIME attempt=$((attempt + 1)) From a82bfd9db9649571aff8f91ac474723fa9d22b16 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:22:31 +0530 Subject: [PATCH 604/658] test commit --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 1a3731835e..2568b254c3 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -72,7 +72,6 @@ jobs: # Loop until Docker daemon is up or max attempts reached attempt=1 docker --version - docker info while ! docker info > /dev/null 2>&1; do echo "Attempt $attempt: Waiting for Docker to start..." sleep $WAIT_TIME From b9eefb15904be13b46516b1c4f314e7447667e3e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:51:15 +0530 Subject: [PATCH 605/658] deactivated runs for windows and mac os docker --- .../test-mlperf-inference-abtf-poc.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 2568b254c3..996e742840 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -38,8 +38,8 @@ jobs: python-version: "3.8" - os: macos-latest docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" - # - os: macos-13 - # docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" + - os: macos-13 + docker: " --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes" steps: - uses: actions/checkout@v3 @@ -54,13 +54,13 @@ jobs: cm pull repo mlcommons@cm4abtf --branch=poc - name: Install Docker on macos - if: runner.os == 'macOS' + if: runner.os == 'macOS-deactivated' run: | brew update brew install --cask docker - name: Start Docker Daemon on macos - if: runner.os == 'macOS' + if: runner.os == 'macOS-deactivated' run: | open /Applications/Docker.app echo "Starting Docker, this may take a while..." @@ -71,7 +71,6 @@ jobs: # Loop until Docker daemon is up or max attempts reached attempt=1 - docker --version while ! docker info > /dev/null 2>&1; do echo "Attempt $attempt: Waiting for Docker to start..." sleep $WAIT_TIME @@ -85,16 +84,14 @@ jobs: done echo "Docker is up and running" - - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v - + - name: Install Docker Desktop on Windows - if: runner.os == 'Windows' + if: runner.os == 'Windows-deactivated' run: | choco install docker-desktop --no-progress -y - name: Start Docker Desktop on Windows - if: runner.os == 'Windows' + if: runner.os == 'Windows-deactivated' run: | Start-Process 'C:\Program Files\Docker\Docker\Docker Desktop.exe' # Wait until Docker daemon is running @@ -112,4 +109,4 @@ jobs: - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | echo "done" - # cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v + cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v From 717281b801844df4a18b743d5ddfead7efe76135 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:07:18 +0530 Subject: [PATCH 606/658] code clean --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 996e742840..de62093015 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -108,5 +108,4 @@ jobs: - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | - echo "done" cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} ${{ matrix.docker }} -v From 09fc291d5c75895cdf7582ca3ea40481c572652e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 14:06:18 +0000 Subject: [PATCH 607/658] Added an option to pull inference src changes for mlperf-inference --- script/app-mlperf-inference/_cm.yaml | 7 ++++++- script/run-mlperf-inference-app/_cm.yaml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 58a675ba80..47b39c0fca 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -103,7 +103,12 @@ deps: - tags: get,mlcommons,inference,src names: - inference-src - + - tags: pull,git,repo + env: + CM_GIT_CHECKOUT_PATH: '<<>>' + enable_if_env: + CM_MLPERF_INFERENCE_PULL_SRC_CHANGES: + - 'yes' - tags: get,mlperf,inference,utils - tags: install,pip-package,for-cmind-python,_package.pandas diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index e92ab4253b..68518baf81 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -81,6 +81,7 @@ input_mapping: preprocess_submission: CM_RUN_MLPERF_SUBMISSION_PREPROCESSOR push_to_github: CM_MLPERF_RESULT_PUSH_TO_GITHUB pull_changes: CM_MLPERF_INFERENCE_PULL_CODE_CHANGES + pull_inference_changes: CM_MLPERF_INFERENCE_PULL_SRC_CHANGES readme: CM_MLPERF_README regenerate_accuracy_file: CM_MLPERF_REGENERATE_ACCURACY_FILE regenerate_files: CM_REGENERATE_MEASURE_FILES From 9c6f5de00b96580b75a04304d5f81815590b07d1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 14:07:04 +0000 Subject: [PATCH 608/658] Added an option to pull inference src changes for mlperf-inference --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 3229f21511..f534aa9e36 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 From 76b97884091d67302d7a08c58725c0ef855b8b1f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:37:42 +0530 Subject: [PATCH 609/658] added docker run --- .github/workflows/test-cm-based-submission-generation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index cd5be0b4fc..2a078e9249 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -20,6 +20,7 @@ jobs: division: ["closed", "open"] category: ["datacenter", "edge"] case: ["case-3", "case-7"] + action: ["script", "docker"] exclude: - os: macos-latest - os: windows-latest @@ -49,6 +50,6 @@ jobs: fi # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm ${{ matrix.action }} script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet echo "::endgroup::" From 9f3b7d2f66dd1e794b2cf569be99d7196b6d5941 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:39:43 +0530 Subject: [PATCH 610/658] added action in the description-steps --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 2a078e9249..19b540636c 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -39,7 +39,7 @@ jobs: - name: Pull repo where test cases are uploaded run: | git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests - - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.category }} ${{ matrix.division }} + - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} run: | if [ "${{ matrix.case }}" == "case-3" ]; then #results_dir="submission_generation_tests/case-3/" From 79d445e12bf4fa9a77aa4e9c7d1c77700a42f6e8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:48:04 +0530 Subject: [PATCH 611/658] fix typo --- .github/workflows/test-cm-based-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml index 19b540636c..91420ecc03 100644 --- a/.github/workflows/test-cm-based-submission-generation.yml +++ b/.github/workflows/test-cm-based-submission-generation.yml @@ -20,7 +20,7 @@ jobs: division: ["closed", "open"] category: ["datacenter", "edge"] case: ["case-3", "case-7"] - action: ["script", "docker"] + action: ["run", "docker"] exclude: - os: macos-latest - os: windows-latest From 8470530393f1ec0ad262f5532d8ad17e03fa8507 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 14:26:53 +0000 Subject: [PATCH 612/658] Update module.py --- automation/script/module.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 8b11ff45e1..9fd16dfd2d 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -8,7 +8,6 @@ # TBD: when we have bandwidth and resources, we should refactor it # and make it cleaner and simpler while keeping full backwards compatibility. # -import re import os import logging @@ -3979,11 +3978,6 @@ def parse_version(self, i): return r string = r['string'] - match = re.search(match_text, string) - if debug: - print(f"Regex Pattern: {match_text}") - print(f"Matched String: {string}") - print(f"Match Groups: {match.groups()}") if r['match'].lastindex and r['match'].lastindex >= group_number: version = r['match'].group(group_number) From 423e692e4808697e22ffe8464fc7bce46adfb107 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 21:00:32 +0530 Subject: [PATCH 613/658] Fix diffuser version for SDXL reference implementation --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 8a06f987cd..caf56a71f0 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -929,6 +929,8 @@ variations: - tags: get,generic-python-lib,_package.diffusers names: - diffusers + version_max: "0.30.3" + version_max_usable: "0.30.3" - tags: get,generic-python-lib,_package.transformers names: - transformers From 244952d648e67f0aaa9483d08d78267f9944f463 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 21:01:53 +0530 Subject: [PATCH 614/658] Update float16 name for SDXL model --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index caf56a71f0..470cd72bcb 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -394,6 +394,7 @@ deps: names: - ml-model - sdxl-model + - ml-model-float16 enable_if_env: CM_MODEL: - stable-diffusion-xl From 79e7cb50d310786e758d819aad177a54de76a5df Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 21:35:18 +0530 Subject: [PATCH 615/658] Use http link for intel conda packages --- script/install-generic-conda-package/_cm.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/install-generic-conda-package/_cm.json b/script/install-generic-conda-package/_cm.json index 4cefd679d4..120bf2d86e 100644 --- a/script/install-generic-conda-package/_cm.json +++ b/script/install-generic-conda-package/_cm.json @@ -61,6 +61,12 @@ "CM_CONDA_PKG_NAME": "#" } }, + "source.intel": { + "group": "package-source", + "env": { + "CM_CONDA_PKG_SRC": "https://software.repos.intel.com/python/conda/" + } + }, "source.#": { "group": "package-source", "env": { From 81ff62222fa724b70508955457be43ec5f898348 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 21:47:24 +0530 Subject: [PATCH 616/658] Dont use '-dt' for Nvidia ml-model-gptj --- script/get-ml-model-gptj/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index b3c61086d1..639efcde87 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -25,7 +25,7 @@ def preprocess(i): elif env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': i['run_script_input']['script_name'] = 'run-nvidia' if str(env.get('CM_DOCKER_DETACHED_MODE','')).lower() in ['yes', 'true', "1"]: - env['DOCKER_RUN_OPTS'] = "--rm -dt --ipc=host --ulimit memlock=-1 --ulimit stack=67108864" + env['DOCKER_RUN_OPTS'] = "--rm --ipc=host --ulimit memlock=-1 --ulimit stack=67108864" gpu_arch = int(float(env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * 10) env['CM_GPU_ARCH'] = gpu_arch env['CM_TMP_REQUIRE_DOWNLOAD'] = 'no' From cd52f7aa1b80440f2dd285628e5f05eb212d9e08 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 23:22:21 +0530 Subject: [PATCH 617/658] Update starting weights filename for SDXL MLPerf inference --- script/get-ml-model-stable-diffusion/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-stable-diffusion/_cm.json b/script/get-ml-model-stable-diffusion/_cm.json index 2e062a080a..5934e5c3e8 100644 --- a/script/get-ml-model-stable-diffusion/_cm.json +++ b/script/get-ml-model-stable-diffusion/_cm.json @@ -106,6 +106,7 @@ "pytorch": { "default": true, "env": { + "CM_ML_MODEL_STARTING_WEIGHTS_FILENAME": "https://github.com/mlcommons/inference/tree/master/text_to_image#download-model", "CM_ML_MODEL_FRAMEWORK": "pytorch" }, "group": "framework" @@ -115,7 +116,6 @@ }, "pytorch,fp32": { "env": { - "CM_ML_MODEL_STARTING_WEIGHTS_FILENAME": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0" }, "required_disk_space": 13000 }, From 0c2fb2950bee2ad83d4b8d8d3c58a8d515c3d5e9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 19:38:25 +0000 Subject: [PATCH 618/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index f534aa9e36..596e6ddf81 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "29 20 * * *" #to be adjusted + - cron: "49 19 * * *" #to be adjusted jobs: build_nvidia: From aa5bfac317f98ff152c24b24a8d144f9681abf30 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 23:22:08 +0000 Subject: [PATCH 619/658] Create test-mlperf-inference-intel --- .github/workflows/test-mlperf-inference-intel | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-intel diff --git a/.github/workflows/test-mlperf-inference-intel b/.github/workflows/test-mlperf-inference-intel new file mode 100644 index 0000000000..42201bd385 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-intel @@ -0,0 +1,25 @@ +name: MLPerf Inference Intel implementations + +on: + schedule: + - cron: "49 2 * * *" #to be adjusted + +jobs: + build_nvidia: + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64, GO-spr ] + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + model: [ "resnet50", "bert-99" ] + steps: + - name: Test MLPerf Inference Intel ${{ matrix.model }} + run: | + if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi + python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM + pip install --upgrade cm4mlops + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c From c4e45d45a3bda44e57d86a3aa5cfbdca4ff34541 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 15:22:13 +0530 Subject: [PATCH 620/658] Stash changes in git pull repo script --- script/pull-git-repo/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/pull-git-repo/customize.py b/script/pull-git-repo/customize.py index 021d42465e..39de6a15b6 100644 --- a/script/pull-git-repo/customize.py +++ b/script/pull-git-repo/customize.py @@ -15,7 +15,7 @@ def preprocess(i): if 'CM_GIT_CHECKOUT_PATH' not in env: return {'return':1, 'error': 'CM_GIT_CHECKOUT_PATH is not set'} - env['CM_GIT_PULL_CMD'] = "git pull --rebase" + env['CM_GIT_PULL_CMD'] = "git stash && git pull --rebase && git stash apply" return {'return':0} From c80071f120026c9e61d3238b794f6106deb2f4b4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 11:14:11 +0000 Subject: [PATCH 621/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 6516939f57..4c3430741f 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -29,7 +29,7 @@ jobs: cm pull repo cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --pull_changes=yes --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean cm run script --tags=run-mlperf,inference,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/scc_gh_action_results --submission_dir=$HOME/scc_gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results + cm run script --tags=generate,inference,submission --clean --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/scc_gh_action_submissions --results_dir=$HOME/scc_gh_action_results/test_results cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/scc_gh_action_submissions build_nvidia: From 45207c288ee604f8a01089e8401b08f3846e8520 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 17:31:57 +0530 Subject: [PATCH 622/658] Temp fix: use inference dev branch for SDXL --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 470cd72bcb..106d9f9bef 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -926,12 +926,13 @@ variations: env: CM_MODEL: stable-diffusion-xl CM_NUM_THREADS: "1" + adr: + mlperf-implementation: + tags: _branch.dev deps: - tags: get,generic-python-lib,_package.diffusers names: - diffusers - version_max: "0.30.3" - version_max_usable: "0.30.3" - tags: get,generic-python-lib,_package.transformers names: - transformers From 31534a98e14718f74d0e3841a1afcf48eb338c17 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 18:46:15 +0530 Subject: [PATCH 623/658] Dont fail in pull-git-repo if there are local changes --- script/pull-git-repo/customize.py | 2 +- script/pull-git-repo/run.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/pull-git-repo/customize.py b/script/pull-git-repo/customize.py index 39de6a15b6..021d42465e 100644 --- a/script/pull-git-repo/customize.py +++ b/script/pull-git-repo/customize.py @@ -15,7 +15,7 @@ def preprocess(i): if 'CM_GIT_CHECKOUT_PATH' not in env: return {'return':1, 'error': 'CM_GIT_CHECKOUT_PATH is not set'} - env['CM_GIT_PULL_CMD'] = "git stash && git pull --rebase && git stash apply" + env['CM_GIT_PULL_CMD'] = "git pull --rebase" return {'return':0} diff --git a/script/pull-git-repo/run.sh b/script/pull-git-repo/run.sh index 34b83cc63e..db8612d56d 100644 --- a/script/pull-git-repo/run.sh +++ b/script/pull-git-repo/run.sh @@ -11,6 +11,7 @@ test $? -eq 0 || exit $? echo ${CM_GIT_PULL_CMD} eval ${CM_GIT_PULL_CMD} -test $? -eq 0 || exit $? +#don't fail if there are local changes +#test $? -eq 0 || exit $? cd $CUR_DIR From fb357c01662eed87fa45e146a22af0c34670aaf4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 18:49:43 +0530 Subject: [PATCH 624/658] fix dependency for nvidia-mlperf-inference-gptj --- script/get-ml-model-gptj/_cm.json | 3 +++ script/get-ml-model-gptj/run-nvidia.sh | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/script/get-ml-model-gptj/_cm.json b/script/get-ml-model-gptj/_cm.json index 396e0d595e..39cb88e488 100644 --- a/script/get-ml-model-gptj/_cm.json +++ b/script/get-ml-model-gptj/_cm.json @@ -253,6 +253,9 @@ "python", "python3" ] + }, + { + "tags": "get,generic-python-lib,_package.safetensors" } ] }, diff --git a/script/get-ml-model-gptj/run-nvidia.sh b/script/get-ml-model-gptj/run-nvidia.sh index deba000ef9..27e5a675ce 100644 --- a/script/get-ml-model-gptj/run-nvidia.sh +++ b/script/get-ml-model-gptj/run-nvidia.sh @@ -12,10 +12,8 @@ cd ${CM_TENSORRT_LLM_CHECKOUT_PATH} make -C docker build test $? -eq 0 || exit $? -RUN_CMD="bash -c 'python3 scripts/build_wheel.py -a=${CM_GPU_ARCH} --clean --install --trt_root /usr/local/tensorrt/ && python examples/quantization/quantize.py --dtype=float16 --output_dir=/mnt/models/GPTJ-6B/fp8-quantized-ammo/GPTJ-FP8-quantized --model_dir=/mnt/models/GPTJ-6B/checkpoint-final --qformat=fp8 --kv_cache_dtype=fp8 '" -DOCKER_RUN_ARGS=" -v ${CM_NVIDIA_MLPERF_SCRATCH_PATH}:/mnt" -export DOCKER_RUN_ARGS="$DOCKER_RUN_ARGS" -export RUN_CMD="$RUN_CMD" +export RUN_CMD="bash -c 'python3 scripts/build_wheel.py -a=${CM_GPU_ARCH} --clean --install --trt_root /usr/local/tensorrt/ && python examples/quantization/quantize.py --dtype=float16 --output_dir=/mnt/models/GPTJ-6B/fp8-quantized-ammo/GPTJ-FP8-quantized --model_dir=/mnt/models/GPTJ-6B/checkpoint-final --qformat=fp8 --kv_cache_dtype=fp8 '" +export DOCKER_RUN_ARGS=" -v ${CM_NVIDIA_MLPERF_SCRATCH_PATH}:/mnt" make -C docker run LOCAL_USER=1 test $? -eq 0 || exit $? From 21425ef62cd7ffae4f0e69c36b113281a67b607e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 19:00:28 +0530 Subject: [PATCH 625/658] Preprocess mlperf inference submission, code cleanup --- script/preprocess-mlperf-inference-submission/_cm.json | 1 + script/preprocess-mlperf-inference-submission/customize.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/script/preprocess-mlperf-inference-submission/_cm.json b/script/preprocess-mlperf-inference-submission/_cm.json index 312ab7c2f1..6e1fade381 100644 --- a/script/preprocess-mlperf-inference-submission/_cm.json +++ b/script/preprocess-mlperf-inference-submission/_cm.json @@ -31,6 +31,7 @@ } ], "input_mapping": { + "input": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", "submission_dir": "CM_MLPERF_INFERENCE_SUBMISSION_DIR", "submitter": "CM_MLPERF_SUBMITTER" }, diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index c5669338be..f0ae63a222 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -14,7 +14,7 @@ def preprocess(i): print("Please set CM_MLPERF_INFERENCE_SUBMISSION_DIR") return {'return': 1, 'error':'CM_MLPERF_INFERENCE_SUBMISSION_DIR is not specified'} - submitter = env.get("CM_MLPERF_SUBMITTER", "cTuning") + submitter = env.get("CM_MLPERF_SUBMITTER", "MLCommons") submission_processed = submission_dir + "_processed" if os.path.exists(submission_processed): From 0c68370bc2eabd88241b561bf663d9c166eccd20 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 14:45:52 +0000 Subject: [PATCH 626/658] Update test-mlperf-inference-intel --- .github/workflows/test-mlperf-inference-intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-intel b/.github/workflows/test-mlperf-inference-intel index 42201bd385..f1d31fc76b 100644 --- a/.github/workflows/test-mlperf-inference-intel +++ b/.github/workflows/test-mlperf-inference-intel @@ -2,7 +2,7 @@ name: MLPerf Inference Intel implementations on: schedule: - - cron: "49 2 * * *" #to be adjusted + - cron: "54 14 * * *" #to be adjusted jobs: build_nvidia: From ea39555bb4231038de51e1e33812dee952b946aa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 15:05:15 +0000 Subject: [PATCH 627/658] Update test-mlperf-inference-intel --- .github/workflows/test-mlperf-inference-intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-intel b/.github/workflows/test-mlperf-inference-intel index f1d31fc76b..b52a7ff708 100644 --- a/.github/workflows/test-mlperf-inference-intel +++ b/.github/workflows/test-mlperf-inference-intel @@ -21,5 +21,5 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops - cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c From b8bdc0ee3004cebdc08c1f54154a5c5f1c12104d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 7 Nov 2024 20:58:43 +0530 Subject: [PATCH 628/658] handle systems where sudo is absent --- setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 47afb3fb19..bab35447dd 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ import importlib.util import platform import os +import shutil # Try to use importlib.metadata for Python 3.8+ try: @@ -76,8 +77,15 @@ def install_system_packages(self): manager, details = self.get_package_manager_details() if manager: if manager == "apt-get": - subprocess.check_call(['sudo', 'apt-get', 'update']) - subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + # Check if 'sudo' is available + if shutil.which('sudo'): + try: + subprocess.check_call(['sudo', 'apt-get', 'update']) + subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + except subprocess.CalledProcessError: + print("Sudo command failed, trying without sudo.") + subprocess.check_call(['apt-get', 'update']) + subprocess.check_call(['apt-get', 'install', '-y'] + packages) elif self.system == 'Windows': print(f"Please install the following packages manually: {packages}") From 7a8a15296fa820d1d915ef472ffdadc285c84a5d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 21:27:06 +0530 Subject: [PATCH 629/658] Cleanups for MLPerf inference preprocess script, use inference dev branch for scc --- .../customize.py | 12 ++++++++---- script/run-mlperf-inference-app/_cm.yaml | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/script/preprocess-mlperf-inference-submission/customize.py b/script/preprocess-mlperf-inference-submission/customize.py index f0ae63a222..474e4a42ed 100644 --- a/script/preprocess-mlperf-inference-submission/customize.py +++ b/script/preprocess-mlperf-inference-submission/customize.py @@ -11,17 +11,21 @@ def preprocess(i): submission_dir = env.get("CM_MLPERF_INFERENCE_SUBMISSION_DIR", "") if submission_dir == "": - print("Please set CM_MLPERF_INFERENCE_SUBMISSION_DIR") + print("Please set --env.CM_MLPERF_INFERENCE_SUBMISSION_DIR") return {'return': 1, 'error':'CM_MLPERF_INFERENCE_SUBMISSION_DIR is not specified'} + if not os.path.exists(submission_dir): + print("Please set --env.CM_MLPERF_INFERENCE_SUBMISSION_DIR to a valid submission directory") + return {'return': 1, 'error':'CM_MLPERF_INFERENCE_SUBMISSION_DIR is not existing'} + + submission_dir = submission_dir.rstrip(os.path.sep) submitter = env.get("CM_MLPERF_SUBMITTER", "MLCommons") - submission_processed = submission_dir + "_processed" + submission_processed = f"{submission_dir}_processed" if os.path.exists(submission_processed): + print(f"Cleaning {submission_processed}") shutil.rmtree(submission_processed) - os.system("rm -rf " + submission_dir + "_processed") - CMD = env['CM_PYTHON_BIN'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "tools", "submission", "preprocess_submission.py") + "' --input '" + submission_dir + "' --submitter '" + submitter + "' --output '" + submission_processed + "'" env['CM_RUN_CMD'] = CMD diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 68518baf81..0105c1c70c 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -264,6 +264,8 @@ variations: tags: _size.50,_with-sample-ids nvidia-preprocess-data: extra_cache_tags: "scc24-base" + inference-src: + tags: _branch.dev deps: - tags: clean,nvidia,scratch,_sdxl,_downloaded-data extra_cache_rm_tags: scc24-main @@ -278,6 +280,8 @@ variations: tags: _size.500,_with-sample-ids nvidia-preprocess-data: extra_cache_tags: "scc24-main" + inference-src: + tags: _branch.dev env: CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX4: scc24-main CM_DOCKER_IMAGE_NAME: scc24 From f80fe87b3f12f29ae8a196e0ba3fa0f5a4d881fc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:23:19 +0000 Subject: [PATCH 630/658] Update and rename test-mlperf-inference-intel to test-mlperf-inference-intel.yml --- ...rf-inference-intel => test-mlperf-inference-intel.yml} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{test-mlperf-inference-intel => test-mlperf-inference-intel.yml} (85%) diff --git a/.github/workflows/test-mlperf-inference-intel b/.github/workflows/test-mlperf-inference-intel.yml similarity index 85% rename from .github/workflows/test-mlperf-inference-intel rename to .github/workflows/test-mlperf-inference-intel.yml index b52a7ff708..2a8e452f33 100644 --- a/.github/workflows/test-mlperf-inference-intel +++ b/.github/workflows/test-mlperf-inference-intel.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Intel implementations on: schedule: - - cron: "54 14 * * *" #to be adjusted + - cron: "29 16 * * *" #to be adjusted jobs: build_nvidia: @@ -16,9 +16,9 @@ jobs: steps: - name: Test MLPerf Inference Intel ${{ matrix.model }} run: | - if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi - python3 -m venv gh_action - source gh_action/bin/activate + if [ -f "gh_action_conda/bin/deactivate" ]; then source gh_action_conda/bin/deactivate; fi + python3 -m venv gh_action_conda + source gh_action_conda/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet From 5200fcb4857678e03a510735fef753bbdb7b7e2b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 21:56:22 +0530 Subject: [PATCH 631/658] Support sample_ids_path in coco2014 accuracy script --- script/process-mlperf-accuracy/customize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 4a6ba1634c..bb5d7a2865 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -103,6 +103,9 @@ def preprocess(i): else: extra_options += f""" --compliance-images-path '{os.path.join(result_dir, "images")}' """ + if env.get('CM_COCO2014_SAMPLE_ID_PATH','') != '': + extra_options += f" --ids-path '{env['CM_COCO2014_SAMPLE_ID_PATH']}' " + if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': extra_options += f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " From 0600c69e759287fc6c6fe9e0a4ff2f1111538ad5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 22:02:03 +0530 Subject: [PATCH 632/658] Added rocm device for AMD mlperf inference --- script/app-mlperf-inference-amd/_cm.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/script/app-mlperf-inference-amd/_cm.yaml b/script/app-mlperf-inference-amd/_cm.yaml index 089a1be1eb..3882380372 100644 --- a/script/app-mlperf-inference-amd/_cm.yaml +++ b/script/app-mlperf-inference-amd/_cm.yaml @@ -146,6 +146,12 @@ variations: CM_MLPERF_DEVICE: gpu CM_MLPERF_DEVICE_LIB_NAMESPEC: cudart + rocm: + group: device + env: + CM_MLPERF_DEVICE: rocm + CM_MLPERF_DEVICE_LIB_NAMESPEC: rocm + openshift: group: backend default: true @@ -161,6 +167,10 @@ variations: deps: - tags: get,generic-python-lib,_torch_cuda + pytorch,rocm: + deps: + - tags: get,generic-python-lib,_torch,_rocm + pytorch,cpu: deps: - tags: get,generic-python-lib,_torch From 2c960c40c3d66e63d553624c4fbb2fc57641b6c3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:36:18 +0000 Subject: [PATCH 633/658] Create test-mlperf-inference-amd.yml --- .../workflows/test-mlperf-inference-amd.yml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-amd.yml diff --git a/.github/workflows/test-mlperf-inference-amd.yml b/.github/workflows/test-mlperf-inference-amd.yml new file mode 100644 index 0000000000..a6c248c579 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-amd.yml @@ -0,0 +1,25 @@ +name: MLPerf Inference AMD implementations + +on: + schedule: + - cron: "29 4 * * *" #to be adjusted + +jobs: + build_nvidia: + if: github.repository_owner == 'gateoverflow' + runs-on: [ self-hosted, linux, x64, GO-spr ] + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + model: [ "llama2-70b" ] + steps: + - name: Test MLPerf Inference AMD (build only) ${{ matrix.model }} + run: | + if [ -f "gh_action_conda/bin/deactivate" ]; then source gh_action_conda/bin/deactivate; fi + python3 -m venv gh_action_conda + source gh_action_conda/bin/activate + export CM_REPOS=$HOME/GH_CM + pip install --upgrade cm4mlops + cm run script --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=amd --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=rocm --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet + # cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c From 50ad695987c798dfa862231c55d2b3ab060b56e0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:49:14 +0000 Subject: [PATCH 634/658] Update test-nvidia-mlperf-implementation.yml --- .github/workflows/test-nvidia-mlperf-implementation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index 596e6ddf81..5373a825b4 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -21,5 +21,6 @@ jobs: source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops + pip install tabulate cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --gpu_name=rtx_4090 --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=RTX4090x2 --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cuda --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on NVIDIA_RTX4090x2" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=RTX4090x2 From 37c4ac2051a6638764fb70eafe7de9b899439d99 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:50:06 +0000 Subject: [PATCH 635/658] Update and rename test-mlperf-inference-amd.yml to test-amd-mlperf-inference-implementationsyml --- ...ence-amd.yml => test-amd-mlperf-inference-implementationsyml} | 1 + 1 file changed, 1 insertion(+) rename .github/workflows/{test-mlperf-inference-amd.yml => test-amd-mlperf-inference-implementationsyml} (98%) diff --git a/.github/workflows/test-mlperf-inference-amd.yml b/.github/workflows/test-amd-mlperf-inference-implementationsyml similarity index 98% rename from .github/workflows/test-mlperf-inference-amd.yml rename to .github/workflows/test-amd-mlperf-inference-implementationsyml index a6c248c579..deff519672 100644 --- a/.github/workflows/test-mlperf-inference-amd.yml +++ b/.github/workflows/test-amd-mlperf-inference-implementationsyml @@ -21,5 +21,6 @@ jobs: source gh_action_conda/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops + pip install tabulate cm run script --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=amd --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=rocm --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet # cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c From 5c0b178637c7e8a9d145b66d7673088e6f6d4a82 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:50:24 +0000 Subject: [PATCH 636/658] Rename test-amd-mlperf-inference-implementationsyml to test-amd-mlperf-inference-implementations.yml --- ...entationsyml => test-amd-mlperf-inference-implementations.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test-amd-mlperf-inference-implementationsyml => test-amd-mlperf-inference-implementations.yml} (100%) diff --git a/.github/workflows/test-amd-mlperf-inference-implementationsyml b/.github/workflows/test-amd-mlperf-inference-implementations.yml similarity index 100% rename from .github/workflows/test-amd-mlperf-inference-implementationsyml rename to .github/workflows/test-amd-mlperf-inference-implementations.yml From 85785652c9fb3797b473444a2c4b7b1c8b22c4c4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:50:53 +0000 Subject: [PATCH 637/658] Rename test-mlperf-inference-intel.yml to test-intel-mlperf-inference-implementations.yml --- ...-intel.yml => test-intel-mlperf-inference-implementations.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test-mlperf-inference-intel.yml => test-intel-mlperf-inference-implementations.yml} (100%) diff --git a/.github/workflows/test-mlperf-inference-intel.yml b/.github/workflows/test-intel-mlperf-inference-implementations.yml similarity index 100% rename from .github/workflows/test-mlperf-inference-intel.yml rename to .github/workflows/test-intel-mlperf-inference-implementations.yml From c5405200345686e349ca810f6f1c5feeb78d4a28 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:51:07 +0000 Subject: [PATCH 638/658] Update test-intel-mlperf-inference-implementations.yml --- .../workflows/test-intel-mlperf-inference-implementations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-intel-mlperf-inference-implementations.yml b/.github/workflows/test-intel-mlperf-inference-implementations.yml index 2a8e452f33..32be0d9e15 100644 --- a/.github/workflows/test-intel-mlperf-inference-implementations.yml +++ b/.github/workflows/test-intel-mlperf-inference-implementations.yml @@ -21,5 +21,6 @@ jobs: source gh_action_conda/bin/activate export CM_REPOS=$HOME/GH_CM pip install --upgrade cm4mlops + pip install tabulate cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c From dd5048ff82e35ee69bfd0b820327166f3b95ee8d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 16:51:34 +0000 Subject: [PATCH 639/658] Rename test-nvidia-mlperf-implementation.yml to test-nvidia-mlperf-inference-implementations.yml --- ...ation.yml => test-nvidia-mlperf-inference-implementations.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test-nvidia-mlperf-implementation.yml => test-nvidia-mlperf-inference-implementations.yml} (100%) diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-inference-implementations.yml similarity index 100% rename from .github/workflows/test-nvidia-mlperf-implementation.yml rename to .github/workflows/test-nvidia-mlperf-inference-implementations.yml From 940b1c35598236849ec17550f5e6ef818287bde4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 17:15:20 +0000 Subject: [PATCH 640/658] Update test-mlperf-inference-mixtral.yml --- .github/workflows/test-mlperf-inference-mixtral.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index b5716e78f8..5b4a5ea959 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -5,12 +5,12 @@ name: MLPerf inference MIXTRAL-8x7B on: schedule: - - cron: "30 20 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST + - cron: "30 03 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST jobs: build_reference: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, GO-i9, linux, x64 ] + runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false matrix: From b7a6e164719a46ac43c059e580bca8166773211b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 17:46:56 +0000 Subject: [PATCH 641/658] Update _cm.yaml --- script/app-mlperf-inference-amd/_cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/app-mlperf-inference-amd/_cm.yaml b/script/app-mlperf-inference-amd/_cm.yaml index e4982b74ff..35a1e913c8 100644 --- a/script/app-mlperf-inference-amd/_cm.yaml +++ b/script/app-mlperf-inference-amd/_cm.yaml @@ -251,6 +251,8 @@ variations: skip_if_env: CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST: - 'yes' + CM_RUN_STATE_DOCKER: + - 'yes' - tags: get,preprocessed,dataset,openorca,_mlc,_validation - tags: download,file,_url.https://github.com/vllm-project/vllm/blob/38c4b7e863570a045308af814c72f4504297222e/tests/fp8_kv/llama2-70b-fp8-kv/kv_cache_scales.json extra_cache_tags: llama2-scales,kv-cache From 42a360df6b21916150095d299d3ec68e0b954982 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 18:52:03 +0000 Subject: [PATCH 642/658] Update test-intel-mlperf-inference-implementations.yml --- .../workflows/test-intel-mlperf-inference-implementations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-intel-mlperf-inference-implementations.yml b/.github/workflows/test-intel-mlperf-inference-implementations.yml index 32be0d9e15..bbb17c166b 100644 --- a/.github/workflows/test-intel-mlperf-inference-implementations.yml +++ b/.github/workflows/test-intel-mlperf-inference-implementations.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Intel implementations on: schedule: - - cron: "29 16 * * *" #to be adjusted + - cron: "29 1 * * *" #to be adjusted jobs: build_nvidia: From 4d6f7a01c70fdabcd13ca9992fa32d4c0f663447 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 19:27:42 +0000 Subject: [PATCH 643/658] Update test-scc24-sdxl.yaml --- .github/workflows/test-scc24-sdxl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-scc24-sdxl.yaml b/.github/workflows/test-scc24-sdxl.yaml index 4c3430741f..802593fe25 100644 --- a/.github/workflows/test-scc24-sdxl.yaml +++ b/.github/workflows/test-scc24-sdxl.yaml @@ -2,7 +2,7 @@ name: MLPerf inference SDXL (SCC) on: schedule: - - cron: "20 14 * * *" + - cron: "35 19 * * *" jobs: build_reference: From a0cc32d72bba6c5d23f607b334f3b9333c5447c2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 21:04:19 +0000 Subject: [PATCH 644/658] Update test-mlperf-inference-llama2.yml --- .github/workflows/test-mlperf-inference-llama2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-llama2.yml b/.github/workflows/test-mlperf-inference-llama2.yml index 97bd1bc6fc..c87a7ea8b1 100644 --- a/.github/workflows/test-mlperf-inference-llama2.yml +++ b/.github/workflows/test-mlperf-inference-llama2.yml @@ -5,12 +5,12 @@ name: MLPerf inference LLAMA 2 70B on: schedule: - - cron: "30 19 * * 4" + - cron: "30 2 * * 4" jobs: build_reference: if: github.repository_owner == 'gateoverflow' - runs-on: [ self-hosted, GO-i9, linux, x64 ] + runs-on: [ self-hosted, GO-spr, linux, x64 ] strategy: fail-fast: false matrix: From 8f076898d2788a2f4d443288cde95faacc2c04af Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 21:04:52 +0000 Subject: [PATCH 645/658] Update test-mlperf-inference-llama2.yml --- .github/workflows/test-mlperf-inference-llama2.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-llama2.yml b/.github/workflows/test-mlperf-inference-llama2.yml index c87a7ea8b1..b6fb155c48 100644 --- a/.github/workflows/test-mlperf-inference-llama2.yml +++ b/.github/workflows/test-mlperf-inference-llama2.yml @@ -24,9 +24,10 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - python3 -m pip install cm4mlops + pip install cm4mlops + pip install tabulate cm pull repo - python3 -m pip install "huggingface_hub[cli]" + pip install "huggingface_hub[cli]" huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential - name: Test MLPerf Inference LLAMA 2 70B reference implementation run: | From 2164e661bee62709ed1f68b0107bba44981c07f2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 02:47:58 +0530 Subject: [PATCH 646/658] Added a retry for git clone failure --- script/get-git-repo/run.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/script/get-git-repo/run.sh b/script/get-git-repo/run.sh index 6cd8ef8fbd..19867a8042 100644 --- a/script/get-git-repo/run.sh +++ b/script/get-git-repo/run.sh @@ -6,7 +6,9 @@ SCRIPT_DIR=${CM_TMP_CURRENT_SCRIPT_PATH} folder=${CM_GIT_CHECKOUT_FOLDER} if [ ! -e "${CM_TMP_GIT_PATH}" ]; then - rm -rf ${folder} + cmd="rm -rf ${folder}" + echo $cmd + eval $cmd echo "******************************************************" echo "Current directory: ${CUR_DIR}" echo "" @@ -16,7 +18,11 @@ if [ ! -e "${CM_TMP_GIT_PATH}" ]; then echo "" ${CM_GIT_CLONE_CMD} - test $? -eq 0 || exit $? + if [ ! $rcode -eq 0 ]; then #try once more + rm -rf $folder + ${CM_GIT_CLONE_CMD} + test $? -eq 0 || exit $? + fi cd ${folder} From e2bb8678fa4946eb5910530d88068428bcd0adff Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 02:59:33 +0530 Subject: [PATCH 647/658] Added a retry for git clone failure --- script/get-git-repo/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-git-repo/run.sh b/script/get-git-repo/run.sh index 19867a8042..2a7b0b51c9 100644 --- a/script/get-git-repo/run.sh +++ b/script/get-git-repo/run.sh @@ -18,6 +18,8 @@ if [ ! -e "${CM_TMP_GIT_PATH}" ]; then echo "" ${CM_GIT_CLONE_CMD} + rcode=$? + if [ ! $rcode -eq 0 ]; then #try once more rm -rf $folder ${CM_GIT_CLONE_CMD} From ba46c63abc0d41c526c23ff7753374040f43a850 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 07:02:50 +0530 Subject: [PATCH 648/658] Use custom version for dev branch of inference-src --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 1 + script/run-mlperf-inference-app/_cm.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 106d9f9bef..0b9e2fd95d 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -929,6 +929,7 @@ variations: adr: mlperf-implementation: tags: _branch.dev + version: custom deps: - tags: get,generic-python-lib,_package.diffusers names: diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 0105c1c70c..6afcc35270 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -266,6 +266,7 @@ variations: extra_cache_tags: "scc24-base" inference-src: tags: _branch.dev + version: custom deps: - tags: clean,nvidia,scratch,_sdxl,_downloaded-data extra_cache_rm_tags: scc24-main @@ -282,6 +283,7 @@ variations: extra_cache_tags: "scc24-main" inference-src: tags: _branch.dev + version: custom env: CM_MLPERF_SUT_NAME_RUN_CONFIG_SUFFIX4: scc24-main CM_DOCKER_IMAGE_NAME: scc24 From 137cd39ac350b3abe33be22a255809373b590933 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:36:37 +0530 Subject: [PATCH 649/658] Update default filename --- script/get-platform-details/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-platform-details/customize.py b/script/get-platform-details/customize.py index 9c26dae026..c618fa0194 100644 --- a/script/get-platform-details/customize.py +++ b/script/get-platform-details/customize.py @@ -23,7 +23,7 @@ def preprocess(i): if env.get('CM_PLATFORM_DETAILS_DIR_PATH', '') == '': env['CM_PLATFORM_DETAILS_DIR_PATH'] = os.getcwd() if env.get('CM_PLATFORM_DETAILS_FILE_NAME', '') == '': - env['CM_PLATFORM_DETAILS_FILE_NAME'] = "system_info.txt" + env['CM_PLATFORM_DETAILS_FILE_NAME'] = "system-info.txt" env['CM_PLATFORM_DETAILS_FILE_PATH'] = os.path.join(env['CM_PLATFORM_DETAILS_DIR_PATH'], env['CM_PLATFORM_DETAILS_FILE_NAME']) return {'return':0} From 12bd285cd2b2506e11f5f0b64cc6cbc022c988f8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:37:51 +0530 Subject: [PATCH 650/658] Updated the file path shown to user --- script/get-platform-details/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index ba2194e669..cab85ca55d 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -135,4 +135,4 @@ else fi echo "------------------------------------------------------------" >> $OUTPUT_FILE -echo "System information has been saved to $PWD/$OUTPUT_FILE" +echo "System information has been saved to $OUTPUT_FILE" From a056d725f31e7ad2b79ae9a99408ed9cf627a0dc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 10:32:47 +0000 Subject: [PATCH 651/658] Update test-mlperf-inference-dlrm.yml --- .github/workflows/test-mlperf-inference-dlrm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-dlrm.yml b/.github/workflows/test-mlperf-inference-dlrm.yml index 5024336aef..6440d04483 100644 --- a/.github/workflows/test-mlperf-inference-dlrm.yml +++ b/.github/workflows/test-mlperf-inference-dlrm.yml @@ -25,7 +25,7 @@ jobs: export CM_REPOS=$HOME/GH_CM python3 -m pip install cm4mlops cm pull repo - cm run script --tags=run-mlperf,inference,_performance-only --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --backend=pytorch --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --clean + cm run script --tags=run-mlperf,inference,_performance-only --adr.mlperf-implementation.tags=_branch.dev --adr.mlperf-implementation.version=custom --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --backend=pytorch --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --clean build_intel: if: github.repository_owner == 'gateoverflow_off' From 1dd0379b94d3c2eda3d70241e9fa8951080a8efc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 10:37:35 +0000 Subject: [PATCH 652/658] Update test-mlperf-inference-mixtral.yml --- .github/workflows/test-mlperf-inference-mixtral.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-mixtral.yml b/.github/workflows/test-mlperf-inference-mixtral.yml index 5b4a5ea959..1ad3f15134 100644 --- a/.github/workflows/test-mlperf-inference-mixtral.yml +++ b/.github/workflows/test-mlperf-inference-mixtral.yml @@ -5,7 +5,7 @@ name: MLPerf inference MIXTRAL-8x7B on: schedule: - - cron: "30 03 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST + - cron: "45 10 * * *" # 30th minute and 20th hour => 20:30 UTC => 2 AM IST jobs: build_reference: @@ -24,7 +24,9 @@ jobs: source gh_action/bin/deactivate || python3 -m venv gh_action source gh_action/bin/activate export CM_REPOS=$HOME/GH_CM - python3 -m pip install cm4mlops + pip install cm4mlops + pip install "huggingface_hub[cli]" + huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential cm pull repo cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=mixtral-8x7b --implementation=reference --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=1 --target_qps=1 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from self hosted Github actions - GO-i9" --quiet --submission_dir=$HOME/gh_action_submissions From 24cd3cb911b9ce07ed3c890d95a4fe17bf3c4124 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:41:20 +0530 Subject: [PATCH 653/658] Response when cmd without sudo fails --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bab35447dd..29af8c9dcd 100644 --- a/setup.py +++ b/setup.py @@ -84,8 +84,11 @@ def install_system_packages(self): subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) except subprocess.CalledProcessError: print("Sudo command failed, trying without sudo.") - subprocess.check_call(['apt-get', 'update']) - subprocess.check_call(['apt-get', 'install', '-y'] + packages) + try: + subprocess.check_call(['apt-get', 'update']) + subprocess.check_call(['apt-get', 'install', '-y'] + packages) + except subprocess.CalledProcessError: + print(f"Installation of {packages} without sudo failed. Please install these packages manually to continue!") elif self.system == 'Windows': print(f"Please install the following packages manually: {packages}") From e65f6bde809f4117bc94e9105f2e0cc6a6331705 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 17:55:43 +0530 Subject: [PATCH 654/658] Added torch_cuda for AMD llama2 quantization --- script/get-ml-model-llama2/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index 839cba1a0d..bd88acb53c 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -188,7 +188,7 @@ "tags": "get,generic-python-lib,_package.nltk" }, { - "tags": "get,generic-python-lib,_torch" + "tags": "get,generic-python-lib,_torch_cuda" }, { "tags": "get,generic-python-lib,_package.compressed_tensors" From 2a5e2006ff3bff8affbfe950b9b40105b7db3a38 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 12:28:51 +0000 Subject: [PATCH 655/658] Delete project directory --- .../README.md | 10 - .../_cm.json | 7 - ...wd-benchmark-mlperf-bert-inference-cuda.md | 285 ------------------ .../docs/generate-bert-submission.md | 87 ------ .../docs/generate-resnet50-submission.md | 74 ----- .../docs/run-nvidia-implementation.md | 47 --- .../docs/setup-aws-graviton.md | 25 -- .../get-mlperf-inference-repos.cmd | 3 - 8 files changed, 538 deletions(-) delete mode 100644 project/mlperf-inference-v3.0-submissions/README.md delete mode 100644 project/mlperf-inference-v3.0-submissions/_cm.json delete mode 100644 project/mlperf-inference-v3.0-submissions/docs/crowd-benchmark-mlperf-bert-inference-cuda.md delete mode 100644 project/mlperf-inference-v3.0-submissions/docs/generate-bert-submission.md delete mode 100644 project/mlperf-inference-v3.0-submissions/docs/generate-resnet50-submission.md delete mode 100644 project/mlperf-inference-v3.0-submissions/docs/run-nvidia-implementation.md delete mode 100644 project/mlperf-inference-v3.0-submissions/docs/setup-aws-graviton.md delete mode 100644 project/mlperf-inference-v3.0-submissions/get-mlperf-inference-repos.cmd diff --git a/project/mlperf-inference-v3.0-submissions/README.md b/project/mlperf-inference-v3.0-submissions/README.md deleted file mode 100644 index 7ad8080b0c..0000000000 --- a/project/mlperf-inference-v3.0-submissions/README.md +++ /dev/null @@ -1,10 +0,0 @@ -Graphs: - https://cknowledge.org/cm-gui-graph/?tags=mlperf-inference,all,open,edge,image-classification,singlestream - https://cknowledge.org/cm-gui-graph/?tags=mlperf-inference,v3.0,open,edge,image-classification,singlestream&x=Result&y=Accuracy - - http://localhost:8501/?tags=mlperf-inference,v3.0,open,edge,image-classification,singlestream&x=Result&y=Accuracy - http://localhost:8501/?tags=mlperf-inference,all,open,edge,image-classification,singlestream&x=Result&y=Accuracy - -Local: - cm run script "get git repo _repo.https://github.com/mlcommons/inference_results_v2.1" --env.CM_GIT_CHECKOUT=master --extra_cache_tags=mlperf-inference-results,version-2.1 - cm run script "gui _graph" diff --git a/project/mlperf-inference-v3.0-submissions/_cm.json b/project/mlperf-inference-v3.0-submissions/_cm.json deleted file mode 100644 index 2cc81aa8b0..0000000000 --- a/project/mlperf-inference-v3.0-submissions/_cm.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "alias": "mlperf-inference-v3.0-submissions", - "automation_alias": "project", - "automation_uid": "6882553224164c56", - "tags": [], - "uid": "f571becbcbd44a7d" -} diff --git a/project/mlperf-inference-v3.0-submissions/docs/crowd-benchmark-mlperf-bert-inference-cuda.md b/project/mlperf-inference-v3.0-submissions/docs/crowd-benchmark-mlperf-bert-inference-cuda.md deleted file mode 100644 index 9aae9bbe55..0000000000 --- a/project/mlperf-inference-v3.0-submissions/docs/crowd-benchmark-mlperf-bert-inference-cuda.md +++ /dev/null @@ -1,285 +0,0 @@ -# Crowd-benchmarking MLPerf BERT inference - -
-Click here to see the table of contents. - -* [Crowd-benchmarking MLPerf BERT inference](#crowd-benchmarking-mlperf-bert-inference) -* [System preparation](#system-preparation) - * [Minimal system requirements](#minimal-system-requirements) - * [Install CM (CK2) automation meta-framework](#install-cm-ck2-automation-meta-framework) - * [Pull CM repository with portable automation recipes](#pull-cm-repository-with-portable-automation-recipes) - * [Detect or install CUDA](#detect-or-install-cuda) - * [Test CUDA installation](#test-cuda-installation) - * [Install Python virtual environment](#install-python-virtual-environment) - * [Detect or install cuDNN](#detect-or-install-cudnn) - * [Detect or install TensorRT](#detect-or-install-tensorrt) - * [Run MLPerf inference benchmark with BERT](#run-mlperf-inference-benchmark-with-bert) - * [Try ONNX runtime backend](#try-onnx-runtime-backend) - * [Do a test run to detect and record the system performance](#do-a-test-run-to-detect-and-record-the-system-performance) - * [Do a full accuracy run for all the scenarios](#do-a-full-accuracy-run-for-all-the-scenarios) - * [Do a full performance run for all the scenarios](#do-a-full-performance-run-for-all-the-scenarios) - * [Populate the README files](#populate-the-readme-files) - * [Generate MLPerf submission tree](#generate-mlperf-submission-tree) - * [Push the results to GitHub repo](#push-the-results-to-github-repo) - * [Try PyTorch backend](#try-pytorch-backend) - * [Test composable ML benchmark with other models, data sets, frameworks and platforms](#test-composable-ml-benchmark-with-other-models-data-sets-frameworks-and-platforms) -* [The next steps](#the-next-steps) - -
- - -This is a pilot community project to collaboratively run MLPerf BERT inference benchmark -across diverse platforms provided by volunteers similar to [SETI@home](https://setiathome.berkeley.edu/). -However, instead of searching for extraterrestrial intelligence, we are -searching for optimal software/hardware combination to run various AI and ML workloads -in terms of performance, accuracy, power and costs ... - -This benchmark is composed from [portable and reusable automation recipes](https://github.com/mlcommons/ck/blob/master/docs/list_of_scripts.md) -developed by [MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) -to modularize complex AI and ML Systems and automate their benchmarking, design space exploration, optimization and deployment -across continuously evolving software, hardware, models and data. - -*If you submit your results before 1pm PST on Friday 3rd, 2023, - they will be accepted for the official MLPerf inference v3.0 submission round - and your name acknowledged in the notes!* - - -# System preparation - -## Minimal system requirements - -* CPU: any x86-64 or Arm64 based machine -* GPU: any relatively modern Nvidia GPU with 8GB+ memory and CUDA 11.4+ -* OS: we have tested this automation on Ubuntu 20.04, Ubuntu 22.04 and Debian 10 -* Disk space: ~10GB -* Python: 3.8+ -* All other dependencies (artifacts and tools) will be installed by the CM meta-framework aka (CK2) - -## Install CM (CK2) automation meta-framework - -Follow [this guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install the [MLCommons CM framework](https://github.com/mlcommons/ck) -(the 2nd generation on the Collective Mind framework) on your system. - -## Pull CM repository with portable automation recipes - -Pull MLCommons CM repository with [cross-platform CM scripts](https://github.com/mlcommons/ck/blob/master/docs/list_of_scripts.md) -supporting portable MLOps and DevOps: - -```bash -cm pull repo mlcommons@ck -``` - -CM pulls all such repositories into the `$HOME/CM` directory to search for portable CM automation recipes and artifacts. - -We use the unified CM CLI & Python API of [portable and reusable CM scripts](https://github.com/mlcommons/ck/blob/master/docs/list_of_scripts.md) -to compose portable automation pipelines (also implemented as CM scripts) that can automatically detect or install all necessary artifacts (tools, models, datasets, libraries, etc) -required to run a given software project such as the MLPerf inference benchmark. - -These CM scripts simply wrap existing native scripts and tools as simple micro-services -with a human-readable CLI and simple Python API to be able to easily connect them together -and run on any platform in a unified way. - -## Detect or install CUDA - -Run the following CM script: -```bash -cm run script "get cuda" --out=json -``` - -If CUDA is automatically detected, it will be registered in the CM cache: -```bash -cm show cache --tags=get,cuda -``` - -Otherwise, this script will attempt to download and install the latest CUDA -from Nvidia website. - -Please report any issue with CM scripts [here](https://github.com/mlcommons/ck/issues). - -### Test CUDA installation - -You can test if CUDA toolkit and driver was detected or installed successfully using the following command: -```bash -cm run script "get cuda-devices" -``` - -You should see similar output: -```txt -Checking compiler version ... - -nvcc: NVIDIA (R) Cuda compiler driver -Copyright (c) 2005-2022 NVIDIA Corporation -Built on Wed_Sep_21_10:33:58_PDT_2022 -Cuda compilation tools, release 11.8, V11.8.89 -Build cuda_11.8.r11.8/compiler.31833905_0 - -Compiling program ... - -Running program ... - - - Running postprocess ... -GPU Device ID: 0 -GPU Name: Tesla K80 -GPU compute capability: 3.7 -CUDA driver version: 11.4 -CUDA runtime version: 11.8 -Global memory: 11997020160 -Max clock rate: 823.500000 MHz -Total amount of shared memory per block: 49152 -Total number of registers available per block: 65536 -Warp size: 32 -Maximum number of threads per multiprocessor: 2048 -Maximum number of threads per block: 1024 -Max dimension size of a thread block X: 1024 -Max dimension size of a thread block Y: 1024 -Max dimension size of a thread block Z: 64 -Max dimension size of a grid size X: 2147483647 -Max dimension size of a grid size Y: 65535 -Max dimension size of a grid size Z: 65535 - - - running time of script "get,cuda-devices": 4.16 sec. - -``` - -## Install Python virtual environment - -```bash -cm run script "get sys-utils-cm" --quiet - -cm run script "install python-venv" --name=mlperf-cuda -``` - -If you want to install specific version of Python use the following command: -```bash -cm run script "install python-venv" --version=3.10.8 --name=mlperf-cuda -``` - -## Detect or install cuDNN - -```bash -cm run script "get cudnn" -``` - -If cuDNN is not detected on your system, you can download a TAR file -from [Nvidia website](https://developer.nvidia.com/cudnn) and then use the same CM script -to install it as follows: -```bash -cm run script "get cudnn" --tar_file= -``` - -We have tested this project with the following tar file `cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz`. - -## Detect or install TensorRT - -```bash -cm run script "get tensorrt" -``` -If TensorRT is not detected on your system, you can download a TAR file -from [Nvidia website](https://developer.nvidia.com/tensorrt) and then use the same CM script -to install it as follows: -```bash -cm run script "get tensorrt" --tar_file= -``` - -We have tested this project with the following tar file `TensorRT-8.5.1.7.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz`. - - -## Run MLPerf inference benchmark with BERT - -### Try ONNX runtime backend - -#### Do a test run to detect and record the system performance - -```bash -cm run script --tags=generate-run-cmds,inference,_find-performance,_all-scenarios \ - --adr.python.name=mlperf-cuda --model=bert-99 --implementation=reference \ - --device=cuda --backend=onnxruntime --quiet -``` - -#### Do a full accuracy run for all the scenarios - -```bash -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios \ - --adr.python.name=mlperf-cuda --model=bert-99 --device=cuda \ - --implementation=reference --backend=onnxruntime --quiet \ - --execution-mode=valid --results_dir=$HOME/inference_3.0_results -``` - -#### Do a full performance run for all the scenarios - -```bash -cm run script --tags=generate-run-cmds,inference,_performance-only,_all-scenarios \ - --adr.python.name=mlperf-cuda --model=bert-99 --device=cuda \ - --implementation=reference --backend=onnxruntime --quiet \ - --execution-mode=valid --results_dir=$HOME/inference_3.0_results -``` - -#### Populate the README files - -```bash -cm run script --tags=generate-run-cmds,inference,_populate-readme,_all-scenarios \ - --adr.python.name=mlperf-cuda --model=bert-99 --device=cuda \ - --implementation=reference --backend=onnxruntime --quiet \ - --execution-mode=valid --results_dir=$HOME/inference_3.0_results -``` - -#### Generate MLPerf submission tree - -We should use the master branch of MLCommons inference repo for the submission checker. -You can use `--hw_note_extra` option to add your name to the notes. - -```bash -cm run script --tags=generate,inference,submission \ - --results_dir=$HOME/inference_3.0_results/valid_results \ - --adr.python.name=mlperf-cuda \ - --device=cuda --submission_dir=$HOME/inference_submission_tree --clean \ - --run-checker --submitter=cTuning --adr.inference-src.version=master - --hw_notes_extra="Result taken by " --quiet -``` - -#### Push the results to GitHub repo - -First create a fork of [this GitHub repo with aggregated results](https://github.com/ctuning/mlperf_inference_submissions_v3.0). -Then run the following command after replacing `--repo_url` with your fork URL. - -```bash -cm run script --tags=push,github,mlperf,inference,submission \ - --submission_dir=$HOME/inference_submission_tree \ - --adr.python.name=mlperf-cuda \ - --repo_url=https://github.com/ctuning/mlperf_inference_submissions_v3.0 \ - --commit_message="Bert crowd-results added" -``` - -Create a PR to the [GitHub repo with aggregated results](https://github.com/ctuning/mlperf_inference_submissions_v3.0/) - - - -### Try PyTorch backend - -You can run the same commands with PyTorch by rerunning all above commands and replacing `--backend=onnxruntime` with `--backend=pytorch`. - -For example, - -```bash -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios \ - --adr.python.name=mlperf-cuda --model=bert-99 --device=cuda \ - --implementation=reference --backend=pytorch --execution-mode=valid \ - --results_dir=$HOME/inference_3.0_results --quiet -``` - - -## Test composable ML benchmark with other models, data sets, frameworks and platforms - -* [GUI to prepare CM command line and run benchmark](https://cknowledge.org/mlperf-inference-gui) -* [GUI to compare performance, accuracy, power and costs of ML/SW/HW combinations](https://cKnowledge.org/cm-gui-graph) - - -# The next steps - -Feel free to join our [open taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) -and the public [Discord server](https://discord.gg/JjWNWXKxwT) to learn about our roadmap and related community projects. - -Our ultimate goal is to help anyone automatically find or generate the optimal software/hardware stack from the cloud to the edge -for their AI/ML tasks based on their requrements and constraints (accuracy, performance, power consumption, costs, etc). - -*Prepared by [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) and [Grigori Fursin](https://cKnowledge.org/gfursin) (OctoML, MLCommons, cTuning foundation)* diff --git a/project/mlperf-inference-v3.0-submissions/docs/generate-bert-submission.md b/project/mlperf-inference-v3.0-submissions/docs/generate-bert-submission.md deleted file mode 100644 index 824279732e..0000000000 --- a/project/mlperf-inference-v3.0-submissions/docs/generate-bert-submission.md +++ /dev/null @@ -1,87 +0,0 @@ -## Setup -Please follow the MLCommons CK [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install CM. -Download the ck repo to get the CM script for MLPerf submission -``` -cm pull repo mlcommons@ck -``` -## Run Commands - -Bert has two variants - `bert-99` and `bert-99.9` where the `99` and `99.9` specifies the required accuracy constraint with respect to the reference floating point model. `bert-99.9` model is applicable only on a datacenter system. - -On edge category `bert-99` has Offline and SingleStream scenarios and in datacenter category both `bert-99` and `bert-99.9` have Offline and Server scenarios. The below commands are assuming an edge category system. - -### Onnxruntime backend - -#### Do a test run to detect and record the system performance - -``` -cm run script --tags=generate-run-cmds,inference,_find-performance,_all-scenarios \ ---model=bert-99 --implementation=reference --device=cpu --backend=onnxruntime --quiet -``` -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for a closed division including the compliance tests -* Use `--category=datacenter` to run datacenter scenarios - -#### Do a full accuracy run for all the scenarios - -``` -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios \ ---model=bert-99 --device=cpu --implementation=reference --backend=onnxruntime \ ---execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -#### Do a full performance run for all the scenarios -``` -cm run script --tags=generate-run-cmds,inference,_performance-only,_all-scenarios \ ---model=bert-99 --device=cpu --implementation=reference --backend=onnxruntime \ ---execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -#### Populate the README files -``` -cm run script --tags=generate-run-cmds,inference,_populate-readme,_all-scenarios \ ---model=bert-99 --device=cpu --implementation=reference --backend=onnxruntime \ ---execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -#### Generate actual submission tree - -We should use the master branch of MLCommons inference repo for the submission checker. You can use `--hw_note_extra` option to add your name to the notes. -``` -cm run script --tags=generate,inference,submission --results_dir=$HOME/inference_3.0_results/valid_results \ ---device=cpu --submission_dir=$HOME/inference_submission_tree --clean --run-checker --submitter=cTuning ---adr.inference-src.version=master --hw_notes_extra="Result taken by NAME" --quiet -``` - -#### Push the results to GitHub repo - -First create a fork of [this repo](https://github.com/ctuning/mlperf_inference_submissions_v3.0/). Then run the following command after replacing `--repo_url` with your fork URL. -``` -cm run script --tags=push,github,mlperf,inference,submission \ ---submission_dir=$HOME/inference_submission_tree \ ---repo_url=https://github.com/ctuning/mlperf_inference_submissions_v3.0/ \ ---commit_message="Bert results added" -``` - -Create a PR to [cTuning repo](https://github.com/ctuning/mlperf_inference_submissions_v3.0/) - -## Tensorflow backend - -Same commands as for `onnxruntime` should work by replacing `backend=onnxruntime` with `--backend=tf`. For example, - -``` -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios \ ---model=bert-99 --device=cpu --implementation=reference --backend=tf --execution-mode=valid \ ---results_dir=$HOME/inference_3.0_results --quiet -``` - -## Pytorch backend - -Same commands as for `onnxruntime` should work by replacing `backend=onnxruntime` with `--backend=pytorch`. For example, - -``` -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios \ ---model=bert-99 --device=cpu --implementation=reference --backend=pytorch \ ---execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - diff --git a/project/mlperf-inference-v3.0-submissions/docs/generate-resnet50-submission.md b/project/mlperf-inference-v3.0-submissions/docs/generate-resnet50-submission.md deleted file mode 100644 index 9129004321..0000000000 --- a/project/mlperf-inference-v3.0-submissions/docs/generate-resnet50-submission.md +++ /dev/null @@ -1,74 +0,0 @@ -## Run Commands - -We need to get imagenet full dataset to make image-classification submissions for MLPerf inference. Since this dataset is not publicly available via a URL please follow the instructions given [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/get-dataset-imagenet-val/README-extra.md) to download the dataset and register in CM. - -On edge category ResNet50 has Offline, SingleStream and MultiStream scenarios and in datacenter category it has Offline and Server scenarios. The below commands are assuming an edge category system. - -### Onnxruntime backend - -#### Do a test run to detect and record the system performance - -``` -cm run script --tags=generate-run-cmds,inference,_find-performance,_full,_all-scenarios --model=resnet50 \ ---device=cpu --backend=onnxruntime --quiet -``` -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for a closed division including the compliance tests -* Use `--category=datacenter` to run datacenter scenarios - -#### Do a full accuracy run for all the scenarios - -``` -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios --model=resnet50 --device=cpu \ ---implementation=reference --backend=onnxruntime --execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -#### Do a full performance run for all the scenarios -``` -cm run script --tags=generate-run-cmds,inference,_performance-only,_all-scenarios --model=resnet50 --device=cpu \ ---implementation=reference --backend=onnxruntime --execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -#### Populate the README files -``` -cm run script --tags=generate-run-cmds,inference,_populate-readme,_all-scenarios --model=resnet50 --device=cpu \ ---implementation=reference --backend=onnxruntime --execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -#### Generate actual submission tree - -We should use the master branch of MLCommons inference repo for the submission checker. You can use `--hw_note_extra` option to add your name to the notes. -``` -cm run script --tags=generate,inference,submission --results_dir=$HOME/inference_3.0_results/valid_results \ ---submission_dir=$HOME/inference_submission_tree --clean \ ---run-checker --submitter=cTuning --adr.inference-src.version=master --hw_notes_extra="Result taken by NAME" --quiet -``` - -#### Push the results to GitHub repo - -First create a fork of [this repo](https://github.com/ctuning/mlperf_inference_submissions_v3.0/). Then run the following command after replacing `--repo_url` with your fork URL. -``` -cm run script --tags=push,github,mlperf,inference,submission --submission_dir=$HOME/inference_submission_tree \ ---repo_url=https://github.com/ctuning/mlperf_inference_submissions_v3.0/ \ ---commit_message="ResNet50 results added" -``` - -Create a PR to [cTuning repo](https://github.com/ctuning/mlperf_inference_submissions_v3.0/) - -## Tensorflow backend - -Same commands as for `onnxruntime` should work by replacing `backend=onnxruntime` with `--backend=tf`. For example, - -``` -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios --model=resnet50 --device=cpu \ ---implementation=reference --backend=tf --execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` - -## TVM backend - -Same commands as for `onnxruntime` should work by replacing `backend=onnxruntime` with `--backend=tvm-onnx`. (Only `--device=cpu` is currently supported for TVM) For example, - -``` -cm run script --tags=generate-run-cmds,inference,_accuracy-only,_all-scenarios --model=resnet50 --device=cpu \ ---implementation=reference --backend=tvm-onnx --execution-mode=valid --results_dir=$HOME/inference_3.0_results --quiet -``` diff --git a/project/mlperf-inference-v3.0-submissions/docs/run-nvidia-implementation.md b/project/mlperf-inference-v3.0-submissions/docs/run-nvidia-implementation.md deleted file mode 100644 index c35aada995..0000000000 --- a/project/mlperf-inference-v3.0-submissions/docs/run-nvidia-implementation.md +++ /dev/null @@ -1,47 +0,0 @@ -## Run Commands - -We need to get imagenet full dataset to make image-classification submissions for MLPerf inference. Since this dataset is not publicly available via a URL please follow the instructions given [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/get-dataset-imagenet-val/README-extra.md) to download the dataset and register in CM. - - -Requirements: You need to have CUDA, cuDNN and TensorRT installed on your system. - -If CUDA is not detected, CM should download and install it automatically when you run the workflow. - -For x86 machines, you can download the tar files for cuDNN and TensorRT and install them using the following commands -```bash -cm run script --tags=get,cudnn --tar_file= -``` - -```bash -cm run script --tags=get,tensorrt --tar_file= -``` - -On other systems you can do a package manager install and then CM should pick up the installation automatically during the workflow run. - -Nvidia run configuration values for each model-sceraio for known systems are stored in `__init__.py` files under configs directory. For custom systems these are stored under `custom.py` files. When custom config files are generated they override the default config values with empty ones (not desirable). So, you'll probably need to open the custom config file and comment out the overrides. Typically `gpu_batch_size` and `offline_expected_qps` are enough for an offline scenario run on a typical single GPU system. - - -## Build Nvidia Inference Server -``` -cm run script --tags=build,nvidia,inference,server -``` - -## Run ResNet50 - -### Find SUT performance - -``` -cm run script --tags=generate,run-cmds,inference,_find-performance --model=resnet50 --implementation=nvidia-original \ ---device=cuda --adr.nvidia-harness.gpu_batch_size=64 --results_dir=$HOME/nvidia_original_results -``` - -### Do a complete submission run - -``` -cm run script --tags=generate,run-cmds,inference,_submission,_full --execution_mode=valid --model=resnet50 \ ---implementation=nvidia-original --device=cuda --adr.nvidia-harness.gpu_batch_size=64 \ ---adr.nvidia-harness.skip_preprocess=yes --adr.nvidia-harness.make_cmd=run_harness \ ---results_dir=$HOME/nvidia_original_results --submission_dir=$HOME/nvidia_original_submissions \ ---division=open --submitter=cTuning --category=edge -``` - diff --git a/project/mlperf-inference-v3.0-submissions/docs/setup-aws-graviton.md b/project/mlperf-inference-v3.0-submissions/docs/setup-aws-graviton.md deleted file mode 100644 index cb74086b54..0000000000 --- a/project/mlperf-inference-v3.0-submissions/docs/setup-aws-graviton.md +++ /dev/null @@ -1,25 +0,0 @@ -## Run Commands - -We need to get imagenet full dataset to make image-classification submissions for MLPerf inference. Since this dataset is not publicly available via a URL please follow the instructions given [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/get-dataset-imagenet-val/README-extra.md) to download the dataset and register in CM. - -## Create an AWS Graviton Instance - -``` -cd $HOME/CM/repos/mlcommon@ck/cm-mlops/script/run-terraform/aws/ -cp credentials.example credentials.sh -``` -Update `credentials.sh` with your AWS Key, Secret and Token - -``` -cm run script --tags=run,terraform,_m7g.xlarge,_storage_size.500,_ubuntu.2204,_us-west-2 \ ---cminit --key_file=$HOME/cmuser.pem -``` - -The above command will output the IP of the created instance which will be having CM setup already done - -Copy the imagenet dataset to the created instance. For example, - -``` -rsync -avz -e 'ssh -i $HOME/cmuser.pem' $HOME/imagenet-2012-val/ ubuntu@54.189.93.134: -``` - diff --git a/project/mlperf-inference-v3.0-submissions/get-mlperf-inference-repos.cmd b/project/mlperf-inference-v3.0-submissions/get-mlperf-inference-repos.cmd deleted file mode 100644 index 87fa4e9ba2..0000000000 --- a/project/mlperf-inference-v3.0-submissions/get-mlperf-inference-repos.cmd +++ /dev/null @@ -1,3 +0,0 @@ -cm run script "get git repo _repo.https://github.com/ctuning/mlperf_inference_submissions_v3.0" --extra_cache_tags=mlperf-inference-results,version-3.0 -cm run script "get git repo _repo.https://github.com/mlcommons/inference_results_v2.1" --env.CM_GIT_CHECKOUT=master --extra_cache_tags=mlperf-inference-results,version-2.1 -cm run script "get git repo _repo.https://github.com/mlcommons/inference_results_v2.0" --env.CM_GIT_CHECKOUT=master --extra_cache_tags=mlperf-inference-results,version-2.0 From 439285432aaa61a6262ab67d0a03cf2aaeb94d4f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Nov 2024 12:34:28 +0000 Subject: [PATCH 656/658] Update test-amd-mlperf-inference-implementations.yml --- .github/workflows/test-amd-mlperf-inference-implementations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-amd-mlperf-inference-implementations.yml b/.github/workflows/test-amd-mlperf-inference-implementations.yml index deff519672..f753fab8d1 100644 --- a/.github/workflows/test-amd-mlperf-inference-implementations.yml +++ b/.github/workflows/test-amd-mlperf-inference-implementations.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.12" ] - model: [ "llama2-70b" ] + model: [ "llama2-70b-99.9" ] steps: - name: Test MLPerf Inference AMD (build only) ${{ matrix.model }} run: | From 11d77ded973b5af4544cb6bb35b9a0ac82da7d44 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:44:16 +0530 Subject: [PATCH 657/658] fix for system without sudo --- setup.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 29af8c9dcd..fed5bd91d1 100644 --- a/setup.py +++ b/setup.py @@ -79,16 +79,15 @@ def install_system_packages(self): if manager == "apt-get": # Check if 'sudo' is available if shutil.which('sudo'): + subprocess.check_call(['sudo', 'apt-get', 'update']) + subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + else: + print("sudo not found, trying without sudo.") try: - subprocess.check_call(['sudo', 'apt-get', 'update']) - subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + subprocess.check_call(['apt-get', 'update']) + subprocess.check_call(['apt-get', 'install', '-y'] + packages) except subprocess.CalledProcessError: - print("Sudo command failed, trying without sudo.") - try: - subprocess.check_call(['apt-get', 'update']) - subprocess.check_call(['apt-get', 'install', '-y'] + packages) - except subprocess.CalledProcessError: - print(f"Installation of {packages} without sudo failed. Please install these packages manually to continue!") + print(f"Installation of {packages} without sudo failed. Please install these packages manually to continue!") elif self.system == 'Windows': print(f"Please install the following packages manually: {packages}") From 9aecee87a3027de8e20c63f9976584850a09ed9e Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Fri, 8 Nov 2024 14:33:57 +0000 Subject: [PATCH 658/658] Increment version to 0.3.25 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cfe389e945..88dbf46f41 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.24 +0.3.25