Merge pull request #517 from mlcommons/main

Sync from main

.github/CODEOWNERS

@@ -2,4 +2,8 @@
 # Unless a later match takes precedence,they will be requested for review when someone opens a pull request.
 * @mlcommons/wg-ck 
 
-/CODEOWNERS @mlcommons/staff
+/.github/CODEOWNERS @mlcommons/systems
+
+/.github/workflows/cla.yml @mlcommons/systems
+
+/LICENSE.md @mlcommons/systems

.github/workflows/build_wheel.yml

@@ -0,0 +1,97 @@
+name: Build wheel and release into PYPI
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+      - mlperf-inference
+    paths:
+      - VERSION
+      - setup.py
+
+
+jobs:
+  build_wheels:
+    if: github.repository_owner == 'mlcommons'
+    name: Build wheel
+    runs-on: ubuntu-latest
+    environment: release
+
+    permissions:
+      id-token: write
+      contents: write
+
+    strategy:
+      fail-fast: false
+    steps:
+      # Step 1: Checkout the code
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      # Step 2: Set up Python
+      - uses: actions/setup-python@v3
+
+      # Step 3: Check if VERSION file has changed in this push
+      - name: Check if VERSION file has changed
+        id: version_changed
+        run: |
+          if git diff --name-only HEAD~1 | grep -q "VERSION"; then
+            echo "VERSION file has been modified"
+            echo "::set-output name=version_changed::true"
+            new_version=$(cat VERSION)
+          else
+            echo "VERSION file has NOT been modified"
+            echo "::set-output name=version_changed::false"
+          fi
+          echo "::set-output name=new_version::$new_version"
+
+      # Step 4: Increment version if VERSION was not changed
+      - name: Increment version if necessary
+        id: do_version_increment
+        if: steps.version_changed.outputs.version_changed == 'false'
+        run: |
+          # Check if VERSION file exists, else initialize it
+          if [ ! -f VERSION ]; then
+            echo "0.0.0" > VERSION
+          fi
+          
+          version=$(cat VERSION)
+          IFS='.' read -r major minor patch <<< "$version"
+          patch=$((patch + 1))
+          new_version="$major.$minor.$patch"
+          echo $new_version > VERSION
+          echo "New version: $new_version"
+          echo "::set-output name=new_version::$new_version"
+
+      # Step 5: Commit the updated version to the repository
+      - name: Commit updated version
+        if: steps.version_changed.outputs.version_changed == 'false'
+        run: |
+          git config --global user.name "${{ github.actor }}"
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git add VERSION
+          git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}"
+          git push
+
+      # Step 6: Install required dependencies
+      - name: Install requirements
+        run: python3 -m pip install setuptools wheel build
+
+      # Step 7: Build the Python wheel
+      - name: Build wheels
+        working-directory: ./
+        run: python3 -m build && rm dist/*.whl
+
+      # Step 8: Publish to PyPI
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verify-metadata: true
+          skip-existing: true
+          packages-dir: dist
+          repository-url: https://upload.pypi.org/legacy/
+          verbose: true

.github/workflows/publish.yaml → .github/workflows/publish-docs.yaml

@@ -1,6 +1,6 @@
 # This is a basic workflow to help you get started with Actions
 
-name: Publish site
+name: Publish docs site
 
 
 on:
@@ -11,6 +11,9 @@ on:
       - main
       - docs
       - mlperf-inference
+    paths:
+      - docs/**
+      - mkdocs.yml
 
 jobs:
 

.github/workflows/run-individual-script-tests.yml

@@ -0,0 +1,37 @@
+# This workflow will run configured tests for any updated CM scripts
+name: Individual CM script Tests
+
+on:
+  pull_request:
+    branches: [ "main", "mlperf-inference", "dev" ]
+    paths:
+      - 'script/**_cm.json'
+      - 'script/**_cm.yml'
+
+jobs:
+  run-script-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test-input-index: [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11+" ]
+    steps:
+    - name: 'Checkout'
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+    - name: Get changed files
+      id: getfile
+      run: |
+        git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
+        git fetch upstream
+        echo "files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT
+    - name: RUN Script Tests
+      run: |
+        echo ${{ steps.getfile.outputs.files }}
+        for file in ${{ steps.getfile.outputs.files }}; do
+          echo $file
+        done
+        python3 -m pip install cmind
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+        DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }}

.github/workflows/test-amd-mlperf-inference-implementations.yml

@@ -0,0 +1,26 @@
+name: MLPerf Inference AMD implementations
+
+on:
+  schedule:
+    - cron: "29 4 * * *" #to be adjusted
+
+jobs:
+  build_nvidia:
+      if: github.repository_owner == 'gateoverflow'
+      runs-on: [ self-hosted, linux, x64, GO-spr ]
+      strategy:
+        fail-fast: false
+        matrix:
+          python-version: [ "3.12" ]
+          model: [ "llama2-70b-99.9" ]
+      steps:
+      - name: Test MLPerf Inference AMD (build only) ${{ matrix.model }}
+        run: |
+          if [ -f "gh_action_conda/bin/deactivate" ]; then source gh_action_conda/bin/deactivate; fi
+          python3 -m venv gh_action_conda
+          source gh_action_conda/bin/activate
+          export CM_REPOS=$HOME/GH_CM
+          pip install --upgrade cm4mlops
+          pip install tabulate
+          cm run script --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev  --execution_mode=valid  --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c  --implementation=amd    --backend=pytorch    --category=datacenter --division=open --scenario=Offline  --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=rocm  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  --docker --quiet
+          # cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c

.github/workflows/test-cm-based-submission-generation.yml

@@ -0,0 +1,55 @@
+# This workflow will test the submission generation capability of CM f
+
+name: CM based Submission Generation
+
+on:
+  pull_request:
+    branches: [ "main", "dev", "mlperf-inference" ]
+    paths:
+      - '.github/workflows/test-cm-based-submission-generation.yml'
+      - '**'  
+      - '!**.md'
+jobs:
+  submission_generation:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.12" ]
+        division: ["closed", "open"]
+        category: ["datacenter", "edge"]
+        case: ["case-3", "case-7"]
+        action: ["run", "docker"]
+        exclude:
+          - os: macos-latest
+          - os: windows-latest
+          - division: "open"
+          - category: "edge"
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+    - name: Pull repo where test cases are uploaded
+      run: |
+        git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests
+    - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} 
+      run: |
+        if [ "${{ matrix.case }}" == "case-3" ]; then
+          #results_dir="submission_generation_tests/case-3/"
+          description="Submission generation (model_mapping.json not present but model name matches with official one)"
+        elif [ "${{ matrix.case }}" == "case-7" ]; then
+          #results_dir="submission_generation_tests/case-7/"
+          description="Submission generation (sut_info.json incomplete, SUT folder name in required format)"
+        fi
+        # Dynamically set the log group to simulate a dynamic step name
+        echo "::group::$description"
+        cm ${{ matrix.action }} script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        echo "::endgroup::"
+  

.github/workflows/test-intel-mlperf-inference-implementations.yml

@@ -0,0 +1,26 @@
+name: MLPerf Inference Intel implementations
+
+on:
+  schedule:
+    - cron: "29 1 * * *" #to be adjusted
+
+jobs:
+  build_nvidia:
+      if: github.repository_owner == 'gateoverflow'
+      runs-on: [ self-hosted, linux, x64, GO-spr ]
+      strategy:
+        fail-fast: false
+        matrix:
+          python-version: [ "3.12" ]
+          model: [ "resnet50", "bert-99" ]
+      steps:
+      - name: Test MLPerf Inference Intel ${{ matrix.model }}
+        run: |
+          if [ -f "gh_action_conda/bin/deactivate" ]; then source gh_action_conda/bin/deactivate; fi
+          python3 -m venv gh_action_conda
+          source gh_action_conda/bin/activate
+          export CM_REPOS=$HOME/GH_CM
+          pip install --upgrade cm4mlops
+          pip install tabulate
+          cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid  --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c  --implementation=intel    --backend=pytorch    --category=datacenter --division=open --scenario=Offline  --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --device=cpu  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  --docker --quiet
+          cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=main --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c