diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e7eb099540439..3598e32166a809 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -108,8 +108,6 @@ /tools/ @openvinotoolkit/openvino-tools-maintainers /tools/benchmark_tool/ @openvinotoolkit/openvino-ie-python-api-maintainers /tools/legacy/ @openvinotoolkit/openvino-samples-maintainers -/tools/openvino_dev/ @openvinotoolkit/openvino-tools-maintainers @openvinotoolkit/openvino-ie-python-api-maintainers -/tools/mo/ @openvinotoolkit/openvino-mo-maintainers /tools/ovc/ @openvinotoolkit/openvino-ovc-maintainers /thirdparty/open_model_zoo/ @openvinotoolkit/omz-maintainers diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1511e6a2c30170..359ff683c9b22a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -41,7 +41,7 @@ updates: - "rkazants" versioning-strategy: increase-if-necessary - # Model Optimizer, openvino_dev and Benchmark tool + # ovc and Benchmark tools - package-ecosystem: pip directory: "/tools" schedule: diff --git a/.github/labeler.yml b/.github/labeler.yml index daa5375b175bd3..e9b2acb26c9072 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -100,10 +100,6 @@ 'category: LP transformations': - 'src/common/low_precision_transformations/**/*' -'category: MO': -- 'tools/mo/**/*' -- 'tests/layer_tests/mo_python_api_tests/**/*' - 'category: OVC': - 'tools/ovc/**/*' - 'tests/layer_tests/ovc_python_api_tests/**/*' @@ -119,7 +115,6 @@ - any: ['src/bindings/js/node/CMakeLists.txt', 'src/bindings/js/node/package.json', 'src/bindings/js/node/package-lock.json'] -- 'tools/openvino_dev/**/*' 'category: PDPD FE': - 'src/frontends/paddle/**/*' @@ -183,7 +178,6 @@ 'category: tools': - any: ['tools/**', - '!tools/mo/**/*', '!tools/ovc/**/*'] 'category: transformations': diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6cb0b2c5b6233c..cde1b9cf67e2fc 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -50,12 +50,6 @@ jobs: python3 -m pip install -r ${{ github.workspace }}/src/frontends/onnx/tests/requirements.txt # For running TensorFlow frontend unit tests python3 -m pip install -r ${{ github.workspace }}/src/frontends/tensorflow/tests/requirements.txt - # For MO unit tests - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_caffe.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_kaldi.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_onnx.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_tf2.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_dev.txt - name: Build OpenVINO with CMake uses: ashutoshvarma/action-cmake-build@ade188313bc7eaa6f14349569a64d8bc716342ff # master @@ -84,9 +78,6 @@ jobs: - name: Install wheel packages run: cmake -DCOMPONENT=python_wheels -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install_pkg -P '${{ github.workspace }}/build/cmake_install.cmake' - - name: Install python wheels - run: python3 -m pip install openvino-dev --find-links=${{ github.workspace }}/install_pkg/tools - - name: List binaries run: ls -la ${{ github.workspace }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }} diff --git a/.github/workflows/job_cxx_unit_tests.yml b/.github/workflows/job_cxx_unit_tests.yml index 8fab17043b7465..52a2b3f4d287c8 100644 --- a/.github/workflows/job_cxx_unit_tests.yml +++ b/.github/workflows/job_cxx_unit_tests.yml @@ -29,7 +29,7 @@ on: description: 'Timeout in minutes for the job' type: number required: false - default: 35 + default: 45 permissions: read-all diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml index c879f0cb6a1efc..39a4b467e74fc1 100644 --- a/.github/workflows/job_onnx_models_tests.yml +++ b/.github/workflows/job_onnx_models_tests.yml @@ -86,14 +86,11 @@ jobs: run: | # Install the core OV wheel python3 -m pip install ./openvino-*.whl - - extras_to_install="onnx" - - # Find and install OV dev wheel - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[$extras_to_install] working-directory: ${{ env.INSTALL_WHEELS_DIR }} + - name: Install ONNX Models tests requirements + run: python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_onnx + - name: Install Python tests dependencies run: | # To enable pytest parallel features diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index 64be9ef4bbcc44..8075f3299fe063 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -91,20 +91,11 @@ jobs: should-setup-pip-paths: ${{ runner.os == 'Linux' }} self-hosted-runner: ${{ runner.os == 'Linux' }} - # - # Tests - # - - name: Install OpenVINO Python wheels run: | # Install the core OV wheel python3 -m pip install ./openvino-*.whl - extras_to_install="caffe,kaldi,onnx,tensorflow2,pytorch" - - # Find and install OV dev wheel - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[$extras_to_install] working-directory: ${{ env.INSTALL_WHEELS_DIR }} - name: Install Python API tests dependencies @@ -112,7 +103,19 @@ jobs: # To enable pytest parallel features python3 -m pip install pytest-xdist[psutil] python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt - python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt + + - name: Install Python Layer tests dependencies and for OVC unit tests + run: | + # For torchvision to OpenVINO preprocessing converter + python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt + + # layer test requirements + python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt + + - name: Install ONNX tests dependencies + run: | + # ONNX tests requirements + python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_onnx # # Tests @@ -127,18 +130,6 @@ jobs: --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ --ignore=${INSTALL_TEST_DIR}/pyopenvino/tests/test_utils/test_utils.py - - name: Model Optimizer unit tests - if: fromJSON(inputs.affected-components).MO.test - run: | - if [[ "${{ runner.os }}" == "Linux" ]] && [[ "${{ runner.arch }}" != "ARM64" ]]; then - # required for MxNet - apt-get install -y libgomp1 libquadmath0 - fi - - # Skips under tickets: 133405, 122666 - python3 -m pytest -s ${INSTALL_TEST_DIR}/mo/unit_tests \ - --junitxml=${INSTALL_TEST_DIR}/TEST-ModelOptimizer.xml - - name: Python ONNX operators tests if: (fromJSON(inputs.affected-components).Python_API.test || fromJSON(inputs.affected-components).ONNX_FE.test) && @@ -153,32 +144,6 @@ jobs: if: fromJSON(inputs.affected-components).MO.test run: python3 -m pytest -s ${INSTALL_TEST_DIR}/ovc/unit_tests --junitxml=${INSTALL_TEST_DIR}/TEST-OpenVinoConversion.xml - - name: Install Python Layer tests dependencies - run: | - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt - - # layer test requirements - python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt - - - name: MO Python API Tests - if: fromJSON(inputs.affected-components).MO.test - run: | - # Import 'test_utils' installed in '/tests/python/openvino' - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - export PYTHONPATH=${INSTALL_TEST_DIR}/python - - if [[ "${{ runner.os }}" == "Linux" ]] && [[ "${{ runner.arch }}" == "ARM64" ]]; then - # Find gomp lib - GOMP_LIB=$(find "${PIP_INSTALL_PATH}/torch/lib/../../torch.libs/" -name '*libgomp-*so*') - export LD_PRELOAD=${GOMP_LIB} - fi - - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests -n logical --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: OVC Python API Tests if: fromJSON(inputs.affected-components).MO.test run: | @@ -205,16 +170,6 @@ jobs: export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/py_frontend_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_py_fontend.xml - - name: ONNX Layer Tests - if: ${{ fromJSON(inputs.affected-components).ONNX_FE.test }} - run: | - # requires 'unit_tests' from 'tools/mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo:$PYTHONPATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/onnx_tests -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: JAX Layer Tests - JAX FE if: ${{ fromJSON(inputs.affected-components).JAX_FE.test && runner.arch != 'ARM64' && runner.os != 'macOS' }} run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit_jax_fe --junitxml=${INSTALL_TEST_DIR}/TEST-jax_fe.xml @@ -230,22 +185,6 @@ jobs: TEST_DEVICE: CPU TEST_PRECISION: FP16 - - name: TensorFlow 1 Layer Tests - Legacy FE - if: fromJSON(inputs.affected-components).TF_FE.test - run: python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --use_legacy_frontend --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 2 Layer Tests - Legacy FE - # no longer workable since TF 2.17 - # will be removed in 2024.5 - if: ${{ 'false' }} - run: python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/test_tf2_keras_activation.py --use_legacy_frontend --ir_version=11 -k "sigmoid" --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_Activation.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: Clone API snippets if: runner.os != 'macOS' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/merge_queue_stub.yml b/.github/workflows/merge_queue_stub.yml new file mode 100644 index 00000000000000..a3d2e0b456a106 --- /dev/null +++ b/.github/workflows/merge_queue_stub.yml @@ -0,0 +1,13 @@ +on: + merge_group: + +jobs: + merge_group_stub_check: + name: ci/jenkins + runs-on: ubuntu-latest + defaults: + run: + shell: bash + if: ${{ github.event_name == 'merge_group' }} + steps: + - run: echo "Just a stub check to keep Jenkins running in pre-commits but not in merge queue" diff --git a/.github/workflows/mo.yml b/.github/workflows/mo.yml deleted file mode 100644 index f48986d4a0d304..00000000000000 --- a/.github/workflows/mo.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: MO -on: - push: - paths: - - 'tools/mo/**' - - '.github/workflows/mo.yml' - branches: - - 'master' - - 'releases/**' - pull_request: - paths: - - 'tools/mo/**' - - '.github/workflows/mo.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: read-all - -jobs: - Pylint-UT: - runs-on: ubuntu-22.04 - if: ${{ github.repository_owner == 'openvinotoolkit' }} - steps: - - name: Clone OpenVINO - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.10' - - - name: Cache pip - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/mo/requirements*.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - # For UT - pip install unittest-xml-reporting==3.0.2 - # MO requirements - pip install -r requirements_caffe.txt - pip install -r requirements_kaldi.txt - pip install -r requirements_onnx.txt - pip install -r requirements_tf2.txt - pip install -r requirements_dev.txt - working-directory: tools/mo - - - name: Pylint-MO - run: pylint -d C,R,W openvino/tools/mo - working-directory: tools/mo diff --git a/.github/workflows/windows_vs2019_release.yml b/.github/workflows/windows_vs2019_release.yml index a416f577cdb3e1..1b218cdf7d430b 100644 --- a/.github/workflows/windows_vs2019_release.yml +++ b/.github/workflows/windows_vs2019_release.yml @@ -296,9 +296,6 @@ jobs: $ovCoreWheelPath=Get-ChildItem -Path . -Filter openvino-*.whl | % { $_.FullName } python3 -m pip install "$ovCoreWheelPath" - # Find and install the dev OV wheel - $ovDevWheelPath=Get-ChildItem -Path . -Filter openvino_dev*.whl | % { $_.FullName } - python3 -m pip install "$ovDevWheelPath[caffe,kaldi,onnx,tensorflow2,pytorch]" working-directory: ${{ env.INSTALL_WHEELS_DIR }} - name: Install Python API tests dependencies @@ -309,8 +306,11 @@ jobs: # For torchvision to OpenVINO preprocessing converter python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/python/preprocess/torchvision/requirements.txt - # TODO: replace with Python API tests requirements - python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/mo/requirements_dev.txt + # For validation of Python API + python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/bindings/python/requirements_test.txt + + # ONNX tests requirements + python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/requirements_onnx # For getting rid of SSL issues during model downloading for unit tests python3 -m pip install certifi @@ -318,34 +318,17 @@ jobs: - name: Set SSL_CERT_FILE for model downloading for unit tests run: echo SSL_CERT_FILE=$(python3 -m certifi) >> $env:GITHUB_ENV - - name: Python API Tests - #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test # Ticket: 127101 - shell: cmd - run: | - set PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% - python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py - - - name: Model Optimizer UT - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - shell: cmd - run: | - python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml - - name: Install Python Layer tests dependencies run: | # layer test requirements python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - name: ONNX Layer Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test + - name: Python API Tests + #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test # Ticket: 127101 shell: cmd run: | - :: requires 'unit_tests' from 'tools/mo' - set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\mo;%PYTHONPATH% - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/onnx_tests -n logical -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 + set PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% + python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py - name: TensorFlow Lite Layer Tests - TFL FE if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test @@ -366,18 +349,6 @@ jobs: --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-onnx_frontend.xml ^ --ignore=${{ env.INSTALL_TEST_DIR }}/onnx/test_python/test_zoo_models.py - - name: MO Python API Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - shell: cmd - run: | - :: Used for 'test_utils' installed in '\python\openvino\test_utils' - set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\python\openvino\test_utils;${{ env.INSTALL_TEST_DIR }}\python;%PYTHONPATH% - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/mo_python_api_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: OVC Python API Tests if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test shell: cmd diff --git a/CMakeLists.txt b/CMakeLists.txt index e9e8d3724d9ac5..65a72ef8f4936e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -138,7 +138,7 @@ function(ov_developer_package_export_targets) endforeach() endif() else() - message(FATAL_ERROR "Internal error: ${target_name} does not represent a cmake target") + message(FATAL_ERROR "Internal error: '${EXPORT_TARGET}' does not represent a cmake target") endif() list(REMOVE_DUPLICATES _OPENVINO_DEVELOPER_PACKAGE_TARGETS) diff --git a/cmake/developer_package/compile_flags/sdl.cmake b/cmake/developer_package/compile_flags/sdl.cmake index 34ad5904519e7f..35f59cb8970573 100644 --- a/cmake/developer_package/compile_flags/sdl.cmake +++ b/cmake/developer_package/compile_flags/sdl.cmake @@ -55,11 +55,20 @@ if(ENABLE_INTEGRITYCHECK) set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /INTEGRITYCHECK") endif() -set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") -set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") -set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") -set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR (OV_COMPILER_IS_INTEL_LLVM AND WIN32)) + # add sdl required flags to both Debug and Release on Windows + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OV_C_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OV_C_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OV_LINKER_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${OV_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OV_LINKER_FLAGS}") +else() + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") +endif() unset(OV_C_CXX_FLAGS) unset(OV_LINKER_FLAGS) diff --git a/cmake/features.cmake b/cmake/features.cmake index e1201ad3a185ac..f12810adf86075 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -200,6 +200,9 @@ ov_dependent_option (ENABLE_SYSTEM_PROTOBUF "Enables use of system Protobuf" OFF # the option is turned off by default, because we don't want to have a dependency on libsnappy.so ov_dependent_option (ENABLE_SYSTEM_SNAPPY "Enables use of system version of Snappy" OFF "ENABLE_SNAPPY_COMPRESSION" OFF) +# the option is turned off by default, because we are not sure that system version of ZE loader is fresh enough +ov_dependent_option (ENABLE_SYSTEM_LEVEL_ZERO "Enables use of system version of Level Zero" OFF + "ENABLE_INTEL_NPU" OFF) ov_dependent_option(ENABLE_JS "Enables JS API building" ${ENABLE_JS_DEFAULT} "NOT ANDROID;NOT EMSCRIPTEN" OFF) diff --git a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst index 3f3d0064e8a4c6..c80dc388568004 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst @@ -76,14 +76,14 @@ Feature Support and API Coverage | HETERO | 61.22 % | 99.24 % | 86.05 % | +-------------------------+-----------+------------------+-------------------+ | || Percentage of API supported by the device, | -| || as of OpenVINO 2024.4, 25 Oct, 2024. | +| || as of OpenVINO 2024.5, 20 Nov. 2024. | +-------------------------+-----------+------------------+-------------------+ For setting up a relevant configuration, refer to the :doc:`Integrate with Customer Application <../../openvino-workflow/running-inference/integrate-openvino-with-your-application>` topic (step 3 "Configure input and output"). -.. dropdown:: Device support across OpenVINO 2024.4 distributions +.. dropdown:: Device support across OpenVINO 2024.5 distributions =============== ========== ====== =============== ======== ============ ========== ========== ========== Device Archives PyPI APT/YUM/ZYPPER Conda Homebrew vcpkg Conan npm diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index 75c7ba90db7e76..8a58dc27df1f83 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -13,7 +13,7 @@ Performance Benchmarks Efficient LLMs for AI PC Performance Information F.A.Q. OpenVINO Accuracy - Getting Performance Numbers + Getting Performance Numbers This page presents benchmark results for the @@ -64,7 +64,7 @@ implemented in your solutions. Click the buttons below to see the chosen benchma :outline: :expand: - :material-regular:`bar_chart;1.4em` OVMS for GenAI + :material-regular:`bar_chart;1.4em` OVMS for GenAI (coming soon) @@ -132,21 +132,21 @@ For a listing of all platforms and configurations used for testing, refer to the .. grid-item:: - .. button-link:: ../_static/benchmarks_files/OV-2024.4-platform_list.pdf + .. button-link:: ../_static/benchmarks_files/OV-2024.5-platform_list.pdf :color: primary :outline: :expand: :material-regular:`download;1.5em` Click for Hardware Platforms [PDF] - .. button-link:: ../_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx + .. button-link:: ../_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx :color: primary :outline: :expand: :material-regular:`download;1.5em` Click for Configuration Details [XLSX] - .. button-link:: ../_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx + .. button-link:: ../_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx :color: primary :outline: :expand: @@ -160,10 +160,10 @@ For a listing of all platforms and configurations used for testing, refer to the **Disclaimers** * Intel® Distribution of OpenVINO™ toolkit performance results are based on release - 2024.3, as of July 31, 2024. + 2024.5, as of November 20, 2024. * OpenVINO Model Server performance results are based on release - 2024.3, as of Aug. 19, 2024. + 2024.4, as of Sept. 30, 2024. The results may not reflect all publicly available updates. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software, or service diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst index b8256af650e2f8..5697fcbf6e4d74 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -5,9 +5,7 @@ This page is regularly updated to help you identify the best-performing LLMs on Intel® Core™ Ultra processor family and AI PCs. The current data is as of OpenVINO 2024.4, 24 Oct. 2024 -The tables below list the key performance indicators for a selection of Large Language Models, -running on an Intel® Core™ Ultra 7-165H, Intel® Core™ Ultra 7-265V, and Intel® Core™ Ultra -7-288V based system, on built-in GPUs. +The tables below list the key performance indicators for inference on built-in GPUs. @@ -16,14 +14,32 @@ running on an Intel® Core™ Ultra 7-165H, Intel® Core™ Ultra 7-265V, and In +.. tab-set:: -.. csv-table:: - :class: modeldata stripe - :name: supportedModelsTableOv - :header-rows: 1 - :file: ../../_static/benchmarks_files/llm_models.csv + .. tab-item:: 9-288V + + .. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models_9-288V.csv + + .. tab-item:: 7-268V + + .. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models_7-258V.csv + + .. tab-item:: 7-155H + + .. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models_7-155H.csv -| .. grid:: 1 1 2 2 :gutter: 4 diff --git a/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst b/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst index 3162bae7254704..e87733a1445356 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst @@ -1,9 +1,6 @@ Model Accuracy ============== - - -The following two tables present the absolute accuracy drop calculated as the accuracy difference between OV-accuracy and the original framework accuracy for FP32, and the same for INT8, BF16, and FP16 representations of a model on three platform architectures. The third table presents the GenAI model accuracies as absolute accuracy values. Refer to notes below the table for more @@ -11,7 +8,7 @@ information. * A - Intel® Core™ i9-9000K (AVX2), INT8 and FP32 * B - Intel® Xeon® 6338, (VNNI), INT8 and FP32 -* C - Intel® Xeon 8480+ (VNNI, AMX), INT8, BF16, FP32 +* C - Intel® Xeon 8580 (VNNI, AMX), INT8, BF16, FP32 * D - Intel® Flex-170, INT8 and FP16 @@ -28,73 +25,52 @@ information. * - bert-base-cased - SST-2_bert_cased_padded - spearman@cosine - - 3.33% - - 3.22% - - 3.05% - - 2.88% - * - bert-large-uncased-whole-word-masking-squad-0001 - - SQUAD_v1_1_bert_msl384_mql64_ds128_lowercase - - F1 - - 0.12% - - 0.03% - - 0.03% - - 0.28% + - 3.06% + - 2.89% + - 2.71% + - 2.71% * - efficientdet-d0 - COCO2017_detection_91cl - coco_precision - - 0.00% - - -0.52% - - -0.54% - - -0.60% + - -0.84% + - -0.59% + - -0.59% + - -0.55% * - mask_rcnn_resnet50_atrous_coco - COCO2017_detection_91cl_bkgr - coco_orig_precision - - 0.05% - - 0.03% - - 0.08% - - -0.09% + - -0.10% + - -0.04% + - 0.07% + - -0.01% * - mobilenet-v2 - ImageNet2012 - accuracy @ top1 - - - -0.87% - - -0.88% - - -0.88% + - -0.97% + - -0.98% + - -0.95% * - resnet-50 - ImageNet2012 - accuracy @ top1 - - -0.17% - - -0.18% - - -0.18% - - -0.16% + - 0.74% + - 0.76% + - 0.74% + - 0.82% * - ssd-resnet34-1200 - COCO2017_detection_80cl_bkgr - map - - -0.03% - - -0.02% - - -0.03% - - 0.02% + - -0.06% + - -0.08% + - -0.07% + - -0.06% * - ssd-mobilenet-v1-coco - COCO2017_detection_80cl_bkgr - coco-precision - - -2.74% - - -0.11% - - -0.13% - - -0.12% - * - unet-camvid-onnx-0001 - - CamVid_12cl - - mean_iou @ mean - - -6.28% - - 6.45% - - 6.46% - - 6.43% - * - yolo_v5m - - COCO2017_detection_80cl - - map - - -0.40% - - -0.32% - - -0.32% - - -0.31% + - -2.94% + - -0.28% + - -0.28% + - -0.26% * - yolo_v8n - COCO2017_detection_80cl - map @@ -121,30 +97,22 @@ information. - 0.00% - 0.00% - -0.01% - - 0.01% - * - bert-large-uncased-whole-word-masking-squad-0001 - - SQUAD_v1_1_bert_msl384_mql64_ds128_lowercase - - F1 - - 0.04% - - 0.04% - - 0.06% - - 0.06% - - 0.04% + - 0.02% * - efficientdet-d0 - COCO2017_detection_91cl - coco_precision - 0.01% - - -0.02% - 0.01% + - 0.01% + - 0.00% - 0.00% - - -0.02% * - mask_rcnn_resnet50_atrous_coco - COCO2017_detection_91cl_bkgr - coco_orig_precision - -0.01% - -0.01% - -0.01% - - -0.05% + - 0.05% - 0.00% * - mobilenet-v2 - ImageNet2012 @@ -160,40 +128,24 @@ information. - 0.00% - 0.00% - 0.00% - - -0.01% - - -0.01% + - 0.01% + - 0.01% * - ssd-resnet34-1200 - COCO2017_detection_80cl_bkgr - map - 0.02% - - 0.00% - - 0.00% - - -0.02% - - 0.04% + - 0.02% + - 0.02% + - -0.01% + - 0.02% * - ssd-mobilenet-v1-coco - COCO2017_detection_80cl_bkgr - coco-precision - - -0.08% - - 0.01% + - 0.04% - 0.01% + - 0.04% - 0.08% - 0.01% - * - unet-camvid-onnx-0001 - - CamVid_12cl - - mean_iou @ mean - - 0.00% - - 0.00% - - 0.00% - - -0.03% - - -0.03% - * - yolo_v5m - - COCO2017_detection_80cl - - map - - 0.00% - - 0.05% - - 0.05% - - 0.07% - - 0.07% * - yolo_v8n - COCO2017_detection_80cl - map @@ -213,46 +165,60 @@ information. - B, VNNI-INT4 - C, FAMX-FP16 - D, MTL-INT4 - * - chatGLM2-6b + * - chatGLM4 - Wikiset - ppl - - 5.24 - - 6.03 - - 5.24 - - 6.03 - * - Falcon-7b-instruct + - + - + - + - + * - Gemma-2-9B - Wikitext - ppl - - 1.65 - - 1.76 - - 1.65 - - 1.76 + - + - 1.57 + - 1.57 + - * - Llama-2-7b-chat - Wikiset - ppl - - 1.58 - - 1.59 - - 1.91 + - + - - 1.59 + - * - Llama-3-8b - Wikiset - ppl - - 1.54 - - 1.56 + - 1.45 + - 1.48 + - 1.45 + - + * - Llama-3.2-3b-instruct + - Wikiset + - ppl + - 1.60 + - 1.62 - 1.17 - - 1.57 + - * - Mistral-7b - Wikitext - ppl - 1.48 - 1.49 - - 1.39 - - 1.49 + - 1.48 + - * - Phi3-mini-4k-instruct - Wikitext - ppl - 1.52 + - 1.55 + - 1.52 - 1.56 + * - Qwen-2-7B + - Wikitext + - ppl + - 1.52 + - 1.53 - 1.52 - 1.56 diff --git a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst index 4bf0b3a0acb19a..0f70c93e9c8b96 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst @@ -31,10 +31,13 @@ Performance Information F.A.Q. .. dropdown:: How can I run the benchmark results on my own? - All of the performance benchmarks are generated using the + All of the performance benchmarks on traditional network models are generated using the open-source tool within the Intel® Distribution of OpenVINO™ toolkit called :doc:`benchmark_app <../../learn-openvino/openvino-samples/benchmark-tool>`. + For diffusers (Stable-Diffusion) and foundational models (aka LLMs) please use the OpenVINO GenAI + opensource repo `OpenVINO GenAI tools/llm_bench `__ + For a simple instruction on testing performance, see the :doc:`Getting Performance Numbers Guide `. .. dropdown:: Where can I find a more detailed description of the workloads used for benchmarking? @@ -50,23 +53,27 @@ Performance Information F.A.Q. - Public Network - Task - Input Size - * - `chatGLM2-6B `__ + * - `chatGLM4-9B `__ - THUDM - Transformer - - 32K - * - `Falcon-7b-instruct `__ + - 128K + * - `Gemma-2-9B `__ - Hugginface - - Causal Decoder-only - - 2048 + - Text-To-Text Decoder-only + - 8K * - `Llama-2-7b-chat `__ - Meta AI - Auto regressive language - - 4096 + - 4K * - `Llama-3-8b `__ - Meta AI - Auto regressive language - - 8192 - * - `Mistral-7b `__ + - 8K + * - `Llama-3.2-3B `__ + - Meta AI + - Auto regressive language + - 128K + * - `Mistral-7b-V0.1 `__ - Mistral AI - Auto regressive language - 4096 @@ -74,6 +81,10 @@ Performance Information F.A.Q. - Huggingface - Auto regressive language - 4096 + * - `Qwen-2-7B `__ + - Huggingface + - Auto regressive language + - 128K * - `Stable-Diffusion-V1-5 `__ - Hugginface - Latent Diffusion Model @@ -82,10 +93,6 @@ Performance Information F.A.Q. - BERT - question / answer - 128 - * - `bert-large-uncased-whole-word-masking-squad-int8-0001 `__ - - BERT-large - - question / answer - - 384 * - `efficientdet-d0 `__ - Efficientdet - classification @@ -110,14 +117,6 @@ Performance Information F.A.Q. - ssd-resnet34 onnx model - object detection - 1200x1200 - * - `unet-camvid-onnx-0001 `__ - - U-Net - - semantic segmentation - - 368x480 - * - `yolo-v5m `__ - - YOLO V5 Medium - - object detection - - 640x640 * - `yolov8n `__ - Yolov8nano - object detection diff --git a/docs/articles_en/about-openvino/release-notes-openvino.rst b/docs/articles_en/about-openvino/release-notes-openvino.rst index 6685a4325d57fe..343c9e780f05dc 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino.rst @@ -1,3 +1,4 @@ +============================= OpenVINO Release Notes ============================= @@ -15,115 +16,462 @@ OpenVINO Release Notes -2024.4 - 19 September 2024 +2024.5 - 20 November 2024 ############################# :doc:`System Requirements <./release-notes-openvino/system-requirements>` | :doc:`Release policy <./release-notes-openvino/release-policy>` | :doc:`Installation Guides <./../get-started/install-openvino>` + + What's new +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* More Gen AI coverage and framework integrations to minimize code changes. +* More GenAI coverage and framework integrations to minimize code changes. - * Support for GLM-4-9B Chat, MiniCPM-1B, Llama 3 and 3.1, Phi-3-Mini, Phi-3-Medium and - YOLOX-s models. - * Noteworthy notebooks added: Florence-2, NuExtract-tiny Structure Extraction, Flux.1 Image - Generation, PixArt-α: Photorealistic Text-to-Image Synthesis, and Phi-3-Vision Visual - Language Assistant. + * New models supported: Llama 3.2 (1B & 3B), Gemma 2 (2B & 9B), and YOLO11. + * LLM support on NPU: Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, Qwen2-7B-Instruct and Phi-3 + Mini-Instruct. + * Noteworthy notebooks added: Sam2, Llama3.2, Llama3.2 - Vision, Wav2Lip, Whisper, and Llava. + * Preview: support for Flax, a high-performance Python neural network library based on JAX. + Its modular design allows for easy customization and accelerated inference on GPUs. * Broader Large Language Model (LLM) support and more model compression techniques. - * OpenVINO™ runtime optimized for Intel® Xe Matrix Extensions (Intel® XMX) systolic arrays on - built-in GPUs for efficient matrix multiplication resulting in significant LLM performance - boost with improved 1st and 2nd token latency, as well as a smaller memory footprint on - Intel® Core™ Ultra Processors (Series 2). - * Memory sharing enabled for NPUs on Intel® Core™ Ultra Processors (Series 2) for efficient - pipeline integration without memory copy overhead. - * Addition of the PagedAttention feature for discrete GPUs* enables a significant boost in - throughput for parallel inferencing when serving LLMs on Intel® Arc™ Graphics or Intel® - Data Center GPU Flex Series. + * Optimizations for built-in GPUs on Intel® Core™ Ultra Processors (Series 1) and Intel® Arc™ + Graphics include KV Cache compression for memory reduction along with improved usability, + and model load time optimizations to improve first token latency for LLMs. + * Dynamic quantization was enabled to improve first token latency for LLMs on built-in + Intel® GPUs without impacting accuracy on Intel® Core™ Ultra Processors (Series 1). Second + token latency will also improve for large batch inference. + * A new method to generate synthetic text data is implemented in the Neural Network + Compression Framework (NNCF). This will allow LLMs to be compressed more accurately using + data-aware methods without datasets. Coming soon: This feature will soon be accessible via + Optimum Intel on Hugging Face. * More portability and performance to run AI at the edge, in the cloud, or locally. - * Support for Intel® Core Ultra Processors Series 2 (formerly codenamed Lunar Lake) on Windows. - * OpenVINO™ Model Server now comes with production-quality support for OpenAI-compatible API - which enables significantly higher throughput for parallel inferencing on Intel® Xeon® - processors when serving LLMs to many concurrent users. - * Improved performance and memory consumption with prefix caching, KV cache compression, and - other optimizations for serving LLMs using OpenVINO™ Model Server. - * Support for Python 3.12. - * Support for Red Hat Enterprise Linux (RHEL) version 9.3 - 9.4. + * Support for + `Intel® Xeon® 6 Processors with P-cores `__ + (formerly codenamed Granite Rapids) and + `Intel® Core™ Ultra 200V series processors `__ + (formerly codenamed Arrow Lake-S). + * Preview: GenAI API enables multimodal AI deployment with support for multimodal pipelines + for improved contextual awareness, transcription pipelines for easy audio-to-text + conversions, and image generation pipelines for streamlined text-to-visual conversions. + * Speculative decoding feature added to the GenAI API for improved performance and efficient + text generation using a small draft model that is periodically corrected by the full-size + model. + * Preview: LoRA adapters are now supported in the GenAI API for developers to quickly and + efficiently customize image and text generation models for specialized tasks. + * The GenAI API now also supports LLMs on NPU allowing developers to specify NPU as the + target device, specifically for WhisperPipeline (for whisper-base, whisper-medium, and + whisper-small) and LLMPipeline (for Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, + Qwen2-7B-Instruct and Phi-3 Mini-instruct). Use driver version 32.0.100.3104 or later for + best performance. Now deprecated +----------------------------------------------------------------------------------------------- + +* Python 3.8 is no longer supported: + + +OpenVINO™ Runtime +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* The following will not be available beyond the 2024.4 OpenVINO version: +Common +----------------------------------------------------------------------------------------------- - * The macOS x86_64 debug bins - * Python 3.8 - * Discrete Keem Bay support +* Numpy 2.x has been adopted for all currently supported components, including NNCF. +* A new constant constructor has been added, enabling constants to be created from data pointer + as shared memory. Additionally, it can take ownership of a shared, or other, object, avoiding + a two-step process to wrap memory into ``ov::Tensor``. +* Files are now read via the async ReadFile API, reducing the bottleneck for LLM model load + times on GPU. +* CPU implementation of SliceScatter operator is now available, used for models such as Gemma, + supporting increased LLM performance. -* Intel® Streaming SIMD Extensions (Intel® SSE) will be supported in source code form, but not - enabled in the binary package by default, starting with OpenVINO 2025.0. -| Check the `deprecation section <#deprecation-and-support>`__ for more information. +CPU Device Plugin +----------------------------------------------------------------------------------------------- +* Gold support of the Intel® Xeon® 6 platform with P-cores (formerly code name Granite Rapids) + has been reached. +* Support of Intel® Core™ Ultra 200V series processors (formerly codenamed Arrow Lake-S) has + been implemented. +* LLM performance has been further improved with Rotary Position Embedding optimization; Query, + Key, and Value; and multi-layer perceptron fusion optimization. +* FP16 support has been extended with SDPA and PagedAttention, improving performance of LLM via + both native APIs and the vLLM integration. +* Models with LoRA adapters are now supported. -Common +GPU Device Plugin +----------------------------------------------------------------------------------------------- + +* The KV cache INT8 compression mechanism is now available for all supported GPUs. It enables a + significant reduction in memory consumption, increasing performance with a minimal impact to + accuracy (it affects systolic devices slightly more than non-systolic ones). The feature is + activated by default for non-systolic devices. +* LoRA adapters are now functionally supported on GPU. +* A new feature of GPU weightless blob caching enables caching model structure only and reusing + the weights from the original model file. Use the new OPTIMIZE_SIZE property to activate. +* Dynamic quantization with INT4 and INT8 precisions has been implemented and enabled by + default on Intel® Core™ Ultra platforms, improving LLM first token latency. + + +NPU Device Plugin +----------------------------------------------------------------------------------------------- + +* Models retrieved from the OpenVINO cache have a smaller memory footprint now. The plugin + releases the cached model (blob) after weights are loaded in NPU regions. Model export is not + available in this scenario. Memory consumption is reduced during inference execution with one + blob size. This optimization requires the latest NPU driver: 32.0.100.3104. +* A driver bug for ``ov::intel_npu::device_total_mem_size`` has been fixed. The plugin will now + report 2GB as the maximum allocatable memory for any driver that does not support graph + extension 1.8. Even if older drivers report a larger amount of memory to be available, memory + allocation would fail when 2GB are exceeded. Plugin reports the number that driver exposes + for any driver that supports graph extension 1.8 (or newer). +* A new API is used to initialize the model (available in graph extension 1.8). +* Inference request set_tensors is now supported. +* ``ov::device::LUID`` is now exposed on Windows. +* LLM-related improvements have been implemented in terms of both memory usage and performance. +* AvgPool and MaxPool operator support has been extended, adding support for more PyTorch models. + +* NOTE: for systems based on Intel® Core™ Ultra Processors Series 2, more than 16GB of RAM may + be required to use larger models, such as Llama-2-7B, Mistral-0.2-7B, and Qwen-2-7B + (exceeding 4B parameters) with prompt sizes over 1024 tokens. + + +OpenVINO Python API +----------------------------------------------------------------------------------------------- + +* Constant now can be created from openvino.Tensor. +* The “release_memory” method has been added for a compiled model, improving control over + memory consumption. + + + +OpenVINO Node.js API +----------------------------------------------------------------------------------------------- + +* Querying the best device to perform inference of a model with specific operations + is now available in JavaScript API. +* Contribution guidelines have been improved to make it easier for developers to contribute. +* Testing scope has been extended by inference in end-to-end tests. +* JavaScript API samples have been improved for readability and ease of running. + + + +TensorFlow Framework Support +----------------------------------------------------------------------------------------------- + +* TensorFlow 2.18.0, Keras 3.6.0, NumPy 2.0.2 in Python 3.12, and NumPy 1.26.4 in other Python + versions have been added to validation. +* Out-of-the-box conversion with static ranks has been improved by devising a new shape for + Switch-Merge condition sub-graphs. +* Complex type for the following operations is now supported: ExpandDims, Pack, Prod, Rsqrt, + ScatterNd, Sub. +* The following issues have been fixed: + + * the corner case with one element in LinSpace to avoid division by zero, + * support FP16 and FP64 input types for LeakyRelu, + * support non-i32/i64 output index type for ArgMin/Max operations. + + + +PyTorch Framework Support +----------------------------------------------------------------------------------------------- + +* PyTorch version 2.5 is now supported. +* OpenVINO Model Converter (OVC) now supports TorchScript and ExportedProgram saved on a drive. +* The issue of aten.index.Tensor conversion for indices with “None” values has been fixed, + helping to support the HF Stable Diffusion model in ExportedProgram format. + + + +ONNX Framework Support +----------------------------------------------------------------------------------------------- + +* ONNX version 1.17.0 is now used. +* Customers' models with DequantizeLinear-21, com.microsoft.MatMulNBits, and + com.microsoft.QuickGelu operations are now supported. + +JAX/Flax Framework Support +----------------------------------------------------------------------------------------------- + +* JAX 0.4.35 and Flax 0.10.0 has been added to validation. +* jax._src.core.ClosedJaxpr object conversion is now supported. +* Vision Transformer from google-research/vision_transformer is now supported + (with support for 37 new operations). + + +OpenVINO Model Server +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* Encryption and decryption of topology in model cache is now supported with callback functions - provided by the user (CPU only for now; ov::cache_encryption_callbacks). -* The Ubuntu20 and Ubuntu22 Docker images now include the tokenizers and GenAI CPP modules, - including pre-installed Python modules, in development versions of these images. -* Python 3.12 is now supported. +* The OpenAI API text embedding endpoint has been added, enabling OVMS to be used as a building + block for AI applications like RAG. + `(read more) `__ +* The rerank endpoint has been added based on Cohere API, enabling easy similarity detection + between a query and a set of documents. It is one of the building blocks for AI applications + like RAG and makes integration with frameworks such as langchain easy. + `(read more) `__ +* The following improvements have been done to LLM text generation: -CPU Device Plugin + * The ``echo`` sampling parameter together with ``logprobs`` in the ``completions`` endpoint + is now supported. + * Performance has been increased on both CPU and GPU. + * Throughput in high-concurrency scenarios has been increased with dynamic_split_fuse for GPU. + * Testing coverage and stability has been improved. + * The procedure for service deployment and model repository preparation has been simplified. + +* An experimental version of a Windows binary package - native model server for Windows OS - is + available. This release includes a set of limitations and has limited tests coverage. It is + intended for testing, while the production-ready release is expected with 2025.0. All feedback + is welcome. + + +Neural Network Compression Framework +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* The following is now supported: +* A new nncf.data.generate_text_data() method has been added for generating a synthetic dataset + for LLM compression. This approach helps to compress LLMs more accurately in situations when + the dataset is not available or not sufficient. + `See our example `__ + for more information about the usage. +* Support of data-free and data-aware weight compression methods - nncf.compress_weights() - + has been extended with NF4 per-channel quantization, making compressed LLMs more accurate and + faster on NPU. +* Caching of computed statistics in nncf.compress_weights() is now available, significantly + reducing compression time when performing compression of the same LLM multiple times, with + different compression parameters. To enable it, set the advanced ``statistics_path`` parameter + of nncf.compress_weights() to the desired file path location. +* The ``backup_mode`` optional parameter has been added to nncf.compress_weights(), for + specifying the data type for embeddings, convolutions, and last linear layers during 4-bit + weight compression. Available options are INT8_ASYM (default), INT8_SYM, and NONE (retains + the original floating-point precision of the model weights). In certain situations, + non-default value might give better accuracy of compressed LLMs. +* Preview support is now available for optimizing models in Torch + `FX format `__, nncf.quantize(), and + nncf.compress_weights() methods. After optimization such models can be directly executed + via torch.compile(compressed_model, backend="openvino"). For more details, see + `INT8 quantization example `__. +* Memory consumption of data-aware weight compression methods - nncf.compress_weights() – has + been reduced significantly, with some variation depending on the model and method. +* Support for the following has changed: + + * NumPy 2 added + * PyTorch upgraded to 2.5.1 + * ONNX upgraded to 1.17 + * Python 3.8 discontinued - * Tensor parallel feature for multi-socket CPU inference, with performance improvement for - LLMs with 6B+ parameters (enabled through model_distribution_policy hint configurations). - * RMSNorm operator, optimized with JIT kernel to improve both the 1st and 2nd token - performance of LLMs. -* The following has been improved: - * vLLM support, with PagedAttention exposing attention score as the second output. It can now - be used in the cache eviction algorithm to improve LLM serving performance. - * 1st token performance with Llama series of models, with additional CPU operator optimization - (such as MLP, SDPA) on BF16 precision. - * Default oneTBB version on Linux is now 2021.13.0, improving overall performance on latest - Intel XEON platforms. - * MXFP4 weight compression models (compressing weights to 4-bit with the e2m1 data type - without a zero point and with 8-bit e8m0 scales) have been optimized for Xeon platforms - thanks to fullyconnected compressed weight LLM support. +OpenVINO Tokenizers ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* The following has been fixed: +* Several operations have been introduced and optimized. +* Conversion parameters and environment info have been added to ``rt_info``, improving + reproducibility and debugging. - * Memory leak when ov::num_streams value is 0. - * CPU affinity mask is changed after OpenVINO execution when OpenVINO is compiled - with -DTHREADING=SEQ. -GPU Device Plugin +OpenVINO.GenAI +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* Dynamic quantization for LLMs is now supported on discrete GPU platforms. -* Stable Diffusion 3 is now supported with good accuracy on Intel GPU platforms. -* Both first and second token latency for LLMs have been improved on Intel GPU platforms. -* The issue of model cache not regenerating with the value changes of - ``ov::hint::performance_mode`` or ``ov::hint::dynamic_quantization_group_size`` has been - fixed. +* The following has been added: + * LoRA adapter for the LLMPipeline. + * Text2ImagePipeline with LoRA adapter and text2image samples. + * VLMPipeline and visual_language_chat sample for text generation models with text and image + inputs. + * WhisperPipeline and whisper_speech_recognition sample. -NPU Device Plugin +* speculative_decoding_lm has been moved to LLMPipeline based implementation and is now + installed as part of the package. +* On NPU, a set of pipelines has been enabled: WhisperPipeline (for whisper-base, + whisper-medium, and whisper-small), LLMPipeline (for Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, + Qwen2-7B-Instruct, and Phi-3 Mini-instruct). Use driver version 32.0.100.3104 or later for + best performance. + + + + + +Other Changes and Known Issues +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +Jupyter Notebooks +----------------------------- + +* `Text-to-Image generation using OpenVINO GenAI `__ +* `Multi LoRA Image Generation `__ +* `Virtual Try-on using OpenVINO and CatVTON `__ +* `Visual Language Assistant using OpenVINO GenAI `__ +* `Speech recognition using OpenVINO GenAI `__ +* `YoloV11 `__ +* `Llama-3.2-vision `__ +* `Pixtral `__ +* `Segment Anything 2 `__ +* `Video Lips-sync using Wav2Lip `__ +* `Convert JAX to OpenVINO tutorial `__ + + +Known Issues +----------------------------- + +| **Component: CPU Plugin** +| ID: 155898 +| Description: +| Description: When using new version of Transformer version to convert some of LLMs + (GPT-J/GPT-NeoX or falcon-7b), the inference accuracy may be impacted on 4th or 5th + generation of Intel® Xeon® processors, due to model structure update triggering inference + precision difference in part of the model. The workaround is to use transformer version of + 4.44.2 or lower. + +| **Component: GPU Plugin** +| ID: 154583 +| Description: +| LLM accuracy can be low especially on non-systolic platforms like Intel® Core™ Ultra. When + facing the low accuracy issue, user needs to manually set a config ACTIVATION_SCALING_FACOTR + with a value of 8.0 in the compile_model() function. From the next release, scaling factor + value will be automatically applied through updated IR. + +| **Component: GenAI** +| ID: 156437, 148933 +| Description: +| When using Python GenAI APIs, if ONNX 17.0 and later is installed, it may encounter the + error “DLL load failed while importing onnx_cpp2py_export: A dynamic link library (DLL) + initialization routine failed.” It is due to the ONNX dependency issue + `onnx/onnx#6267 `__, + Install + `Microsoft Visual C++ Redistributable `__ + latest supported downloads to fix the issue. + +| **Component: GenAI** +| ID: 156944 +| Description: +| There were backward incompatible changes resulting in different text generated by LLMs like + Mistralai/Mistral-7B-Instruct-v0.2 and TinyLlama/TinyLlama-1.1B-Chat-v1.0 when using a + tokenizer converted by older openvino_tolenizers. A way to resolve the issue is to convert + tokenizer and detokenizer models using the latest openvino_tokenizers. + + + + + + + + +Previous 2024 releases ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +.. dropdown:: 2024.4 - 19 September 2024 + :animate: fade-in-slide-down + :color: secondary + + **What's new** + + * More Gen AI coverage and framework integrations to minimize code changes. + + * Support for GLM-4-9B Chat, MiniCPM-1B, Llama 3 and 3.1, Phi-3-Mini, Phi-3-Medium and + YOLOX-s models. + * Noteworthy notebooks added: Florence-2, NuExtract-tiny Structure Extraction, Flux.1 Image + Generation, PixArt-α: Photorealistic Text-to-Image Synthesis, and Phi-3-Vision Visual + Language Assistant. + + * Broader Large Language Model (LLM) support and more model compression techniques. + + * OpenVINO™ runtime optimized for Intel® Xe Matrix Extensions (Intel® XMX) systolic arrays on + built-in GPUs for efficient matrix multiplication resulting in significant LLM performance + boost with improved 1st and 2nd token latency, as well as a smaller memory footprint on + Intel® Core™ Ultra Processors (Series 2). + * Memory sharing enabled for NPUs on Intel® Core™ Ultra Processors (Series 2) for efficient + pipeline integration without memory copy overhead. + * Addition of the PagedAttention feature for discrete GPUs* enables a significant boost in + throughput for parallel inferencing when serving LLMs on Intel® Arc™ Graphics or Intel® + Data Center GPU Flex Series. + + * More portability and performance to run AI at the edge, in the cloud, or locally. + + * Support for Intel® Core™ Ultra Processors Series 2 (formerly codenamed Lunar Lake) on Windows. + * OpenVINO™ Model Server now comes with production-quality support for OpenAI-compatible API + which enables significantly higher throughput for parallel inferencing on Intel® Xeon® + processors when serving LLMs to many concurrent users. + * Improved performance and memory consumption with prefix caching, KV cache compression, and + other optimizations for serving LLMs using OpenVINO™ Model Server. + * Support for Python 3.12. + * Support for Red Hat Enterprise Linux (RHEL) version 9.3 - 9.4. + + *Now deprecated* + + * The following will not be available beyond the 2024.4 OpenVINO version: + + * The macOS x86_64 debug bins + * Python 3.8 + * Discrete Keem Bay support + + * Intel® Streaming SIMD Extensions (Intel® SSE) will be supported in source code form, but not + enabled in the binary package by default, starting with OpenVINO 2025.0. + + Check the `deprecation section <#deprecation-and-support>`__ for more information. + + **OpenVINO™ Runtime** + + *Common* + + * Encryption and decryption of topology in model cache is now supported with callback functions + provided by the user (CPU only for now; ov::cache_encryption_callbacks). + * The Ubuntu20 and Ubuntu22 Docker images now include the tokenizers and GenAI CPP modules, + including pre-installed Python modules, in development versions of these images. + * Python 3.12 is now supported. + + *CPU Device Plugin* + + * The following is now supported: + + * Tensor parallel feature for multi-socket CPU inference, with performance improvement for + LLMs with 6B+ parameters (enabled through model_distribution_policy hint configurations). + * RMSNorm operator, optimized with JIT kernel to improve both the 1st and 2nd token + performance of LLMs. + + * The following has been improved: + + * vLLM support, with PagedAttention exposing attention score as the second output. It can now + be used in the cache eviction algorithm to improve LLM serving performance. + * 1st token performance with Llama series of models, with additional CPU operator optimization + (such as MLP, SDPA) on BF16 precision. + * Default oneTBB version on Linux is now 2021.13.0, improving overall performance on latest + Intel® Xeon® platforms. + * MXFP4 weight compression models (compressing weights to 4-bit with the e2m1 data type + without a zero point and with 8-bit e8m0 scales) have been optimized for Intel® Xeon® + platforms thanks to fullyconnected compressed weight LLM support. + + * The following has been fixed: + + * Memory leak when ov::num_streams value is 0. + * CPU affinity mask is changed after OpenVINO execution when OpenVINO is compiled + with -DTHREADING=SEQ. + + + *GPU Device Plugin* + + * Dynamic quantization for LLMs is now supported on discrete GPU platforms. + * Stable Diffusion 3 is now supported with good accuracy on Intel GPU platforms. + * Both first and second token latency for LLMs have been improved on Intel GPU platforms. + * The issue of model cache not regenerating with the value changes of + ``ov::hint::performance_mode`` or ``ov::hint::dynamic_quantization_group_size`` has been + fixed. + + + *NPU Device Plugin* + * `Remote Tensor API `__ is now supported. * You can now query the available number of tiles (ov::intel_npu::max_tiles) and force a @@ -140,193 +488,178 @@ NPU Device Plugin only during the export method. -OpenVINO Python API -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *OpenVINO Python API* -* Openvino.Tensor, when created in the shared memory mode, now prevents “garbage collection” of - numpy memory. -* The ``openvino.experimental`` submodule is now available, providing access to experimental - functionalities under development. -* New python-exclusive openvino.Model constructors have been added. -* Image padding in PreProcessor is now available. -* OpenVINO Runtime is now compatible with numpy 2.0. + * Openvino.Tensor, when created in the shared memory mode, now prevents “garbage collection” of + numpy memory. + * The ``openvino.experimental`` submodule is now available, providing access to experimental + functionalities under development. + * New python-exclusive openvino.Model constructors have been added. + * Image padding in PreProcessor is now available. + * OpenVINO Runtime is now compatible with numpy 2.0. -OpenVINO Node.js API -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *OpenVINO Node.js API* -* The following has been improved + * The following has been improved - * Unit tests for increased efficiency and stability - * Security updates applied to dependencies + * Unit tests for increased efficiency and stability + * Security updates applied to dependencies -* `Electron `__ - compatibility is now confirmed with new end-to-end tests. -* `New API methods `__ added. + * `Electron `__ + compatibility is now confirmed with new end-to-end tests. + * `New API methods `__ added. -TensorFlow Framework Support -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *TensorFlow Framework Support* -* TensorFlow 2.17.0 is now supported. -* JAX 0.4.31 is now supported via a path of jax2tf with native_serialization=False -* `8 NEW* operations `__ - have been added. -* Tensor lists with multiple undefined dimensions in element_shape are now supported, enabling - support for TF Hub lite0-detection/versions/1 model. + * TensorFlow 2.17.0 is now supported. + * JAX 0.4.31 is now supported via a path of jax2tf with native_serialization=False + * `8 NEW* operations `__ + have been added. + * Tensor lists with multiple undefined dimensions in element_shape are now supported, enabling + support for TF Hub lite0-detection/versions/1 model. -PyTorch Framework Support -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *PyTorch Framework Support* -* Torch 2.4 is now supported. -* Inplace ops are now supported automatically if the regular version is supported. -* Symmetric GPTQ model from Hugging Face will now be automatically converted to the signed type - (INT4) and zero-points will be removed. + * Torch 2.4 is now supported. + * Inplace ops are now supported automatically if the regular version is supported. + * Symmetric GPTQ model from Hugging Face will now be automatically converted to the signed type + (INT4) and zero-points will be removed. -ONNX Framework Support -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *ONNX Framework Support* -* ONNX 1.16.0 is now supported -* models with constants/inputs of uint4/int4 types are now supported. -* 4 NEW operations have been added. + * ONNX 1.16.0 is now supported + * models with constants/inputs of uINT4/INT4 types are now supported. + * 4 NEW operations have been added. -OpenVINO Model Server -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **OpenVINO Model Server** -* OpenAI API for text generation is now officially supported and recommended for production - usage. It comes with the following new features: + * OpenAI API for text generation is now officially supported and recommended for production + usage. It comes with the following new features: - * Prefix caching feature, caching the prompt evaluation to speed up text generation. - * Ability to compress the KV Cache to a lower precision, reducing memory consumption without - a significant loss of accuracy. - * ``stop`` sampling parameters, to define a sequence that stops text generation. - * ``logprobs`` sampling parameter, returning the probabilities to returned tokens. - * Generic metrics related to execution of the MediaPipe graph that can be used for autoscaling - based on the current load and the level of concurrency. - * `Demo of text generation horizontal scalability `__ - using basic docker containers and Kubernetes. - * Automatic cancelling of text generation for disconnected clients. - * Non-UTF-8 responses from the model can be now automatically changed to Unicode replacement - characters, due to their configurable handling. - * Intel GPU with paged attention is now supported. - * Support for Llama3.1 models. + * Prefix caching feature, caching the prompt evaluation to speed up text generation. + * Ability to compress the KV Cache to a lower precision, reducing memory consumption without + a significant loss of accuracy. + * ``stop`` sampling parameters, to define a sequence that stops text generation. + * ``logprobs`` sampling parameter, returning the probabilities to returned tokens. + * Generic metrics related to execution of the MediaPipe graph that can be used for autoscaling + based on the current load and the level of concurrency. + * `Demo of text generation horizontal scalability `__ + using basic docker containers and Kubernetes. + * Automatic cancelling of text generation for disconnected clients. + * Non-UTF-8 responses from the model can be now automatically changed to Unicode replacement + characters, due to their configurable handling. + * Intel GPU with paged attention is now supported. + * Support for Llama3.1 models. -* The following has been improved: + * The following has been improved: - * Handling of model templates without bos_token is now fixed. - * Performance of the multinomial sampling algorithm. - * ``finish_reason`` in the response correctly determines reaching max_tokens (length) and - completing the sequence (stop). - * Security and stability. + * Handling of model templates without bos_token is now fixed. + * Performance of the multinomial sampling algorithm. + * ``finish_reason`` in the response correctly determines reaching max_tokens (length) and + completing the sequence (stop). + * Security and stability. -Neural Network Compression Framework -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **Neural Network Compression Framework** -* The LoRA Correction algorithm is now included in the Weight Compression method, improving the - accuracy of INT4-compressed models on top of other data-aware algorithms, such as AWQ and - Scale Estimation. To enable it, set the lora_correction option to True in - nncf.compress_weights(). -* The GPTQ compression algorithm can now be combined with the Scale Estimation algorithm, - making it possible to run GPTQ, AWQ, and Scale Estimation together, for the optimum-accuracy - INT4-compressed models. -* INT8 quantization of LSTMSequence and Convolution operations for constant inputs is now - enabled, resulting in better performance and reduced model size. + * The LoRA Correction algorithm is now included in the Weight Compression method, improving the + accuracy of INT4-compressed models on top of other data-aware algorithms, such as AWQ and + Scale Estimation. To enable it, set the lora_correction option to True in + nncf.compress_weights(). + * The GPTQ compression algorithm can now be combined with the Scale Estimation algorithm, + making it possible to run GPTQ, AWQ, and Scale Estimation together, for the optimum-accuracy + INT4-compressed models. + * INT8 quantization of LSTMSequence and Convolution operations for constant inputs is now + enabled, resulting in better performance and reduced model size. -OpenVINO Tokenizers -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **OpenVINO Tokenizers** -* Split and BPE tokenization operations have been reimplemented, resulting in improved - tokenization accuracy and performance. -* New building options are now available, offering up to a 12x reduction in binary size. -* An operation is now available to validate and skip/replace model-generated non-Unicode - bytecode sequences during detokenization. + * Split and BPE tokenization operations have been reimplemented, resulting in improved + tokenization accuracy and performance. + * New building options are now available, offering up to a 12x reduction in binary size. + * An operation is now available to validate and skip/replace model-generated non-Unicode + bytecode sequences during detokenization. -OpenVINO.GenAI -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **OpenVINO.GenAI** -* New samples and pipelines are now available: + * New samples and pipelines are now available: - * An example IterableStreamer implementation in - `multinomial_causal_lm/python sample `__ + * An example IterableStreamer implementation in + `multinomial_causal_lm/python sample `__ -* GenAI compilation is now available as part of OpenVINO via the –DOPENVINO_EXTRA_MODULES CMake - option. + * GenAI compilation is now available as part of OpenVINO via the –DOPENVINO_EXTRA_MODULES CMake + option. -Other Changes and Known Issues -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **Other Changes and Known Issues** -Jupyter Notebooks ------------------------------ + *Jupyter Notebooks* -* `Florence-2 `__ -* `NuExtract: Structure Extraction `__ -* `Flux.1 Image Generation `__ -* `PixArt-α: Photorealistic Text-to-Image Synthesis `__ -* `Phi-3-Vision Visual Language Assistant `__ -* `MiniCPMV2.6 `__ -* `InternVL2 `__ -* The list of supported models in - `LLM chatbot `__ - now includes Phi3.5, Gemma2 support + * `Florence-2 `__ + * `NuExtract: Structure Extraction `__ + * `Flux.1 Image Generation `__ + * `PixArt-α: Photorealistic Text-to-Image Synthesis `__ + * `Phi-3-Vision Visual Language Assistant `__ + * `MiniCPMV2.6 `__ + * `InternVL2 `__ + * The list of supported models in + `LLM chatbot `__ + now includes Phi3.5, Gemma2 support -Known Issues ------------------------------ + *Known Issues* -| **Component: CPU** -| ID: CVS-150542, CVS-145996 -| Description: -| The upgrade of default oneTBB on Linux platforms to 2021.13.0 improves overall - performance on latest Intel XEON platform but causes regression in some cases. Limit the - threads usage of postprocessing done by Torch can mitigate the regression (For example: - torch.set_num_threads(n), n can be 1, beam search number, prompt batch size or other - numbers). - -| **Component: OpenVINO.Genai** -| ID: 149694 -| Description: -| Passing openvino.Tensor instance to LLMPipleine triggers incompatible arguments error if - OpenVINO and GenAI are installed from PyPI on Windows. + | **Component: CPU** + | ID: CVS-150542, CVS-145996 + | Description: + | The upgrade of default oneTBB on Linux platforms to 2021.13.0 improves overall + performance on latest Intel® Xeon® platform but causes regression in some cases. Limit the + threads usage of postprocessing done by Torch can mitigate the regression (For example: + torch.set_num_threads(n), n can be 1, beam search number, prompt batch size or other + numbers). + + | **Component: OpenVINO.Genai** + | ID: 149694 + | Description: + | Passing openvino.Tensor instance to LLMPipleine triggers incompatible arguments error if + OpenVINO and GenAI are installed from PyPI on Windows. -| **Component: OpenVINO.Genai** -| ID: 148308 -| Description: -| OpenVINO.GenAI archive doesn't have debug libraries for OpenVINO Tokenizers and - OpenVINO.GenAI. + | **Component: OpenVINO.Genai** + | ID: 148308 + | Description: + | OpenVINO.GenAI archive doesn't have debug libraries for OpenVINO Tokenizers and + OpenVINO.GenAI. + + | **Component: ONNX for ARM** + | ID: n/a + | Description: + | For ARM binaries, the `1.16 ONNX library `__ + is not yet available. The ONNX library for ARM, version 1.15, does not include the latest + functional and security updates. Users should update to the latest version as it becomes + available. + | Currently, if an unverified AI model is supplied to the ONNX frontend, it could lead to a + directory traversal issue. Ensure that the file name and file path that a model contains + are verified and correct. To learn more about the vulnerability, see: + `CVE-2024-27318 `__ and + `CVE-2024-27319 `__. + + | **Component: Kaldi** + | ID: n/a + | Description: + | There is a known issue with the Kaldi DL framework support on the Python version 3.12 due + to the numpy version incompatibilities. As Kaldi support in OpenVINO is currently deprecated + and will be discontinued with version 2025.0, the issue will not be addressed. -| **Component: ONNX for ARM** -| ID: n/a -| Description: -| For ARM binaries, the `1.16 ONNX library `__ - is not yet available. The ONNX library for ARM, version 1.15, does not include the latest - functional and security updates. Users should update to the latest version as it becomes - available. -| Currently, if an unverified AI model is supplied to the ONNX frontend, it could lead to a - directory traversal issue. Ensure that the file name and file path that a model contains - are verified and correct. To learn more about the vulnerability, see: - `CVE-2024-27318 `__ and - `CVE-2024-27319 `__. - -| **Component: Kaldi** -| ID: n/a -| Description: -| There is a known issue with the Kaldi DL framework support on the Python version 3.12 due - to the numpy version incompatibilities. As Kaldi support in OpenVINO is currently deprecated - and will be discontinued with version 2025.0, the issue will not be addressed. -Previous 2024 releases -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. dropdown:: 2024.3 - 31 July 2024 :animate: fade-in-slide-down @@ -614,7 +947,7 @@ Previous 2024 releases * Preview: addition of the :doc:`Generate API <../learn-openvino/llm_inference_guide/genai-guide>`, a simplified API for text generation using large language models with only a few lines of code. The API is available through the newly launched OpenVINO GenAI package. - * Support for Intel Atom® Processor X Series. For more details, see :doc:`System Requirements <./release-notes-openvino/system-requirements>`. + * Support for Intel® Atom® Processor X Series. For more details, see :doc:`System Requirements <./release-notes-openvino/system-requirements>`. * Preview: Support for Intel® Xeon® 6 processor. **OpenVINO™ Runtime** @@ -638,8 +971,8 @@ Previous 2024 releases *CPU Device Plugin* * Performance when using latency mode in FP32 precision has been improved on Intel client - platforms, including Core Ultra (codename Meteor Lake) and 13th Gen Core processors - (codename Raptor Lake). + platforms, including Intel® Core™ Ultra (formerly codenamed Meteor Lake) and 13th Gen Core + processors (formerly codenamed Raptor Lake). * 2nd token latency and memory footprint for FP16 LLMs have been improved significantly on AVX2 and AVX512 based CPU platforms, particularly for small batch sizes. * PagedAttention has been optimized on AVX2, AVX512 and AMX platforms together with INT8 KV cache @@ -653,9 +986,9 @@ Previous 2024 releases * Both first token and average token latency of LLMs is improved on all GPU platforms, most significantly on discrete GPUs. Memory usage of LLMs has been reduced as well. - * Stable Diffusion FP16 performance improved on Core Ultra platforms, with significant pipeline - improvement for models with dynamic-shaped input. Memory usage of the pipeline has been reduced, - as well. + * Stable Diffusion FP16 performance improved on Intel® Core™ Ultra platforms, with significant + pipeline improvement for models with dynamic-shaped input. Memory usage of the pipeline + has been reduced, as well. * Optimized permute_f_y kernel performance has been improved. *NPU Device Plugin* @@ -710,7 +1043,7 @@ Previous 2024 releases * OpenVINO Model server can be now used for text generation use cases using OpenAI compatible API. * Added support for continuous batching and PagedAttention algorithms for text generation with - fast and efficient in high concurrency load especially on Intel Xeon processors. + fast and efficient in high concurrency load especially on Intel® Xeon® processors. `Learn more about it `__. **Neural Network Compression Framework** @@ -753,8 +1086,9 @@ Previous 2024 releases | Description: | In 2024.2, oneTBB 2021.2.x is used for Intel Distribution of OpenVINO Ubuntu and Red Hat archives, instead of system TBB/oneTBB. This improves performance on the new generation of - Xeon platforms but may increase latency of some models on the previous generation. You can - build OpenVINO with **-DSYSTEM_TBB=ON** to get better latency performance for these models. + Intel® Xeon® platforms but may increase latency of some models on the previous generation. + You can build OpenVINO with **-DSYSTEM_TBB=ON** to get better latency performance for + these models. | **Component: python API** | ID: CVS-141744 @@ -1069,8 +1403,8 @@ Previous 2024 releases * More portability and performance to run AI at the edge, in the cloud, or locally. * A preview plugin architecture of the integrated Neural Processor Unit (NPU) as part of - Intel® Core™ Ultra processor (codename Meteor Lake) is now included in the main OpenVINO™ - package on PyPI. + Intel® Core™ Ultra processor (formerly codenamed Meteor Lake) is now included in the + main OpenVINO™ package on PyPI. * Improved performance on ARM by enabling the ARM threading library. In addition, we now support multi-core ARM processors and enabled FP16 precision by default on MacOS. * New and improved LLM serving samples from OpenVINO Model Server for multi-batch inputs and @@ -1263,10 +1597,11 @@ Previous 2024 releases | **Component: CPU runtime** | *ID:* N/A | *Description:* - | Performance results (first token latency) may vary from those offered by the previous OpenVINO version, for - “latency” hint inference of LLMs with long prompts on Xeon platforms with 2 or more - sockets. The reason is that all CPU cores of just the single socket running the application - are employed, lowering the memory overhead for LLMs when numa control is not used. + | Performance results (first token latency) may vary from those offered by the previous + OpenVINO version, for “latency” hint inference of LLMs with long prompts on Intel® Xeon® + platforms with 2 or more sockets. The reason is that all CPU cores of just the single + socket running the application are employed, lowering the memory overhead for LLMs when + numa control is not used. | *Workaround:* | The behavior is expected but stream and thread configuration may be used to include cores from all sockets. @@ -1315,22 +1650,26 @@ Discontinued in 2024 for applying NNCF optimization on top of models from Hugging Face. * Support for Apache MXNet, Caffe, and Kaldi model formats. Conversion to ONNX may be used as a solution. + * The macOS x86_64 debug bins are no longer provided with the OpenVINO toolkit, starting + with OpenVINO 2024.5. + * Python 3.8 is no longer supported, starting with OpenVINO 2024.5. -Deprecated and to be removed in the future --------------------------------------------- + * As MxNet doesn't support Python version higher than 3.8, according to the + `MxNet PyPI project `__, + it is no longer supported by OpenVINO, either. -* The macOS x86_64 debug bins will no longer be provided with the OpenVINO toolkit, starting - with OpenVINO 2024.5. -* Python 3.8 is now considered deprecated, and it will not be available beyond the 2024.4 - OpenVINO version. + * Discrete Keem Bay support is no longer supported, starting with OpenVINO 2024.5. + * Support for discrete devices (formerly codenamed Raptor Lake) is no longer available for + NPU. - * As MxNet doesn't support Python version higher than 3.8, according to the - `MxNet PyPI project `__, - it will no longer be supported in future versions, either. -* Discrete Keem Bay support is now considered deprecated and will be fully removed with OpenVINO 2024.5 +Deprecated and to be removed in the future +-------------------------------------------- + * Intel® Streaming SIMD Extensions (Intel® SSE) will be supported in source code form, but not - enabled in the binary package by default, starting with OpenVINO 2025.0 + enabled in the binary package by default, starting with OpenVINO 2025.0. +* Ubuntu 20.04 support will be deprecated in future OpenVINO releases due to the end of + standard support. * The openvino-nightly PyPI module will soon be discontinued. End-users should proceed with the Simple PyPI nightly repo instead. More information in `Release Policy `__. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst index 164033bdd2831c..abad632e5ae86c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst @@ -26,13 +26,12 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce * **Type**: ``int`` * **Required**: *yes* -* *activations* +* *direction* - * **Description**: *activations* specifies activation functions for gates, there are three gates, so three activation functions should be specified as a value for this attributes - * **Range of values**: any combination of *relu*, *sigmoid*, *tanh* - * **Type**: a list of strings - * **Default value**: *sigmoid,tanh,tanh* - * **Required**: *no* + * **Description**: Specify if the RNN is forward, reverse, or bidirectional. If it is one of *forward* or *reverse* then ``num_directions = 1``, if it is *bidirectional*, then ``num_directions = 2``. This ``num_directions`` value specifies input/output shape requirements. + * **Range of values**: *forward*, *reverse*, *bidirectional* + * **Type**: ``string`` + * **Required**: *yes* * *activations_alpha, activations_beta* @@ -42,6 +41,14 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce * **Default value**: None * **Required**: *no* +* *activations* + + * **Description**: *activations* specifies activation functions for gates, there are three gates, so three activation functions should be specified as a value for this attributes + * **Range of values**: any combination of *relu*, *sigmoid*, *tanh* + * **Type**: a list of strings + * **Default value**: *sigmoid,tanh,tanh* + * **Required**: *no* + * *clip* * **Description**: *clip* specifies bound values *[-C, C]* for tensor clipping. Clipping is performed before activations. @@ -50,12 +57,6 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce * **Default value**: *infinity* that means that the clipping is not applied * **Required**: *no* -* *direction* - - * **Description**: Specify if the RNN is forward, reverse, or bidirectional. If it is one of *forward* or *reverse* then ``num_directions = 1``, if it is *bidirectional*, then ``num_directions = 2``. This ``num_directions`` value specifies input/output shape requirements. - * **Range of values**: *forward*, *reverse*, *bidirectional* - * **Type**: ``string`` - * **Required**: *yes* **Inputs** diff --git a/docs/articles_en/get-started/configurations/genai-dependencies.rst b/docs/articles_en/get-started/configurations/genai-dependencies.rst index e347c11953fbc2..59d29ef3108da0 100644 --- a/docs/articles_en/get-started/configurations/genai-dependencies.rst +++ b/docs/articles_en/get-started/configurations/genai-dependencies.rst @@ -4,8 +4,8 @@ OpenVINO™ GenAI Dependencies OpenVINO™ GenAI depends on both `OpenVINO `__ and `OpenVINO Tokenizers `__. During OpenVINO™ GenAI installation from PyPi, the same versions of OpenVINO and OpenVINO Tokenizers -are used (e.g. ``openvino==2024.4.0`` and ``openvino-tokenizers==2024.4.0.0`` are installed for -``openvino-genai==2024.4.0``). +are used (e.g. ``openvino==2024.5.0`` and ``openvino-tokenizers==2024.5.0.0`` are installed for +``openvino-genai==2024.5.0``). Trying to update any of the dependency packages might result in a version incompatiblibty due to different Application Binary Interfaces (ABIs), which will result in errors while running diff --git a/docs/articles_en/get-started/install-openvino.rst b/docs/articles_en/get-started/install-openvino.rst index 7f26ab9ec72c9f..be00804faa01d2 100644 --- a/docs/articles_en/get-started/install-openvino.rst +++ b/docs/articles_en/get-started/install-openvino.rst @@ -1,4 +1,4 @@ -Install OpenVINO™ 2024.4 +Install OpenVINO™ 2024.5 ========================== @@ -19,16 +19,16 @@ Install OpenVINO™ 2024.4 .. raw:: html - + - + -OpenVINO 2024.4, described here, is not a Long-Term-Support version! +OpenVINO 2024.5, described here, is not a Long-Term-Support version! All currently supported versions are: -* 2024.4 (development) +* 2024.5 (development) * 2023.3 (LTS) -* 2022.3 (LTS) + .. dropdown:: Effortless GenAI integration with OpenVINO GenAI Flavor diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst index e777c06253a37a..20965f2f22d095 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst @@ -58,7 +58,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -73,9 +73,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu24_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu24_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu24_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu24_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 @@ -83,9 +83,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu22_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu22_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu22_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu22_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: Ubuntu 20.04 :sync: ubuntu-20 @@ -93,9 +93,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: RHEL 8 :sync: rhel-8 @@ -103,18 +103,18 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_rhel8_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_rhel8_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_rhel8_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_rhel8_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: CentOS 7 :sync: centos-7 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_centos7_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_centos7_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_centos7_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_centos7_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: ARM 64-bit @@ -122,25 +122,25 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_arm64.tgz -O openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_arm64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_arm64.tgz -O openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_arm64 /opt/intel/openvino_2024.5.0 .. tab-item:: ARM 32-bit :sync: arm-32 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_debian10_2024.4.0.16579.c3152d32c9c_armhf.tgz -O openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_debian10_2024.4.0.16579.c3152d32c9c_armhf /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_debian10_2024.5.0.17288.7975fa5da0c_armhf.tgz -O openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_debian10_2024.5.0.17288.7975fa5da0c_armhf /opt/intel/openvino_2024.5.0 5. Install required system dependencies on Linux. To do this, OpenVINO provides a script in the extracted installation directory. Run the following command: .. code-block:: sh - cd /opt/intel/openvino_2024.4.0 + cd /opt/intel/openvino_2024.5.0 sudo -E ./install_dependencies/install_openvino_dependencies.sh 6. (Optional) Install *numpy* Python Library: @@ -149,11 +149,11 @@ Step 1: Download and Install the OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.4.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.5.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.4.0 + cd /opt/intel/openvino_2024.5.0 python3 -m pip install -r ./python/requirements.txt 7. For simplicity, it is useful to create a symbolic link as below: @@ -162,7 +162,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /opt/intel - sudo ln -s openvino_2024.4.0 openvino_2024 + sudo ln -s openvino_2024.5.0 openvino_2024 .. note:: If you have already installed a previous release of OpenVINO 2024, a symbolic link to the ``openvino_2024`` folder may already exist. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst index e9157a99e1c882..e4bff378106122 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst @@ -47,7 +47,7 @@ Step 1: Install OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -57,9 +57,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/macos/m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/macos/m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: ARM, 64-bit :sync: arm-64 @@ -67,9 +67,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/macos/m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_arm64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_arm64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/macos/m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_arm64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_arm64 /opt/intel/openvino_2024.5.0 5. (Optional) Install *numpy* Python Library: @@ -78,11 +78,11 @@ Step 1: Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.4.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.5.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.4.0 + cd /opt/intel/openvino_2024.5.0 python3 -m pip install -r ./python/requirements.txt 6. For simplicity, it is useful to create a symbolic link as below: @@ -90,7 +90,7 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - sudo ln -s /opt/intel/openvino_2024.4.0 /opt/intel/openvino_2024 + sudo ln -s /opt/intel/openvino_2024.5.0 /opt/intel/openvino_2024 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst index 8f3efeeb720dc9..9db280ec81472e 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst @@ -41,18 +41,18 @@ Step 1: Download and Install OpenVINO Core Components ``C:\Program Files (x86)\Intel`` is the recommended folder. You may also use a different path if desired or if you don't have administrator privileges on your computer. -2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. +2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. If you prefer using command-lines, run the following commands in the command prompt window you opened: .. code-block:: sh cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/windows/w_openvino_toolkit_windows_2024.4.0.16579.c3152d32c9c_x86_64.zip --output openvino_2024.4.0.zip + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/windows/w_openvino_toolkit_windows_2024.5.0.17288.7975fa5da0c_x86_64.zip --output openvino_2024.5.0.zip .. note:: - A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.4.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. + A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.5.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. 3. Use your favorite tool to extract the archive file, rename the extracted folder, and move it to the ``C:\Program Files (x86)\Intel`` directory. @@ -61,9 +61,9 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh - tar -xf openvino_2024.4.0.zip - ren w_openvino_toolkit_windows_2024.4.0.16579.c3152d32c9c_x86_64 openvino_2024.4.0 - move openvino_2024.4.0 "C:\Program Files (x86)\Intel" + tar -xf openvino_2024.5.0.zip + ren w_openvino_toolkit_windows_2024.5.0.17288.7975fa5da0c_x86_64 openvino_2024.5.0 + move openvino_2024.5.0 "C:\Program Files (x86)\Intel" 4. (Optional) Install *numpy* Python Library: @@ -72,11 +72,11 @@ Step 1: Download and Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.4.0\python`` folder: + You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.5.0\python`` folder: .. code-block:: sh - cd "C:\Program Files (x86)\Intel\openvino_2024.4.0" + cd "C:\Program Files (x86)\Intel\openvino_2024.5.0" python -m pip install -r .\python\requirements.txt @@ -85,7 +85,7 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh cd C:\Program Files (x86)\Intel - mklink /D openvino_2024 openvino_2024.4.0 + mklink /D openvino_2024 openvino_2024.5.0 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst index 34f04669e50d63..a10b0d0c7bbce4 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst @@ -47,24 +47,24 @@ Linux .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu24_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu24_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu22_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu22_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: Ubuntu 20.04 :sync: ubuntu-20 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu20_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu20_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: ARM 64-bit @@ -72,8 +72,8 @@ Linux .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu20_2024.4.0.0_arm64.tar.gz -O openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu20_2024.5.0.0_arm64.tar.gz -O openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz Windows @@ -82,7 +82,7 @@ Windows .. code-block:: sh cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/windows/openvino_genai_windows_2024.4.0.0_x86_64.zip --output openvino_genai_2024.4.0.0.zip + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/windows/openvino_genai_windows_2024.5.0.0_x86_64.zip --output openvino_genai_2024.5.0.0.zip macOS ++++++++++++++++++++++++++ @@ -94,16 +94,16 @@ macOS .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/macos/openvino_genai_macos_12_6_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/macos/openvino_genai_macos_12_6_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: ARM, 64-bit :sync: arm-64 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/macos/openvino_genai_macos_12_6_2024.4.0.0_arm64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/macos/openvino_genai_macos_12_6_2024.5.0.0_arm64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz Here are the full guides: diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst index 6917d809c7e5d6..5a641300a68edb 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst @@ -20,6 +20,10 @@ Install required dependencies: pip install nncf==2.12 onnx==1.16.1 optimum-intel==1.19.0 pip install --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly +NOTE that for systems based on Intel® Core Ultra Processors Series 2 and 16 GB of RAM, +prompts longer then 1024 characters will not work with a model of 7B or more parameters, +such as Llama-2-7B, Mistral-0.2-7B, and Qwen-2-7B. + Export an LLM model via Hugging Face Optimum-Intel ################################################## diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst index 953784c03fdef0..6033bd8ed96106 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst @@ -249,23 +249,21 @@ and use audio files in WAV format at a sampling rate of 16 kHz as input. def infer(model_dir: str, wav_file_path: str): - raw_speech = read_wav(wav_file_path) - pipe = openvino_genai.WhisperPipeline(model_dir) - - def streamer(word: str) -> bool: - print(word, end="") - return False + device = "CPU" # GPU or NPU can be used as well. + pipe = openvino_genai.WhisperPipeline(model_dir, device) + # The pipeline expects normalized audio with a sampling rate of 16kHz. + raw_speech = read_wav(wav_file_path) result = pipe.generate( raw_speech, max_new_tokens=100, language="<|en|>", task="transcribe", return_timestamps=True, - streamer=streamer, ) - print() + print(result) + for chunk in result.chunks: print(f"timestamps: [{chunk.start_ts}, {chunk.end_ts}] text: {chunk.text}") @@ -288,26 +286,21 @@ and use audio files in WAV format at a sampling rate of 16 kHz as input. std::filesystem::path models_path = argv[1]; std::string wav_file_path = argv[2]; - std::string device = "CPU"; // GPU can be used as well + std::string device = "CPU"; // GPU or NPU can be used as well. ov::genai::WhisperPipeline pipeline(models_path, device); - ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); - ov::genai::WhisperGenerationConfig config(models_path / "generation_config.json"); config.max_new_tokens = 100; config.language = "<|en|>"; config.task = "transcribe"; config.return_timestamps = true; - auto streamer = [](std::string word) { - std::cout << word; - return false; - }; - - auto result = pipeline.generate(raw_speech, config, streamer); + // The pipeline expects normalized audio with a sampling rate of 16kHz. + ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); + auto result = pipeline.generate(raw_speech, config); - std::cout << "\n"; + std::cout << result << "\n"; for (auto& chunk : *result.chunks) { std::cout << "timestamps: [" << chunk.start_ts << ", " << chunk.end_ts << "] text: " << chunk.text << "\n"; diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/quantizing-models-post-training/basic-quantization-flow.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/quantizing-models-post-training/basic-quantization-flow.rst index b4f31daedfa3e4..62c10e52266ec9 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/quantizing-models-post-training/basic-quantization-flow.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/quantizing-models-post-training/basic-quantization-flow.rst @@ -63,6 +63,13 @@ The transformation function is a function that takes a sample from the dataset a :language: python :fragment: [dataset] + .. tab-item:: TorchFX + :sync: torch_fx + + .. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_torch_fx.py + :language: python + :fragment: [dataset] + If there is no framework dataset object, you can create your own entity that implements the ``Iterable`` interface in Python, for example the list of images, and returns data samples feasible for inference. In this case, a transformation function is not required. @@ -102,6 +109,12 @@ See the `example section <#examples-of-how-to-apply-nncf-post-training-quantizat :language: python :fragment: [quantization] + .. tab-item:: TorchFX + :sync: torch_fx + + .. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_torch_fx.py + :language: python + :fragment: [quantization] After that the model can be converted into the OpenVINO Intermediate Representation (IR) if needed, compiled and run with OpenVINO. If you have not already installed OpenVINO developer tools, install it with ``pip install openvino``. @@ -136,6 +149,17 @@ If you have not already installed OpenVINO developer tools, install it with ``pi :language: python :fragment: [inference] +TorchFX models can utilize OpenVINO optimizations using `torch.compile(..., backend="openvino") `__ functionality: + +.. tab-set:: + + .. tab-item:: TorchFX + :sync: torch_fx + + .. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_torch_fx.py + :language: python + :fragment: [inference] + Tune quantization parameters ############################ diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst index bbc09ccd4b5fbb..046dde9661c3bb 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst @@ -6,38 +6,36 @@ LLM Weight Compression :hidden: weight-compression/microscaling-quantization + weight-compression/4-bit-weight-quantization -Weight compression is a technique for enhancing the efficiency of models, -especially those with large memory requirements. This method reduces the model's -memory footprint, a crucial factor for Large Language Models (LLMs). +Weight compression enhances the efficiency of models by reducing their memory footprint, +a crucial factor for Large Language Models (LLMs). It is especially effective for networks with high memory requirements. -Unlike full model quantization, where weights and activations are quantized, -weight compression in `Neural Network Compression Framework (NNCF) `__ -only targets the model's weights. This approach allows the activations to remain as -floating-point numbers, preserving most of the model's accuracy while improving its -speed and reducing its size. +Unlike full model quantization, where both weights and activations are quantized, it +only targets weights, keeping activations as floating-point numbers. This means preserving most +of the model's accuracy while improving its +speed and reducing its size. The reduction in size is especially noticeable with larger models. +For instance the 7 billion parameter Llama 2 model can be reduced +from about 25GB to 4GB using 4-bit weight compression. -The reduction in size is especially noticeable with larger models, -for instance the 7 billion parameter Llama 2 model can be reduced -from about 25GB to 4GB using 4-bit weight compression. With smaller models (i.e. less -than 1B parameters), weight compression may result in more accuracy reduction than -with larger models. +.. note:: + + With smaller language models (i.e. less than 1B parameters), weight + compression may result in more accuracy reduction than with larger models. + Therefore, weight compression is recommended for use with LLMs only. -LLMs and other models that require +LLMs and other GenAI models that require extensive memory to store the weights during inference can benefit from weight compression as it: * enables inference of exceptionally large models that cannot be accommodated in the device memory; - * reduces storage and memory overhead, making models more lightweight and less resource intensive for deployment; - * improves inference speed by reducing the latency of memory access when computing the operations with weights, for example, Linear layers. The weights are smaller and thus faster to load from memory; - * unlike quantization, does not require sample data to calibrate the range of activation values. @@ -46,197 +44,228 @@ provides weight quantization to 8 and 4-bit integer data types as a compression method primarily designed to optimize LLMs. +Compression Methods (8-bit vs. 4-bit) +##################################### + +For models that come from `Hugging Face `__ and are supported +by Optimum, it is recommended to use the **Optimum Intel API**, which employs NNCF weight +compression capabilities to optimize various large Transformer models. + +The NNCF ``nncf.compress_weights()`` API, with most of its options, is exposed in the +``.from_pretrained()`` method of Optimum Intel classes. Optimum also has several datasets +for data-aware quantization available out-of-the-box. -Compress Model Weights -###################### +You can use the examples below to perform data-free 8-bit or 4-bit weight quantization. +Before you start, make sure Optimum Intel is installed in your environment +by running the following command: -**8-bit weight quantization** method offers a balance between model size reduction and -maintaining accuracy, which usually leads to significant performance improvements for -Transformer-based models. Models with 8-bit compressed weights are performant on the -vast majority of supported CPU and GPU platforms. By default, weights are compressed -asymmetrically to "INT8_ASYM" mode. +.. code-block:: python + pip install optimum[openvino] -The code snippet below shows how to do asymmetrical 8-bit quantization of the model weights -represented in OpenVINO IR using NNCF: +**8-bit weight quantization** offers a good balance between reducing the size and lowering the +accuracy of a model. It usually results in significant improvements for transformer-based models +and guarantees good model performance for a vast majority of supported CPU and GPU platforms. +By default, weights are compressed asymmetrically to "INT8_ASYM" mode. .. tab-set:: - .. tab-item:: OpenVINO - :sync: openvino + .. tab-item:: Compression with Optimum-Intel + :sync: optimum - .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py - :language: python - :fragment: [compression_8bit] + Load a pre-trained Hugging Face model, compress it to INT8_ASYM, using the + Optimum Intel API, and then execute inference with a text phrase: + Simply use the optimum-cli command line tool: -Now, the model is ready for compilation and inference. -It can be also saved into a compressed format, resulting in a smaller binary file. + .. code-block:: console -**4-bit weight quantization** method stands for an INT4-INT8 mixed-precision weight quantization, -where INT4 is considered as the primary precision and asymmetric INT8 is the backup one. -It usually results in a smaller model size and lower inference latency, although the accuracy -degradation could be higher, depending on the model. + optimum-cli export openvino --model microsoft/Phi-3.5-mini-instruct --weight-format int8 ov_phi-3.5-mini-instruct -The code snippet below shows how to do 4-bit quantization of the model weights represented -in OpenVINO IR using NNCF: + You can also use the code sample to the same effect: -.. tab-set:: + .. code-block:: python - .. tab-item:: OpenVINO - :sync: openvino + from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig + from transformers import AutoTokenizer, pipeline - .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py - :language: python - :fragment: [compression_4bit] + # Load and compress a model from Hugging Face. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained( + model_id, + export=True, + quantization_config=OVWeightQuantizationConfig(bits=8) + ) + # Inference + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) -The table below summarizes the benefits and trade-offs for each compression type in terms of -memory reduction, speed gain, and accuracy loss. + For more details, refer to the article on how to + :doc:`infer LLMs using Optimum Intel <../../learn-openvino/llm_inference_guide/llm-inference-hf>`. -.. list-table:: - :widths: 25 20 20 20 - :header-rows: 1 + .. tab-item:: Compression with NNCF + :sync: nncf - * - - - Memory Reduction - - Latency Improvement - - Accuracy Loss - * - INT8 Asymmetric - - Low - - Medium - - Low - * - INT4 Symmetric - - High - - High - - High - * - INT4 Asymmetric - - High - - Medium - - Medium + Load a pre-trained Hugging Face model, using the Optimum Intel API, + compress it to INT8_ASYM, using NNCF, and then execute inference with a text phrase: + .. code-block:: python + from nncf import compress_weights, CompressWeightsMode + from optimum.intel.openvino import OVModelForCausalLM + from transformers import AutoTokenizer, pipeline -The INT4 method has several parameters that can provide different performance-accuracy -trade-offs after optimization: + # Load a model and compress it with NNCF. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained(model_id, export=True, load_in_8bit=False, compile=False) + model.model = compress_weights(model.model, mode=CompressWeightsMode.INT8_ASYM) -* ``mode`` - there are two optimization modes: symmetric and asymmetric. + # Inference + model.compile() + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) - **Symmetric Compression** - ``INT4_SYM`` - INT4 Symmetric mode involves quantizing weights to a signed 4-bit integer - symmetrically without zero point. This mode is faster than the INT8_ASYM, making - it ideal for situations where **speed and size reduction are prioritized over accuracy**. +Here is an example of code using NNCF to perform asymmetrical 8-bit weight quantization of +a model in the OpenVINO IR format: - .. code-block:: python +.. tab-set:: - from nncf import compress_weights - from nncf import CompressWeightsMode + .. tab-item:: OpenVINO + :sync: openvino - compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_SYM) + .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py + :language: python + :fragment: [compression_8bit] - **Asymmetric Compression** - ``INT4_ASYM`` - INT4 Asymmetric mode also uses an unsigned 4-bit integer but quantizes weights - asymmetrically with a non-fixed zero point. This mode slightly compromises speed in - favor of better accuracy compared to the symmetric mode. This mode is useful when - **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. +**4-bit weight quantization** is actually a mixed-precision compression, +primarily INT4 and a backup asymmetric INT8 precisions. It produces a smaller model, +offering lower inference latency but potentially noticeable accuracy degradation, +depending on the model. - .. code-block:: python +.. tab-set:: - from nncf import compress_weights - from nncf import CompressWeightsMode + .. tab-item:: Compression with Optimum-Intel + :sync: optimum - compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM) + Load a pre-trained Hugging Face model, compress it to INT4, using the + Optimum Intel API, and then execute inference with a text phrase: -* ``group_size`` controls the size of the group of weights that share the same - quantization parameters. Shared quantization parameters help to speed up the - calculation of activation values as they are dequantized and quantized between - layers. However, they can reduce accuracy. The following group sizes are - recommended: ``128``, ``64``, ``32`` (``128`` is default value). + Simply use the optimum-cli command line tool: - `Smaller Group Size`: Leads to a more accurate model but increases the model's - footprint and reduces inference speed. + .. code-block:: console - `Larger Group Size`: Results in faster inference and a smaller model, but might - compromise accuracy. + optimum-cli export openvino --model microsoft/Phi-3.5-mini-instruct --weight-format int4 --awq --scale-estimation --dataset wikitext2 --group-size 64 --ratio 1.0 ov_phi-3.5-mini-instruct -* ``ratio`` controls the ratio between the layers compressed to the precision defined - by ``mode`` and the rest of the layers that will be kept in the ``backup_mode`` in the optimized model. - Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be - compressed to the precision defined by ``mode``, while the rest will be compressed to - ``backup_mode`` precision. The default value for ratio is 1. + You can also use the code sample to the same effect: - `Higher Ratio (more layers set to mode precision)`: Reduces the model size and increase inference speed but - might lead to higher accuracy degradation. + .. code-block:: python - `Lower Ratio (more layers set to backup_mode precision)`: Maintains better accuracy but results in a larger model size - and potentially slower inference. + from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig + from transformers import AutoTokenizer, pipeline - In this example, 90% of the model's layers are quantized to INT4 asymmetrically with - a group size of 64: + # Load and compress a model from Hugging Face. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained( + model_id, + export=True, + quantization_config=OVWeightQuantizationConfig( + bits=4, + quant_method="awq", + scale_estimation=True, + dataset="wikitext2", + group_size=64, + ratio=1.0 + ) + ) - .. code-block:: python + # Inference + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) - from nncf import compress_weights, CompressWeightsMode + .. tab-item:: Compression with NNCF + :sync: nncf - # Example: Compressing weights with INT4_ASYM mode, group size of 64, and 90% INT4 ratio - compressed_model = compress_weights( - model, - mode=CompressWeightsMode.INT4_ASYM, - group_size=64, - ratio=0.9, - ) + Load a pre-trained Hugging Face model, using the Optimum Intel API, + compress it to INT4 using NNCF, and then execute inference with a text phrase: -* ``scale_estimation`` - boolean parameter that enables more accurate estimation of - quantization scales. Especially helpful when the weights of all layers are quantized to - 4 bits. Requires dataset. + .. code-block:: python -* ``awq`` - boolean parameter that enables the AWQ method for more accurate INT4 weight - quantization. Especially helpful when the weights of all the layers are quantized to - 4 bits. The method can sometimes result in reduced accuracy when used with - Dynamic Quantization of activations. Requires dataset. + from nncf import compress_weights, CompressWeightsMode + from optimum.intel.openvino import OVModelForCausalLM + from transformers import AutoTokenizer, pipeline -* ``gptq`` - boolean parameter that enables the GPTQ method for more accurate INT4 weight - quantization. Requires dataset. + # Load a model and compress it with NNCF. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained(model_id, export=True, load_in_8bit=False, compile=False) + model.model = compress_weights(model.model, mode=CompressWeightsMode.INT4_SYM) -* ``dataset`` - calibration dataset for data-aware weight compression. It is required - for some compression options, for example, ``scale_estimation``, ``gptq`` or ``awq``. Some types - of ``sensitivity_metric`` can use data for precision selection. + # Inference + model.compile() + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) -* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing - layers in the bit-width selection algorithm. Some of the metrics require dataset to be - provided. The following types are supported: - * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - data-free metric computed as - the inverted 8-bit quantization noise. Weights with highest value of this metric can - be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in - 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more - accurate than per-channel, accuracy should not degrade. + For more details, refer to the article on how to + :doc:`infer LLMs using Optimum Intel <../../../learn-openvino/llm_inference_guide/llm-inference-hf>`. - * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires dataset. The average - Hessian trace of weights with respect to the layer-wise quantization error multiplied - by L2 norm of 8-bit quantization noise. +The code snippet below shows how to do 4-bit quantization of the model weights represented +in OpenVINO IR using NNCF: - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires dataset. The mean - variance of the layers' inputs multiplied by inverted 8-bit quantization noise. +.. tab-set:: - * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires dataset. The maximum - variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + .. tab-item:: OpenVINO + :sync: openvino - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires dataset. The mean - magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. + .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py + :language: python + :fragment: [compression_4bit] + +Refer to the article about +:doc:`4-bit weight quantization <./weight-compression/4-bit-weight-quantization>` +for more details. -* ``all_layers`` - boolean parameter that enables INT4 weight quantization of all - Fully-Connected and Embedding layers, including the first and last layers in the model. +Once the model has been optimized, it is ready for compilation and inference. The model can +also be :ref:`saved into a compressed format `, resulting in a +smaller binary file. + +The table below summarizes the benefits and trade-offs for each compression type in terms of +memory reduction, speed gain, and accuracy loss. -* ``lora_correction`` - boolean parameter that enables the LoRA Correction Algorithm - to further improve the accuracy of INT4 compressed models on top of other - algorithms - AWQ and Scale Estimation. +.. list-table:: + :widths: 25 20 20 20 + :header-rows: 1 -* ``backup_mode`` - defines a backup precision for mixed-precision weight compression. - There are three modes: INT8_ASYM, INT8_SYM, and NONE, which retains - the original floating-point precision of the model weights (``INT8_ASYM`` is default value). + * - + - Memory Reduction + - Latency Improvement + - Accuracy Loss + * - INT8 Asymmetric + - Low + - Medium + - Low + * - INT4 Symmetric + - High + - High + - High + * - INT4 Asymmetric + - High + - Medium + - Medium **Use synthetic data for LLM weight compression** @@ -268,8 +297,8 @@ for details of the usage. # Synthetic-based compression synthetic_dataset = nncf.data.generate_text_data(hf_model, tokenizer, dataset_size=100) quantization_dataset = nncf.Dataset( - synthetic_dataset, - transform_fn # see example in NNCF repo how to make transform_fn + synthetic_dataset, + transform_fn # See the example in NNCF repo to learn how to make transform_fn. ) model = compress_weights( @@ -280,58 +309,16 @@ for details of the usage. dataset=quantization_dataset, awq=True, scale_estimation=True - ) # model is openvino.Model + ) # The model is openvino.Model. For data-aware weight compression refer to the following `example `__. .. note:: - Some methods can be stacked on top of one another to achieve a better - accuracy-performance trade-off after weight quantization. For example, the **Scale Estimation** - method can be applied along with **AWQ** and mixed-precision quantization (the ``ratio`` parameter). - - -**Hugging Face Optimum-Intel API** - -Hugging Face Optimum-Intel provides an easy way to use NNCF Weight Compression capabilities to optimize -various large Transformer models. Most of the options of the NNCF ``nncf.compress_weights()`` API are -exposed in the ``.from_pretrained()`` method of Optimum-Intel classes. Optimum also has several datasets -for data-aware quantization available out-of-the-box. -The example below shows data-free 4-bit weight quantization -applied on top of OpenVINO IR. Before trying the example, make sure Optimum Intel -is installed in your environment by running the following command: - -.. code-block:: python - - pip install optimum[openvino] - -.. code-block:: python - - from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig - from transformers import AutoTokenizer, pipeline - - # Load and compress model from Hugging Face - model_id = "microsoft/Phi-3.5-mini-instruct" - model = OVModelForCausalLM.from_pretrained( - model_id, - export=True, - quantization_config=OVWeightQuantizationConfig( - bits=4, - quant_method="awq", - scale_estimation=True, - dataset="wikitext2", - group_size=64, - ratio=1.0 - ) - ) - - # Inference - tokenizer = AutoTokenizer.from_pretrained(model_id) - pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) - phrase = "The weather is" - results = pipe(phrase) - print(results) + Some methods can be stacked on top of one another to achieve a better + accuracy-performance trade-off after weight quantization. For example, the **Scale Estimation** + method can be applied along with **AWQ** and mixed-precision quantization (the ``ratio`` parameter). Exporting and Loading Compressed Models @@ -344,179 +331,157 @@ so it is preferable to compress the model once, save it, and then load the compressed model later for faster time to first inference. .. code-block:: python + :name: save_pretrained - # Save compressed model for faster loading later - model.save_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") - tokenizer.save_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") - - # Load a saved model - model = OVModelForCausalLM.from_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") - tokenizer = AutoTokenizer.from_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") - -GPTQ Models -############ + # Save compressed model for faster loading later + model.save_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") + tokenizer.save_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") -OpenVINO also supports 4-bit models from Hugging Face -`Transformers `__ library optimized -with `GPTQ `__. In this case, there is no -need for an additional model optimization step because model conversion will -automatically preserve the INT4 optimization results, allowing model inference to benefit from it. + # Load a saved model + model = OVModelForCausalLM.from_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") + tokenizer = AutoTokenizer.from_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") -A compression example using a GPTQ model is shown below. -Make sure to install GPTQ dependencies by running the following command: +.. tip:: -.. code-block:: python - - pip install optimum[openvino] auto-gptq - -.. code-block:: python + Models optimized with with NNCF or Optimum Intel can be used with + :doc:`OpenVINO GenAI <../../learn-openvino/llm_inference_guide/genai-guide>`. - from optimum.intel.openvino import OVModelForCausalLM - from transformers import AutoTokenizer, pipeline - # Load model from Hugging Face already optimized with GPTQ - model_id = "TheBloke/Llama-2-7B-Chat-GPTQ" - model = OVModelForCausalLM.from_pretrained(model_id, export=True) +Auto-tuning of Weight Compression Parameters +############################################ - # Inference - tokenizer = AutoTokenizer.from_pretrained(model_id) - pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) - phrase = "The weather is" - results = pipe(phrase) - print(results) +To find the optimal weight compression parameters for a particular model, refer to the +`example `__ , +where weight compression parameters are being searched from the subset of values. +To speed up the search, a self-designed validation pipeline called +`WhoWhatBench `__ +is used. The pipeline can quickly evaluate the changes in the accuracy of the optimized +model compared to the baseline. -An `example of a model `__ -that has been optimized using GPTQ. Compression Metrics Examples -######################################## +############################ -The table below shows examples of text-generation Language Models with different +Below you will find examples of text-generation Language Models with different optimization settings in a data-free setup, where no dataset is used at the optimization step. The Perplexity metric is a measurement of response accuracy, where a higher complexity score indicates a lower accuracy. It is measured on the `Lambada OpenAI dataset `__. -.. list-table:: - :widths: 40 55 25 25 - :header-rows: 1 - - * - Model - - Optimization - - Perplexity\* - - Model Size (Gb) - * - databricks/dolly-v2-3b - - FP32 - - 5.01 - - 10.3 - * - databricks/dolly-v2-3b - - INT8_ASYM - - 5.07 - - 2.6 - * - databricks/dolly-v2-3b - - INT4_ASYM,group_size=32,ratio=0.5 - - 5.28 - - 2.2 - * - facebook/opt-6.7b - - FP32 - - 4.25 - - 24.8 - * - facebook/opt-6.7b - - INT8_ASYM - - 4.27 - - 6.2 - * - facebook/opt-6.7b - - INT4_ASYM,group_size=64,ratio=0.8 - - 4.32 - - 4.1 - * - meta-llama/Llama-2-7b-chat-hf - - FP32 - - 3.28 - - 25.1 - * - meta-llama/Llama-2-7b-chat-hf - - INT8_ASYM - - 3.29 - - 6.3 - * - meta-llama/Llama-2-7b-chat-hf - - INT4_ASYM,group_size=128,ratio=0.8 - - 3.41 - - 4.0 - * - togethercomputer/RedPajama-INCITE-7B-Instruct - - FP32 - - 4.15 - - 25.6 - * - togethercomputer/RedPajama-INCITE-7B-Instruct - - INT8_ASYM - - 4.17 - - 6.4 - * - togethercomputer/RedPajama-INCITE-7B-Instruct - - INT4_ASYM,group_size=128,ratio=1.0 - - 4.17 - - 3.6 - * - meta-llama/Llama-2-13b-chat-hf - - FP32 - - 2.92 - - 48.5 - * - meta-llama/Llama-2-13b-chat-hf - - INT8_ASYM - - 2.91 - - 12.1 - * - meta-llama/Llama-2-13b-chat-hf - - INT4_SYM,group_size=64,ratio=0.8 - - 2.98 - - 8.0 - - -The following table shows accuracy metric in a data-aware 4-bit weight quantization -setup measured on the `Wikitext dataset `__. - -.. list-table:: - :widths: 40 55 25 25 - :header-rows: 1 - - * - Model - - Optimization - - Word perplexity\* - - Model Size (Gb) - * - meta-llama/llama-7b-chat-hf - - FP32 - - 11.57 - - 12.61 - * - meta-llama/llama-7b-chat-hf - - INT4_SYM,group_size=128,ratio=1.0,awq=True - - 12.34 - - 2.6 - * - stabilityai_stablelm-3b-4e1t - - FP32 - - 10.17 - - 10.41 - * - stabilityai_stablelm-3b-4e1t - - INT4_SYM,group_size=64,ratio=1.0,awq=True - - 10.89 - - 2.6 - * - HuggingFaceH4/zephyr-7b-beta - - FP32 - - 9.82 - - 13.99 - * - HuggingFaceH4/zephyr-7b-beta - - INT4_SYM,group_size=128,ratio=1.0 - - 10.32 - - 2.6 +.. dropdown:: Perplexity\* in data-free optimization + + .. list-table:: + :widths: 40 55 25 25 + :header-rows: 1 + + * - Model + - Optimization + - Perplexity\* + - Model Size (Gb) + * - databricks/dolly-v2-3b + - FP32 + - 5.01 + - 10.3 + * - databricks/dolly-v2-3b + - INT8_ASYM + - 5.07 + - 2.6 + * - databricks/dolly-v2-3b + - INT4_ASYM,group_size=32,ratio=0.5 + - 5.28 + - 2.2 + * - facebook/opt-6.7b + - FP32 + - 4.25 + - 24.8 + * - facebook/opt-6.7b + - INT8_ASYM + - 4.27 + - 6.2 + * - facebook/opt-6.7b + - INT4_ASYM,group_size=64,ratio=0.8 + - 4.32 + - 4.1 + * - meta-llama/Llama-2-7b-chat-hf + - FP32 + - 3.28 + - 25.1 + * - meta-llama/Llama-2-7b-chat-hf + - INT8_ASYM + - 3.29 + - 6.3 + * - meta-llama/Llama-2-7b-chat-hf + - INT4_ASYM,group_size=128,ratio=0.8 + - 3.41 + - 4.0 + * - togethercomputer/RedPajama-INCITE-7B-Instruct + - FP32 + - 4.15 + - 25.6 + * - togethercomputer/RedPajama-INCITE-7B-Instruct + - INT8_ASYM + - 4.17 + - 6.4 + * - togethercomputer/RedPajama-INCITE-7B-Instruct + - INT4_ASYM,group_size=128,ratio=1.0 + - 4.17 + - 3.6 + * - meta-llama/Llama-2-13b-chat-hf + - FP32 + - 2.92 + - 48.5 + * - meta-llama/Llama-2-13b-chat-hf + - INT8_ASYM + - 2.91 + - 12.1 + * - meta-llama/Llama-2-13b-chat-hf + - INT4_SYM,group_size=64,ratio=0.8 + - 2.98 + - 8.0 + + +.. dropdown:: Perplexity\* in data-aware optimization + + The following table shows accuracy metric in a data-aware 4-bit weight quantization + setup measured on the `Wikitext dataset `__. + + .. list-table:: + :widths: 40 55 25 25 + :header-rows: 1 + + * - Model + - Optimization + - Word perplexity\* + - Model Size (Gb) + * - meta-llama/llama-7b-chat-hf + - FP32 + - 11.57 + - 12.61 + * - meta-llama/llama-7b-chat-hf + - INT4_SYM,group_size=128,ratio=1.0,awq=True + - 12.34 + - 2.6 + * - stabilityai_stablelm-3b-4e1t + - FP32 + - 10.17 + - 10.41 + * - stabilityai_stablelm-3b-4e1t + - INT4_SYM,group_size=64,ratio=1.0,awq=True + - 10.89 + - 2.6 + * - HuggingFaceH4/zephyr-7b-beta + - FP32 + - 9.82 + - 13.99 + * - HuggingFaceH4/zephyr-7b-beta + - INT4_SYM,group_size=128,ratio=1.0 + - 10.32 + - 2.6 \*Perplexity metric in both tables was measured without the Dynamic Quantization feature enabled in the OpenVINO runtime. -Auto-tuning of Weight Compression Parameters -############################################ - -To find the optimal weight compression parameters for a particular model, refer to the -`example `__ , -where weight compression parameters are being searched from the subset of values. -To speed up the search, a self-designed validation pipeline called -`WhoWhatBench `__ -is used. The pipeline can quickly evaluate the changes in the accuracy of the optimized -model compared to the baseline. Additional Resources #################### diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression/4-bit-weight-quantization.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression/4-bit-weight-quantization.rst new file mode 100644 index 00000000000000..ae9bc7d7b8b4a3 --- /dev/null +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression/4-bit-weight-quantization.rst @@ -0,0 +1,175 @@ +4-bit Weight Quantization +========================= + +The 4-bit weight quantization method results in significant reduction in model size and +memory usage, making LLMs more accessible to less performant devices. +It also usually offers lower inference latency, however, depending on specific models, +it may potentially impact the accuracy. + +Nevertheless, the INT4 method has several parameters that can provide different performance-accuracy +trade-offs after optimization: + +* ``mode`` - there are two optimization modes: symmetric and asymmetric. + + .. tab-set:: + + .. tab-item:: Symmetric Compression + :sync: int4-sym + + INT4 Symmetric mode (``INT4_SYM``) involves quantizing weights to a signed 4-bit integer + symmetrically without zero point. This mode is faster than the INT8_ASYM, making + it ideal for situations where **speed and size reduction are prioritized over accuracy**. + + .. code-block:: python + + from nncf import compress_weights + from nncf import CompressWeightsMode + + compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_SYM) + + .. tab-item:: Asymmetric Compression + :sync: int4-asym + + INT4 Asymmetric mode (``INT4_ASYM``) also uses an unsigned 4-bit integer but quantizes weights + asymmetrically with a non-fixed zero point. This mode slightly compromises speed in + favor of better accuracy compared to the symmetric mode. This mode is useful when + **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. + + .. code-block:: python + + from nncf import compress_weights + from nncf import CompressWeightsMode + + compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM) + +* ``group_size`` controls the size of the group of weights that share the same + quantization parameters. Shared quantization parameters help to speed up the + calculation of activation values as they are dequantized and quantized between + layers. However, they can reduce accuracy. The following group sizes are + recommended: ``128``, ``64``, ``32`` (``128`` is default value). + + `Smaller Group Size`: Leads to a more accurate model but increases the model's + footprint and reduces inference speed. + + `Larger Group Size`: Results in faster inference and a smaller model, but might + compromise accuracy. + +* ``ratio`` controls the ratio between the layers compressed to the precision defined + by ``mode`` and the rest of the layers that will be kept in the ``backup_mode`` in the optimized model. + Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be + compressed to the precision defined by ``mode``, while the rest will be compressed to + ``backup_mode`` precision. The default value for ratio is 1. + + | **Higher Ratio (more layers set to mode precision)**: + | Reduces the model size and increase inference speed but + might lead to higher accuracy degradation. + + | **Lower Ratio (more layers set to backup_mode precision)**: + | Maintains better accuracy but results in a larger model size + and potentially slower inference. + + In the example below, 90% of the model's layers are quantized to INT4 asymmetrically with + a group size of 64: + + .. code-block:: python + + from nncf import compress_weights, CompressWeightsMode + + # Example: Compressing weights with INT4_ASYM mode, group size of 64, and 90% INT4 ratio + compressed_model = compress_weights( + model, + mode=CompressWeightsMode.INT4_ASYM, + group_size=64, + ratio=0.9, + ) + +* ``scale_estimation`` - a boolean parameter that enables more accurate estimation of + quantization scales. Especially helpful when the weights of all layers are quantized to + 4 bits. Requires dataset. + +* ``awq`` - a boolean parameter that enables the AWQ method for more accurate INT4 weight + quantization. Especially helpful when the weights of all the layers are quantized to + 4 bits. The method can sometimes result in reduced accuracy when used with + Dynamic Quantization of activations. Requires dataset. + +* ``gptq`` - a boolean parameter that enables the GPTQ method for more accurate INT4 weight + quantization. Requires dataset. + +* ``dataset`` - a calibration dataset for data-aware weight compression. It is required + for some compression options, for example, ``scale_estimation``, ``gptq`` or ``awq``. Some types + of ``sensitivity_metric`` can use data for precision selection. + +* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing + layers in the bit-width selection algorithm. Some of the metrics require dataset to be + provided. The following types are supported: + + * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - a data-free metric computed as + the inverted 8-bit quantization noise. Weights with highest value of this metric can + be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in + 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more + accurate than per-channel, accuracy should not degrade. + + * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires a dataset. The average + Hessian trace of weights with respect to the layer-wise quantization error multiplied + by L2 norm of 8-bit quantization noise. + + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires a dataset. The mean + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + + * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires a dataset. The maximum + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires a dataset. The mean + magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. + +* ``all_layers`` - a boolean parameter that enables INT4 weight quantization of all + Fully-Connected and Embedding layers, including the first and last layers in the model. + +* ``lora_correction`` - a boolean parameter that enables the LoRA Correction Algorithm + to further improve the accuracy of INT4 compressed models on top of other + algorithms - AWQ and Scale Estimation. + +* ``backup_mode`` - defines a backup precision for mixed-precision weight compression. + There are three modes: INT8_ASYM, INT8_SYM, and NONE, which retains + the original floating-point precision of the model weights (``INT8_ASYM`` is default value). + +| + +4-bit Weight Quantization with GPTQ +################################### + +You can use models from Hugging Face +`Transformers `__ library, which are quantized +with `GPTQ `__ algorithm. Such models do not require +additional optimization step because the conversion will automatically preserve +the INT4 optimization results, and model inference will eventually benefit from it. + +See the `example of a model `__ +that has been optimized with GPTQ. + +You can also refer to the code sample below which shows how to load a 4-bit +GPTQ model and run inference. + +.. dropdown:: Using a GPTQ model. + + Make sure to install GPTQ dependencies by running the following command: + + .. code-block:: python + + pip install optimum[openvino] auto-gptq + + .. code-block:: python + + from optimum.intel.openvino import OVModelForCausalLM + from transformers import AutoTokenizer, pipeline + + # Load model from Hugging Face already optimized with GPTQ + model_id = "TheBloke/Llama-2-7B-Chat-GPTQ" + model = OVModelForCausalLM.from_pretrained(model_id, export=True) + + # Inference + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) diff --git a/docs/dev/ov_dependencies.txt b/docs/dev/ov_dependencies.txt index 7cf26c58a9436d..d9c344d2c3048d 100644 --- a/docs/dev/ov_dependencies.txt +++ b/docs/dev/ov_dependencies.txt @@ -1,6 +1,6 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#This file provides a comprehensive list of all dependencies of OpenVINO 2024.4 +#This file provides a comprehensive list of all dependencies of OpenVINO 2024.5 #The file is part of the automation pipeline for posting OpenVINO IR models on the HuggingFace Hub, including OneBOM dependency checks. diff --git a/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py b/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py index ca93d02d75c6a9..dcce229308da24 100644 --- a/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py +++ b/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py @@ -126,7 +126,7 @@ def process_coveo_meta(meta, url, link): if tag_name == 'ovdoctype': ET.SubElement(namespace_element, tag_name).text = process_link(link) elif tag_name == 'ovcategory' and loc_element is not None: - ET.SubElement(namespace_element, tag_name).text = extract_link(loc_element.text) + ET.SubElement(namespace_element, tag_name).text = extract_hierarchy(loc_element.text) elif tag_name == 'ovversion': ET.SubElement(namespace_element, tag_name).text = tag_value @@ -135,9 +135,14 @@ def process_link(link): return link.split('/')[0].replace("-", " ") return link.split('.html')[0].replace("-", " ") -def extract_link(link): +def extract_hierarchy(link): path = link.split("://")[-1] segments = path.split('/')[1:] if segments and segments[-1].endswith('.html'): segments = segments[:-1] - return '|'.join(segments) \ No newline at end of file + + hierarchy = [] + for i in range(1, len(segments) + 1): + hierarchy.append('|'.join(segments[:i])) + + return ';'.join(hierarchy) \ No newline at end of file diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_torch_fx.py b/docs/optimization_guide/nncf/ptq/code/ptq_torch_fx.py new file mode 100644 index 00000000000000..b8f76304099ae6 --- /dev/null +++ b/docs/optimization_guide/nncf/ptq/code/ptq_torch_fx.py @@ -0,0 +1,44 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +#! [dataset] +import nncf +import torch + +calibration_loader = torch.utils.data.DataLoader(...) + +def transform_fn(data_item): + images, _ = data_item + return images + +calibration_dataset = nncf.Dataset(calibration_loader, transform_fn) +#! [dataset] + +#! [quantization] +import torchvision +from nncf.torch import disable_patching + +input_fp32 = torch.ones((1, 3, 224, 224)) # FP32 model input +model = torchvision.models.resnet50(pretrained=True) + +with disable_patching(): + exported_model = torch.export.export_for_training(model, args=(input_fp32,)).module() + quantized_model = nncf.quantize(exported_model, calibration_dataset) +#! [quantization] + +#! [inference] +import openvino.torch + +input_fp32 = ... # FP32 model input + +# compile quantized model using torch.compile API +with disable_patching(): + compiled_model_int8 = torch.compile(quantized_model, backend="openvino") + # OpenVINO backend compiles the model during the first call, + # so the first call is expected to be slower than the following calls + res = compiled_model_int8(input_fp32) + + # save the model + exported_program = torch.export.export(quantized_model, args=(input_fp32,)) + torch.export.save(exported_program, 'exported_program.pt2') +#! [inference] diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx deleted file mode 100644 index 9b53d90e0862db..00000000000000 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx and /dev/null differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-platform_list.pdf b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-platform_list.pdf deleted file mode 100644 index cba78e5244acf1..00000000000000 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-platform_list.pdf and /dev/null differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx deleted file mode 100644 index 4e243b8190c876..00000000000000 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx and /dev/null differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx new file mode 100644 index 00000000000000..0c29b3282790fa Binary files /dev/null and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-platform_list.pdf b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-platform_list.pdf new file mode 100644 index 00000000000000..9cb20570020cc8 Binary files /dev/null and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-platform_list.pdf differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx new file mode 100644 index 00000000000000..60150e1a457eaf Binary files /dev/null and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json index 7576fd96148554..59e06ef51f812d 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json @@ -1,18 +1,22 @@ [ { - "Platform": "Intel® Celeron® 6305E CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 11.75, - "fp16": "", - "fp32": 4.32, - "bf16": "" + "int8": 312.06, + "fp16": 345.49, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -22,7 +26,7 @@ "Precisions": [ { "int4": "", - "int8": 87.69, + "int8": 4.83, "fp16": "", "fp32": "", "bf16": "" @@ -34,19 +38,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.17, - "fp16": "", - "fp32": 15.03, - "bf16": "" + "int8": 328.55, + "fp16": 285.3, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -56,7 +64,7 @@ "Precisions": [ { "int4": "", - "int8": 49.24, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -68,19 +76,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 32.5, + "int8": "", "fp16": "", - "fp32": 21.9, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 20.07, + "token_int8": 17.42, + "token_fp16": "" } ], "Unit": "FPS", @@ -89,8 +101,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 35.99, + "int4": 49.81, + "int8": 57.4, "fp16": "", "fp32": "", "bf16": "" @@ -102,19 +114,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 33.02, + "int8": "", "fp16": "", - "fp32": 12.59, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 36.48, + "token_int8": 27.59, + "token_fp16": "" } ], "Unit": "FPS", @@ -123,8 +139,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 45.4, + "int4": 27.41, + "int8": 36.24, "fp16": "", "fp32": "", "bf16": "" @@ -136,19 +152,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39.93, + "int8": "", "fp16": "", - "fp32": 15.99, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 42.82, + "token_int8": 33.97, + "token_fp16": 22.23 } ], "Unit": "FPS", @@ -157,9 +177,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 40.15, - "fp16": "", + "int4": 23.35, + "int8": 29.43, + "fp16": 44.97, "fp32": "", "bf16": "" } @@ -170,19 +190,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 119.95, + "int8": "", "fp16": "", - "fp32": 47.19, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 39.6, + "token_int8": 30.59, + "token_fp16": "" } ], "Unit": "FPS", @@ -191,8 +215,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 13.31, + "int4": 25.25, + "int8": 32.69, "fp16": "", "fp32": "", "bf16": "" @@ -204,19 +228,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 33.13, + "int8": "", "fp16": "", - "fp32": 22.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 55.37, + "token_int8": 51.62, + "token_fp16": 35.82 } ], "Unit": "FPS", @@ -225,9 +253,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 31.1, - "fp16": "", + "int4": 18.06, + "int8": 19.37, + "fp16": 27.91, "fp32": "", "bf16": "" } @@ -238,19 +266,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 50.91, - "fp16": "", - "fp32": 18.37, - "bf16": "" + "int8": 34.84, + "fp16": 19.43, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -260,7 +292,7 @@ "Precisions": [ { "int4": "", - "int8": 22.52, + "int8": 48.51, "fp16": "", "fp32": "", "bf16": "" @@ -272,19 +304,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38.31, + "int8": "", "fp16": "", - "fp32": 13.71, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 43.4, + "token_int8": 32.32, + "token_fp16": 20.91 } ], "Unit": "FPS", @@ -293,9 +329,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 28.51, - "fp16": "", + "int4": 23.04, + "int8": 30.94, + "fp16": 47.82, "fp32": "", "bf16": "" } @@ -306,19 +342,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 86.13, - "fp16": "", - "fp32": 33.75, - "bf16": "" + "int8": 2348.6, + "fp16": 2074.34, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -328,7 +368,7 @@ "Precisions": [ { "int4": "", - "int8": 16.59, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -340,19 +380,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 44.68, + "int8": "", "fp16": "", - "fp32": 17.96, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 59.06, + "token_int8": 47.96, + "token_fp16": 29.29 } ], "Unit": "FPS", @@ -361,9 +405,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 37.63, - "fp16": "", + "int4": 16.93, + "int8": 20.85, + "fp16": 34.14, "fp32": "", "bf16": "" } @@ -374,19 +418,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 60.67, + "int8": "", "fp16": "", - "fp32": 23.99, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 40.48, + "token_int8": 32.79, + "token_fp16": 20.67 } ], "Unit": "FPS", @@ -395,9 +443,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 24.02, - "fp16": "", + "int4": 24.7, + "int8": 30.49, + "fp16": 48.37, "fp32": "", "bf16": "" } @@ -408,19 +456,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 27.34, - "fp16": "", - "fp32": 18.04, - "bf16": "" + "int8": 1401.85, + "fp16": 1046.9, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -430,7 +482,7 @@ "Precisions": [ { "int4": "", - "int8": 30.86, + "int8": 1.42, "fp16": "", "fp32": "", "bf16": "" @@ -442,19 +494,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 32.78, - "fp16": "", - "fp32": 21.32, - "bf16": "" + "int8": 112.21, + "fp16": 73.01, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -464,7 +520,7 @@ "Precisions": [ { "int4": "", - "int8": 38.38, + "int8": 14.86, "fp16": "", "fp32": "", "bf16": "" @@ -476,19 +532,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 168.92, - "fp16": "", - "fp32": 67.23, - "bf16": "" + "int8": 1308.1, + "fp16": 1201.69, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -498,7 +558,7 @@ "Precisions": [ { "int4": "", - "int8": 10.73, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -510,19 +570,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "stable-diffusion-v1-5", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.48, + "int8": "", "fp16": "", - "fp32": 35.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -532,7 +596,7 @@ "Precisions": [ { "int4": "", - "int8": 26.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -544,19 +608,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 20.68, - "fp16": "", - "fp32": 14.76, - "bf16": "" + "int8": 517.1, + "fp16": 550.33, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -566,7 +634,7 @@ "Precisions": [ { "int4": "", - "int8": 49.95, + "int8": 3.21, "fp16": "", "fp32": "", "bf16": "" @@ -578,19 +646,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 218.51, + "int8": 23.3, "fp16": "", - "fp32": 80.07, - "bf16": "" + "fp32": 23.72, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -600,7 +672,7 @@ "Precisions": [ { "int4": "", - "int8": 14.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -612,19 +684,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 590.15, + "int8": 228.97, "fp16": "", - "fp32": 224.94, - "bf16": "" + "fp32": 219.37, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -634,7 +710,7 @@ "Precisions": [ { "int4": "", - "int8": 9.22, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -646,19 +722,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 881.6, + "int8": 59.38, "fp16": "", - "fp32": 338.79, - "bf16": "" + "fp32": 54.24, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -668,7 +748,7 @@ "Precisions": [ { "int4": "", - "int8": 5.08, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -680,19 +760,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3032.01, + "int8": 1.26, "fp16": "", - "fp32": 488.41, - "bf16": 1975.07 + "fp32": 1.08, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -702,10 +786,10 @@ "Precisions": [ { "int4": "", - "int8": 3.74, + "int8": "", "fp16": "", "fp32": "", - "bf16": 4.78 + "bf16": "" } ], "Unit": "ms", @@ -714,19 +798,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4693.95, + "int8": 111.92, "fp16": "", - "fp32": 562.05, - "bf16": 3202.49 + "fp32": 98.44, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -736,10 +824,10 @@ "Precisions": [ { "int4": "", - "int8": 3.77, + "int8": "", "fp16": "", "fp32": "", - "bf16": 4.61 + "bf16": "" } ], "Unit": "ms", @@ -748,19 +836,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 427.23, + "int8": "", "fp16": "", - "fp32": 164.01, - "bf16": "" + "fp32": 34.99, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -770,7 +862,7 @@ "Precisions": [ { "int4": "", - "int8": 11.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -782,19 +874,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Atom® X6425E CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 429.84, + "int8": 36.35, "fp16": "", - "fp32": 167.86, - "bf16": "" + "fp32": 33.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -804,7 +900,7 @@ "Precisions": [ { "int4": "", - "int8": 8.0, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -816,19 +912,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 76.06, + "int8": 7.26, "fp16": "", - "fp32": 30.37, - "bf16": "" + "fp32": 5.01, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -838,7 +938,7 @@ "Precisions": [ { "int4": "", - "int8": 25.96, + "int8": 139.68, "fp16": "", "fp32": "", "bf16": "" @@ -850,19 +950,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.79, + "int8": 134.16, "fp16": "", - "fp32": 21.03, - "bf16": "" + "fp32": 80.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -872,7 +976,7 @@ "Precisions": [ { "int4": "", - "int8": 32.08, + "int8": 7.8, "fp16": "", "fp32": "", "bf16": "" @@ -884,19 +988,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 73.09, + "int8": 19.87, "fp16": "", - "fp32": 26.35, - "bf16": "" + "fp32": 8.15, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -906,7 +1014,7 @@ "Precisions": [ { "int4": "", - "int8": 19.84, + "int8": 51.33, "fp16": "", "fp32": "", "bf16": "" @@ -918,19 +1026,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 404.72, - "fp16": 444.29, - "fp32": "", - "bf16": "" + "int8": 0.33, + "fp16": "", + "fp32": 0.13, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -940,7 +1052,7 @@ "Precisions": [ { "int4": "", - "int8": 3.24, + "int8": 2995.1, "fp16": "", "fp32": "", "bf16": "" @@ -952,19 +1064,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 45.84, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 21.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -974,7 +1090,7 @@ "Precisions": [ { "int4": "", - "int8": 4.76, + "int8": 22.72, "fp16": "", "fp32": "", "bf16": "" @@ -986,19 +1102,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 187.77, - "fp16": 149.65, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 5.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1008,7 +1128,7 @@ "Precisions": [ { "int4": "", - "int8": 5.71, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1020,19 +1140,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Atom® X6425E CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 89.12, - "fp16": 74.2, - "fp32": "", - "bf16": "" + "int8": 10.31, + "fp16": "", + "fp32": 5.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1042,7 +1166,7 @@ "Precisions": [ { "int4": "", - "int8": 12.51, + "int8": 99.61, "fp16": "", "fp32": "", "bf16": "" @@ -1054,19 +1178,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Atom® X6425E iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 89.38, - "fp16": 74.26, + "int8": 22.02, + "fp16": 25.05, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1076,7 +1204,7 @@ "Precisions": [ { "int4": "", - "int8": 12.3, + "int8": 60.1, "fp16": "", "fp32": "", "bf16": "" @@ -1088,19 +1216,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Atom® X6425E iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 177.17, - "fp16": 139.65, + "int8": 187.37, + "fp16": 222.58, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1110,7 +1242,7 @@ "Precisions": [ { "int4": "", - "int8": 6.02, + "int8": 7.71, "fp16": "", "fp32": "", "bf16": "" @@ -1122,19 +1254,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Atom® X6425E iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 42.88, - "fp16": 33.61, + "int8": 48.1, + "fp16": 51.68, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1144,7 +1280,7 @@ "Precisions": [ { "int4": "", - "int8": 26.85, + "int8": 22.89, "fp16": "", "fp32": "", "bf16": "" @@ -1156,19 +1292,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® X6425E iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 47.07, - "fp16": 39.06, - "fp32": "", - "bf16": "" + "int8": 1.16, + "fp16": 1.16, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1178,7 +1318,7 @@ "Precisions": [ { "int4": "", - "int8": 19.89, + "int8": 870.65, "fp16": "", "fp32": "", "bf16": "" @@ -1190,19 +1330,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® X6425E iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 47.0, - "fp16": 39.61, + "int8": 93.36, + "fp16": 95.62, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1212,7 +1356,7 @@ "Precisions": [ { "int4": "", - "int8": 18.15, + "int8": 13.54, "fp16": "", "fp32": "", "bf16": "" @@ -1224,19 +1368,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® X6425E iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 67.79, - "fp16": 52.83, + "int8": 31.79, + "fp16": 33.13, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1246,7 +1394,7 @@ "Precisions": [ { "int4": "", - "int8": 17.21, + "int8": 35.83, "fp16": "", "fp32": "", "bf16": "" @@ -1258,19 +1406,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 47.29, - "fp16": 40.73, - "fp32": "", - "bf16": "" + "int8": 39.3, + "fp16": "", + "fp32": 28.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1280,7 +1432,7 @@ "Precisions": [ { "int4": "", - "int8": 21.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1292,19 +1444,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 88.92, - "fp16": 69.92, - "fp32": "", - "bf16": "" + "int8": 480.45, + "fp16": "", + "fp32": 302.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1314,7 +1470,7 @@ "Precisions": [ { "int4": "", - "int8": 12.82, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1326,19 +1482,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 66.18, - "fp16": 52.8, - "fp32": "", - "bf16": "" + "int8": 129.7, + "fp16": "", + "fp32": 54.69, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1348,7 +1508,7 @@ "Precisions": [ { "int4": "", - "int8": 14.46, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1360,19 +1520,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.49, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 0.86, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1382,7 +1546,7 @@ "Precisions": [ { "int4": "", - "int8": 12.42, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1394,19 +1558,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 159.37, - "fp16": 100.87, - "fp32": "", - "bf16": "" + "int8": 233.16, + "fp16": "", + "fp32": 114.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1416,7 +1584,7 @@ "Precisions": [ { "int4": "", - "int8": 6.11, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1428,19 +1596,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 152.5, - "fp16": 108.49, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 41.37, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1450,7 +1622,7 @@ "Precisions": [ { "int4": "", - "int8": 7.14, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1462,19 +1634,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 316.53, - "fp16": 268.1, - "fp32": "", - "bf16": "" + "int8": 67.73, + "fp16": "", + "fp32": 36.05, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1484,7 +1660,7 @@ "Precisions": [ { "int4": "", - "int8": 4.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1496,19 +1672,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 44.78, + "int8": 14.29, "fp16": "", - "fp32": 33.39, - "bf16": "" + "fp32": 11.18, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1518,7 +1698,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 71.84, "fp16": "", "fp32": "", "bf16": "" @@ -1530,19 +1710,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 273.98, "fp16": "", - "fp32": 16.3, - "bf16": "" + "fp32": 169.54, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1552,7 +1736,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 4.05, "fp16": "", "fp32": "", "bf16": "" @@ -1564,19 +1748,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 49.58, + "int8": 45.27, "fp16": "", - "fp32": 26.72, - "bf16": "" + "fp32": 18.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1586,7 +1774,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 23.76, "fp16": "", "fp32": "", "bf16": "" @@ -1598,19 +1786,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 85.59, + "int8": 0.76, "fp16": "", - "fp32": 51.66, - "bf16": "" + "fp32": 0.31, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1620,7 +1812,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1317.43, "fp16": "", "fp32": "", "bf16": "" @@ -1632,19 +1824,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 48.88, + "int8": 98.2, "fp16": "", - "fp32": 25.61, - "bf16": "" + "fp32": 45.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1654,7 +1850,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 10.52, "fp16": "", "fp32": "", "bf16": "" @@ -1666,10 +1862,11 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ @@ -1677,8 +1874,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 13.77, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1700,19 +1900,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 61.03, + "int8": 21.58, "fp16": "", - "fp32": 32.25, - "bf16": "" + "fp32": 11.78, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1722,7 +1926,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 47.39, "fp16": "", "fp32": "", "bf16": "" @@ -1734,19 +1938,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 46.74, - "bf16": "" + "int8": 40.0, + "fp16": 34.31, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1756,7 +1964,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 34.13, "fp16": "", "fp32": "", "bf16": "" @@ -1768,19 +1976,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 240.55, - "fp16": "", - "fp32": 157.84, - "bf16": "" + "int8": 414.66, + "fp16": 324.8, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1790,7 +2002,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.49, "fp16": "", "fp32": "", "bf16": "" @@ -1802,19 +2014,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Atom® x7425E iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 73.76, - "bf16": "" + "int8": 106.34, + "fp16": 64.69, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1824,7 +2040,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 10.56, "fp16": "", "fp32": "", "bf16": "" @@ -1836,19 +2052,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Atom® x7425E iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.18, - "fp16": "", - "fp32": 0.38, - "bf16": "" + "int8": 2.16, + "fp16": 1.32, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1858,7 +2078,7 @@ "Precisions": [ { "int4": "", - "int8": 853.85, + "int8": 472.59, "fp16": "", "fp32": "", "bf16": "" @@ -1870,19 +2090,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Atom® x7425E iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.0, - "fp16": "", - "fp32": 1.27, - "bf16": "" + "int8": 211.07, + "fp16": 137.13, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1892,7 +2116,7 @@ "Precisions": [ { "int4": "", - "int8": 510.0, + "int8": 6.2, "fp16": "", "fp32": "", "bf16": "" @@ -1904,19 +2128,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Atom® x7425E iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.05, - "fp16": "", - "fp32": 1.87, - "bf16": "" + "int8": 60.92, + "fp16": 44.64, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1926,7 +2154,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 18.51, "fp16": "", "fp32": "", "bf16": "" @@ -1938,19 +2166,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 45.34, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 33.5, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1960,7 +2192,7 @@ "Precisions": [ { "int4": "", - "int8": 424.59, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1972,19 +2204,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 57.78, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 48.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2006,19 +2242,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.56, "fp16": "", - "fp32": 3.91, - "bf16": "" + "fp32": 0.51, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2028,7 +2268,7 @@ "Precisions": [ { "int4": "", - "int8": 128.89, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2040,19 +2280,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.01, + "int8": 525.47, "fp16": "", - "fp32": 1.89, - "bf16": "" + "fp32": 392.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2062,7 +2306,7 @@ "Precisions": [ { "int4": "", - "int8": 324.87, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2074,19 +2318,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.03, + "int8": 197.41, "fp16": "", - "fp32": 1.64, - "bf16": "" + "fp32": 115.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2096,7 +2344,7 @@ "Precisions": [ { "int4": "", - "int8": 198.73, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2108,19 +2356,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.69, + "int8": 5.38, "fp16": "", - "fp32": 1.2, - "bf16": "" + "fp32": 2.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2130,7 +2382,7 @@ "Precisions": [ { "int4": "", - "int8": 268.0, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2142,19 +2394,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.52, + "int8": 316.13, "fp16": "", - "fp32": 2.85, - "bf16": "" + "fp32": 194.29, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2176,10 +2432,11 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ @@ -2187,8 +2444,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 80.2, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2198,7 +2458,7 @@ "Precisions": [ { "int4": "", - "int8": 360.08, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2210,19 +2470,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.26, + "int8": 114.67, "fp16": "", - "fp32": 2.05, - "bf16": "" + "fp32": 78.26, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2244,19 +2508,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.77, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 4.32, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2266,7 +2534,7 @@ "Precisions": [ { "int4": "", - "int8": 329.86, + "int8": 87.73, "fp16": "", "fp32": "", "bf16": "" @@ -2278,19 +2546,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.14, + "int8": 18.94, "fp16": "", - "fp32": 1.9, - "bf16": "" + "fp32": 11.49, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2300,7 +2572,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 55.76, "fp16": "", "fp32": "", "bf16": "" @@ -2312,19 +2584,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 16.05, + "int8": 0.17, "fp16": "", - "fp32": 5.99, - "bf16": "" + "fp32": 0.04, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2334,7 +2610,7 @@ "Precisions": [ { "int4": "", - "int8": 94.97, + "int8": 5772.15, "fp16": "", "fp32": "", "bf16": "" @@ -2346,19 +2622,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.17, + "int8": 301.05, "fp16": "", - "fp32": 3.33, - "bf16": "" + "fp32": 132.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2368,7 +2648,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.6, "fp16": "", "fp32": "", "bf16": "" @@ -2380,19 +2660,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 51.66, "fp16": "", - "fp32": 1.27, - "bf16": "" + "fp32": 14.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2402,7 +2686,7 @@ "Precisions": [ { "int4": "", - "int8": 513.28, + "int8": 19.8, "fp16": "", "fp32": "", "bf16": "" @@ -2414,19 +2698,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.82, + "int8": 0.89, "fp16": "", - "fp32": 6.98, - "bf16": "" + "fp32": 0.23, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2436,7 +2724,7 @@ "Precisions": [ { "int4": "", - "int8": 102.33, + "int8": 1118.71, "fp16": "", "fp32": "", "bf16": "" @@ -2448,19 +2736,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 51.17, + "int8": 115.03, "fp16": "", - "fp32": 18.46, - "bf16": "" + "fp32": 36.99, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2470,7 +2762,7 @@ "Precisions": [ { "int4": "", - "int8": 48.78, + "int8": 9.06, "fp16": "", "fp32": "", "bf16": "" @@ -2482,19 +2774,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 78.31, + "int8": "", "fp16": "", - "fp32": 29.72, - "bf16": "" + "fp32": 11.94, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2504,7 +2800,7 @@ "Precisions": [ { "int4": "", - "int8": 38.13, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2516,19 +2812,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 244.06, + "int8": 25.97, "fp16": "", - "fp32": 41.97, - "bf16": 211.62 + "fp32": 9.66, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2538,10 +2838,10 @@ "Precisions": [ { "int4": "", - "int8": 25.21, + "int8": 40.21, "fp16": "", "fp32": "", - "bf16": 27.63 + "bf16": "" } ], "Unit": "ms", @@ -2550,19 +2850,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 305.38, - "fp16": "", - "fp32": 55.37, - "bf16": 289.16 + "int8": 43.69, + "fp16": 33.8, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2572,10 +2876,10 @@ "Precisions": [ { "int4": "", - "int8": 19.29, + "int8": 26.56, "fp16": "", "fp32": "", - "bf16": 25.79 + "bf16": "" } ], "Unit": "ms", @@ -2584,19 +2888,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38.28, - "fp16": "", - "fp32": 13.45, - "bf16": "" + "int8": 73.58, + "fp16": 58.53, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2606,7 +2914,7 @@ "Precisions": [ { "int4": "", - "int8": 69.24, + "int8": 25.45, "fp16": "", "fp32": "", "bf16": "" @@ -2618,19 +2926,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38.81, - "fp16": "", - "fp32": 15.09, - "bf16": "" + "int8": 0.48, + "fp16": 0.52, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2640,7 +2952,7 @@ "Precisions": [ { "int4": "", - "int8": 62.64, + "int8": 2110.65, "fp16": "", "fp32": "", "bf16": "" @@ -2652,19 +2964,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.34, - "fp16": "", - "fp32": 2.49, - "bf16": "" + "int8": 671.35, + "fp16": 504.8, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2674,7 +2990,7 @@ "Precisions": [ { "int4": "", - "int8": 194.83, + "int8": 2.72, "fp16": "", "fp32": "", "bf16": "" @@ -2686,19 +3002,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.5, - "fp16": "", - "fp32": 1.72, - "bf16": "" + "int8": 203.17, + "fp16": 118.59, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2708,7 +3028,7 @@ "Precisions": [ { "int4": "", - "int8": 246.14, + "int8": 6.3, "fp16": "", "fp32": "", "bf16": "" @@ -2720,19 +3040,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.21, - "fp16": "", - "fp32": 2.31, - "bf16": "" + "int8": 5.09, + "fp16": 2.78, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2742,7 +3066,7 @@ "Precisions": [ { "int4": "", - "int8": 195.21, + "int8": 210.41, "fp16": "", "fp32": "", "bf16": "" @@ -2754,19 +3078,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 121.51, - "fp16": 110.4, + "int8": 396.07, + "fp16": 221.18, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2776,7 +3104,7 @@ "Precisions": [ { "int4": "", - "int8": 9.1, + "int8": 4.3, "fp16": "", "fp32": "", "bf16": "" @@ -2788,19 +3116,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Celeron® 6305E iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 121.77, + "fp16": 81.6, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2810,7 +3142,7 @@ "Precisions": [ { "int4": "", - "int8": 12.35, + "int8": 10.34, "fp16": "", "fp32": "", "bf16": "" @@ -2822,19 +3154,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 30.58, - "fp16": 21.73, - "fp32": "", - "bf16": "" + "int8": 243.99, + "fp16": "", + "fp32": 157.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2844,7 +3180,7 @@ "Precisions": [ { "int4": "", - "int8": 32.8, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2856,19 +3192,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10.38, - "fp16": 6.72, - "fp32": "", - "bf16": "" + "int8": 189.52, + "fp16": "", + "fp32": 154.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2878,7 +3218,7 @@ "Precisions": [ { "int4": "", - "int8": 97.03, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2890,19 +3230,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10.38, - "fp16": 6.74, - "fp32": "", - "bf16": "" + "int8": 2.45, + "fp16": "", + "fp32": 1.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2912,7 +3256,7 @@ "Precisions": [ { "int4": "", - "int8": 97.28, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2924,19 +3268,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 32.32, + "int8": 4485.9, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 2415.8, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2946,7 +3294,7 @@ "Precisions": [ { "int4": "", - "int8": 43.5, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2958,19 +3306,23 @@ } }, { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.91, + "int8": 1097.16, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 475.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2980,7 +3332,7 @@ "Precisions": [ { "int4": "", - "int8": 329.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2992,19 +3344,23 @@ } }, { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 18.81, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 9.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3014,7 +3370,7 @@ "Precisions": [ { "int4": "", - "int8": 720.67, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3026,19 +3382,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.47, - "fp16": 3.64, - "fp32": "", - "bf16": "" + "int8": 1120.99, + "fp16": "", + "fp32": 624.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3048,7 +3408,7 @@ "Precisions": [ { "int4": "", - "int8": 192.94, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3060,19 +3420,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.41, - "fp16": 4.03, - "fp32": "", - "bf16": "" + "int8": 374.74, + "fp16": "", + "fp32": 236.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3082,7 +3446,7 @@ "Precisions": [ { "int4": "", - "int8": 173.57, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3094,19 +3458,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.17, - "fp16": 4.28, - "fp32": "", - "bf16": "" + "int8": 76.15, + "fp16": "", + "fp32": 30.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3116,7 +3484,7 @@ "Precisions": [ { "int4": "", - "int8": 137.83, + "int8": 25.21, "fp16": "", "fp32": "", "bf16": "" @@ -3128,19 +3496,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.85, - "fp16": 6.59, - "fp32": "", - "bf16": "" + "int8": 97.68, + "fp16": "", + "fp32": 66.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3150,7 +3522,7 @@ "Precisions": [ { "int4": "", - "int8": 106.93, + "int8": 22.16, "fp16": "", "fp32": "", "bf16": "" @@ -3162,19 +3534,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.41, - "fp16": 4.47, - "fp32": "", - "bf16": "" + "int8": 1.2, + "fp16": "", + "fp32": 0.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3184,7 +3560,7 @@ "Precisions": [ { "int4": "", - "int8": 179.75, + "int8": 1025.52, "fp16": "", "fp32": "", "bf16": "" @@ -3196,19 +3572,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 11.76, - "fp16": 8.39, - "fp32": "", - "bf16": "" + "int8": 1969.75, + "fp16": "", + "fp32": 815.83, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3218,7 +3598,7 @@ "Precisions": [ { "int4": "", - "int8": 87.43, + "int8": 1.36, "fp16": "", "fp32": "", "bf16": "" @@ -3230,19 +3610,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.08, - "fp16": 5.89, - "fp32": "", - "bf16": "" + "int8": 390.17, + "fp16": "", + "fp32": 94.82, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3252,7 +3636,7 @@ "Precisions": [ { "int4": "", - "int8": 110.63, + "int8": 6.23, "fp16": "", "fp32": "", "bf16": "" @@ -3264,19 +3648,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 6.38, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1.6, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3286,7 +3674,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 209.14, "fp16": "", "fp32": "", "bf16": "" @@ -3298,19 +3686,23 @@ } }, { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.39, + "int8": 685.79, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 242.78, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3320,7 +3712,7 @@ "Precisions": [ { "int4": "", - "int8": 408.09, + "int8": 2.71, "fp16": "", "fp32": "", "bf16": "" @@ -3332,19 +3724,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 23.39, - "fp16": 14.13, - "fp32": "", - "bf16": "" + "int8": 166.55, + "fp16": "", + "fp32": 64.31, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3354,7 +3750,7 @@ "Precisions": [ { "int4": "", - "int8": 39.66, + "int8": 12.75, "fp16": "", "fp32": "", "bf16": "" @@ -3366,19 +3762,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 88.41, + "fp16": 74.04, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3388,7 +3788,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 12.15, "fp16": "", "fp32": "", "bf16": "" @@ -3400,19 +3800,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 54.9, - "fp16": 49.62, + "int8": 37.81, + "fp16": 34.74, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3422,7 +3826,7 @@ "Precisions": [ { "int4": "", - "int8": 20.12, + "int8": 27.47, "fp16": "", "fp32": "", "bf16": "" @@ -3434,19 +3838,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.85, + "int8": "", "fp16": "", - "fp32": 3.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 0.27, + "token_fp16": 2.55 } ], "Unit": "FPS", @@ -3456,8 +3864,8 @@ "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 3688.24, + "fp16": 390.94, "fp32": "", "bf16": "" } @@ -3468,19 +3876,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 1.5, - "bf16": "" + "int8": 1966.11, + "fp16": 1346.18, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3490,7 +3902,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.79, "fp16": "", "fp32": "", "bf16": "" @@ -3502,19 +3914,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.12, + "int8": "", "fp16": "", - "fp32": 2.56, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 3.7, + "token_int8": 0.49, + "token_fp16": 3.91 } ], "Unit": "FPS", @@ -3523,9 +3939,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 269.82, + "int8": 2003.58, + "fp16": 255.57, "fp32": "", "bf16": "" } @@ -3536,19 +3952,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 9.46, - "fp16": "", - "fp32": 5.5, - "bf16": "" + "int8": 771.23, + "fp16": 382.83, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3558,7 +3978,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.58, "fp16": "", "fp32": "", "bf16": "" @@ -3570,19 +3990,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.05, - "fp16": "", - "fp32": 2.57, - "bf16": "" + "int8": 705.76, + "fp16": 453.35, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3592,7 +4016,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.67, "fp16": "", "fp32": "", "bf16": "" @@ -3604,19 +4028,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 5.4, - "bf16": "" + "int8": 126.18, + "fp16": 129.18, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3626,7 +4054,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.71, "fp16": "", "fp32": "", "bf16": "" @@ -3638,19 +4066,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.41, - "fp16": "", - "fp32": 3.19, - "bf16": "" + "int8": 164.18, + "fp16": 107.12, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3672,19 +4104,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.01, - "fp16": "", - "fp32": 4.49, - "bf16": "" + "int8": 195.27, + "fp16": 164.33, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3706,19 +4142,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 22.64, + "int8": "", "fp16": "", - "fp32": 12.18, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 8.94, + "token_int8": "", + "token_fp16": 0.94 } ], "Unit": "FPS", @@ -3727,9 +4167,9 @@ "latency": { "Precisions": [ { - "int4": "", + "int4": 111.74, "int8": "", - "fp16": "", + "fp16": 1056.4, "fp32": "", "bf16": "" } @@ -3740,19 +4180,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.39, + "int8": "", "fp16": "", - "fp32": 6.33, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 10.82, + "token_int8": 6.3, + "token_fp16": 1.1 } ], "Unit": "FPS", @@ -3761,9 +4205,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 92.41, + "int8": 158.68, + "fp16": 906.89, "fp32": "", "bf16": "" } @@ -3774,19 +4218,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 14.28, + "int8": "", "fp16": "", - "fp32": 11.21, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 14.62, + "token_int8": 8.53, + "token_fp16": "" } ], "Unit": "FPS", @@ -3795,8 +4243,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 71.92, + "int4": 68.39, + "int8": 117.1, "fp16": "", "fp32": "", "bf16": "" @@ -3808,19 +4256,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.25, + "int8": "", "fp16": "", - "fp32": 4.96, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 14.82, + "token_int8": 7.84, + "token_fp16": 4.04 } ], "Unit": "FPS", @@ -3829,9 +4281,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 139.91, - "fp16": "", + "int4": 67.44, + "int8": 127.51, + "fp16": 247.29, "fp32": "", "bf16": "" } @@ -3842,19 +4294,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 18.99, + "int8": "", "fp16": "", - "fp32": 11.5, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.17, + "token_int8": 20.38, + "token_fp16": 10.76 } ], "Unit": "FPS", @@ -3863,9 +4319,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 55.68, - "fp16": "", + "int4": 38.21, + "int8": 49.06, + "fp16": 92.92, "fp32": "", "bf16": "" } @@ -3876,19 +4332,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.48, - "fp16": "", - "fp32": 27.9, - "bf16": "" + "int8": 2.35, + "fp16": 1.58, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3898,7 +4358,7 @@ "Precisions": [ { "int4": "", - "int8": 27.49, + "int8": 421.72, "fp16": "", "fp32": "", "bf16": "" @@ -3910,19 +4370,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 59.12, + "int8": "", "fp16": "", - "fp32": 32.99, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 15.03, + "token_int8": 8.94, + "token_fp16": "" } ], "Unit": "FPS", @@ -3931,8 +4395,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 20.56, + "int4": 66.52, + "int8": 111.8, "fp16": "", "fp32": "", "bf16": "" @@ -3944,19 +4408,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 44.97, - "fp16": "", - "fp32": 24.31, - "bf16": "" + "int8": 1293.98, + "fp16": 1371.59, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3966,7 +4434,7 @@ "Precisions": [ { "int4": "", - "int8": 32.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3978,19 +4446,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 56.87, + "int8": "", "fp16": "", - "fp32": 36.06, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 23.61, + "token_int8": 18.01, + "token_fp16": 9.36 } ], "Unit": "FPS", @@ -3999,9 +4471,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 28.54, - "fp16": "", + "int4": 42.34, + "int8": 55.51, + "fp16": 106.82, "fp32": "", "bf16": "" } @@ -4012,19 +4484,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 152.17, + "int8": "", "fp16": "", - "fp32": 93.19, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 16.68, + "token_int8": 9.5, + "token_fp16": "" } ], "Unit": "FPS", @@ -4033,8 +4509,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 9.14, + "int4": 59.95, + "int8": 105.26, "fp16": "", "fp32": "", "bf16": "" @@ -4046,19 +4522,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 59.7, - "fp16": "", - "fp32": 42.45, - "bf16": "" + "int8": 563.96, + "fp16": 416.13, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4068,7 +4548,7 @@ "Precisions": [ { "int4": "", - "int8": 17.62, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4080,19 +4560,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 75.02, - "fp16": "", - "fp32": 41.47, - "bf16": "" + "int8": 21.26, + "fp16": 12.84, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4102,7 +4586,7 @@ "Precisions": [ { "int4": "", - "int8": 14.65, + "int8": 47.61, "fp16": "", "fp32": "", "bf16": "" @@ -4114,19 +4598,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.75, - "fp16": "", - "fp32": 21.83, - "bf16": "" + "int8": 1030.66, + "fp16": 811.13, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4136,7 +4624,7 @@ "Precisions": [ { "int4": "", - "int8": 20.19, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4148,19 +4636,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 112.1, - "fp16": "", - "fp32": 61.82, - "bf16": "" + "int8": 403.44, + "fp16": 306.22, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4170,7 +4662,7 @@ "Precisions": [ { "int4": "", - "int8": 11.47, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4182,19 +4674,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 62.08, + "int8": 223.99, "fp16": "", - "fp32": 39.8, - "bf16": "" + "fp32": 189.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4204,7 +4700,7 @@ "Precisions": [ { "int4": "", - "int8": 26.84, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4216,19 +4712,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 83.44, + "int8": 174.87, "fp16": "", - "fp32": 43.76, - "bf16": "" + "fp32": 149.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4238,7 +4738,7 @@ "Precisions": [ { "int4": "", - "int8": 16.75, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4250,19 +4750,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.39, + "int8": 7.24, "fp16": "", - "fp32": 38.12, - "bf16": "" + "fp32": 3.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4272,7 +4776,7 @@ "Precisions": [ { "int4": "", - "int8": 17.79, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4284,19 +4788,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 65.23, + "int8": 4846.91, "fp16": "", - "fp32": 41.09, - "bf16": "" + "fp32": 2888.98, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4306,7 +4814,7 @@ "Precisions": [ { "int4": "", - "int8": 18.79, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4318,19 +4826,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 220.01, + "int8": 1975.45, "fp16": "", - "fp32": 126.72, - "bf16": "" + "fp32": 922.35, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4340,7 +4852,7 @@ "Precisions": [ { "int4": "", - "int8": 7.33, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4352,19 +4864,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 15.45, + "int8": "", "fp16": "", - "fp32": 12.76, - "bf16": "" + "fp32": 20.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4374,7 +4890,7 @@ "Precisions": [ { "int4": "", - "int8": 66.3, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4386,19 +4902,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 97.27, + "int8": "", "fp16": "", - "fp32": 48.02, - "bf16": "" + "fp32": 585.46, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4408,7 +4928,7 @@ "Precisions": [ { "int4": "", - "int8": 13.26, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4420,19 +4940,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 35.92, + "int8": 343.07, "fp16": "", - "fp32": 30.28, - "bf16": "" + "fp32": 274.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4442,7 +4966,7 @@ "Precisions": [ { "int4": "", - "int8": 28.54, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4454,19 +4978,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 271.61, + "int8": 44.06, "fp16": "", - "fp32": 166.53, - "bf16": "" + "fp32": 16.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4476,7 +5004,7 @@ "Precisions": [ { "int4": "", - "int8": 11.3, + "int8": 41.27, "fp16": "", "fp32": "", "bf16": "" @@ -4488,19 +5016,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 576.89, + "int8": 53.32, "fp16": "", - "fp32": 325.0, - "bf16": "" + "fp32": 38.06, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4510,7 +5042,7 @@ "Precisions": [ { "int4": "", - "int8": 7.03, + "int8": 28.44, "fp16": "", "fp32": "", "bf16": "" @@ -4522,19 +5054,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1014.6, + "int8": 0.65, "fp16": "", - "fp32": 582.63, - "bf16": "" + "fp32": 0.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4544,7 +5080,7 @@ "Precisions": [ { "int4": "", - "int8": 4.2, + "int8": 2598.78, "fp16": "", "fp32": "", "bf16": "" @@ -4556,19 +5092,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1454.4, + "int8": 917.84, "fp16": "", - "fp32": 872.52, - "bf16": 1037.91 + "fp32": 490.87, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4578,10 +5118,10 @@ "Precisions": [ { "int4": "", - "int8": 4.6, + "int8": 2.07, "fp16": "", "fp32": "", - "bf16": 4.96 + "bf16": "" } ], "Unit": "ms", @@ -4590,19 +5130,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1736.65, + "int8": 194.09, "fp16": "", - "fp32": 1132.21, - "bf16": 1408.76 + "fp32": 52.09, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4612,10 +5156,10 @@ "Precisions": [ { "int4": "", - "int8": 4.6, + "int8": 9.58, "fp16": "", "fp32": "", - "bf16": 4.68 + "bf16": "" } ], "Unit": "ms", @@ -4624,19 +5168,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 413.78, + "int8": 3.52, "fp16": "", - "fp32": 257.07, - "bf16": "" + "fp32": 0.87, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4646,7 +5194,7 @@ "Precisions": [ { "int4": "", - "int8": 8.54, + "int8": 493.86, "fp16": "", "fp32": "", "bf16": "" @@ -4658,19 +5206,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 497.16, + "int8": 380.37, "fp16": "", - "fp32": 303.25, - "bf16": "" + "fp32": 135.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4680,7 +5232,7 @@ "Precisions": [ { "int4": "", - "int8": 5.86, + "int8": 4.64, "fp16": "", "fp32": "", "bf16": "" @@ -4692,19 +5244,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 97.68, + "int8": 80.52, "fp16": "", - "fp32": 67.18, - "bf16": "" + "fp32": 34.88, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4714,7 +5270,7 @@ "Precisions": [ { "int4": "", - "int8": 21.46, + "int8": 20.34, "fp16": "", "fp32": "", "bf16": "" @@ -4726,19 +5282,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 70.93, - "fp16": "", - "fp32": 46.39, - "bf16": "" + "int8": 265.97, + "fp16": 198.16, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4748,7 +5308,7 @@ "Precisions": [ { "int4": "", - "int8": 25.46, + "int8": 5.25, "fp16": "", "fp32": "", "bf16": "" @@ -4760,19 +5320,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 92.57, - "fp16": "", - "fp32": 61.6, - "bf16": "" + "int8": 13.69, + "fp16": 13.65, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4782,7 +5346,7 @@ "Precisions": [ { "int4": "", - "int8": 13.58, + "int8": 119.56, "fp16": "", "fp32": "", "bf16": "" @@ -4794,19 +5358,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 456.95, - "fp16": 402.8, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": 0.24, + "token_fp16": 4.4 } ], "Unit": "FPS", @@ -4816,8 +5384,8 @@ "Precisions": [ { "int4": "", - "int8": 2.93, - "fp16": "", + "int8": 4094.9, + "fp16": 226.87, "fp32": "", "bf16": "" } @@ -4828,19 +5396,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 3799.36, + "fp16": 3178.95, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4850,7 +5422,7 @@ "Precisions": [ { "int4": "", - "int8": 4.8, + "int8": 0.46, "fp16": "", "fp32": "", "bf16": "" @@ -4862,19 +5434,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 147.83, - "fp16": 120.48, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 3.31, + "token_int8": 0.72, + "token_fp16": 6.86 } ], "Unit": "FPS", @@ -4883,9 +5459,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 9.67, - "fp16": "", + "int4": 301.49, + "int8": 1378.29, + "fp16": 145.76, "fp32": "", "bf16": "" } @@ -4896,19 +5472,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.71, - "fp16": 34.85, + "int8": 2161.26, + "fp16": 948.32, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4918,7 +5498,7 @@ "Precisions": [ { "int4": "", - "int8": 27.94, + "int8": 0.79, "fp16": "", "fp32": "", "bf16": "" @@ -4930,19 +5510,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.99, - "fp16": 34.96, + "int8": 230.18, + "fp16": 192.78, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4952,7 +5536,7 @@ "Precisions": [ { "int4": "", - "int8": 27.51, + "int8": 8.29, "fp16": "", "fp32": "", "bf16": "" @@ -4964,19 +5548,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 13.38, - "fp16": 13.66, + "int8": 401.12, + "fp16": 497.56, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4986,7 +5574,7 @@ "Precisions": [ { "int4": "", - "int8": 124.82, + "int8": 3.97, "fp16": "", "fp32": "", "bf16": "" @@ -4998,19 +5586,23 @@ } }, { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 40.02, - "fp16": 34.39, + "int8": 225.83, + "fp16": 298.39, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5020,7 +5612,7 @@ "Precisions": [ { "int4": "", - "int8": 34.08, + "int8": 3.93, "fp16": "", "fp32": "", "bf16": "" @@ -5032,19 +5624,23 @@ } }, { - "Platform": "Intel® Atom® X6425E iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.7, - "fp16": 25.09, + "int8": 114.57, + "fp16": 121.87, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5054,7 +5650,7 @@ "Precisions": [ { "int4": "", - "int8": 63.02, + "int8": 10.22, "fp16": "", "fp32": "", "bf16": "" @@ -5066,19 +5662,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 73.6, - "fp16": 58.55, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 14.49, + "token_int8": 8.34, + "token_fp16": 0.59 } ], "Unit": "FPS", @@ -5087,9 +5687,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 24.96, - "fp16": "", + "int4": 68.99, + "int8": 119.77, + "fp16": 1691.52, "fp32": "", "bf16": "" } @@ -5100,19 +5700,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 68.83, - "fp16": 51.74, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 17.63, + "token_int8": 9.8, + "token_fp16": 0.71 } ], "Unit": "FPS", @@ -5121,9 +5725,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 19.37, - "fp16": "", + "int4": 56.72, + "int8": 102.04, + "fp16": 1402.74, "fp32": "", "bf16": "" } @@ -5134,19 +5738,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 80.63, - "fp16": 60.04, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 18.48, + "token_int8": 11.87, + "token_fp16": 6.44 } ], "Unit": "FPS", @@ -5155,9 +5763,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 14.48, - "fp16": "", + "int4": 54.09, + "int8": 84.18, + "fp16": 155.17, "fp32": "", "bf16": "" } @@ -5168,19 +5776,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 91.41, + "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 20.41, + "token_int8": 11.07, + "token_fp16": 5.81 } ], "Unit": "FPS", @@ -5189,9 +5801,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 48.98, + "int8": 90.29, + "fp16": 171.98, "fp32": "", "bf16": "" } @@ -5202,19 +5814,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 56.47, - "fp16": 42.15, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 36.58, + "token_int8": 23.94, + "token_fp16": 12.86 } ], "Unit": "FPS", @@ -5223,9 +5839,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 24.14, - "fp16": "", + "int4": 27.33, + "int8": 41.77, + "fp16": 77.71, "fp32": "", "bf16": "" } @@ -5236,19 +5852,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 128.8, - "fp16": 97.86, + "int8": 10.4, + "fp16": 5.7, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5258,7 +5878,7 @@ "Precisions": [ { "int4": "", - "int8": 12.63, + "int8": 109.21, "fp16": "", "fp32": "", "bf16": "" @@ -5270,19 +5890,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 100.05, - "fp16": 74.24, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 20.06, + "token_int8": 11.6, + "token_fp16": 6.05 } ], "Unit": "FPS", @@ -5291,9 +5915,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 13.33, - "fp16": "", + "int4": 49.85, + "int8": 86.18, + "fp16": 165.15, "fp32": "", "bf16": "" } @@ -5304,19 +5928,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 1007.75, + "fp16": 862.8, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5326,7 +5954,7 @@ "Precisions": [ { "int4": "", - "int8": 14.0, + "int8": 1.2, "fp16": "", "fp32": "", "bf16": "" @@ -5338,19 +5966,23 @@ } }, { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 33.67, - "fp16": 30.88, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 31.27, + "token_int8": 20.55, + "token_fp16": 11.04 } ], "Unit": "FPS", @@ -5359,9 +5991,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 38.07, - "fp16": "", + "int4": 31.97, + "int8": 48.66, + "fp16": 90.57, "fp32": "", "bf16": "" } @@ -5372,19 +6004,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 201.37, - "fp16": 162.08, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 20.99, + "token_int8": 12.69, + "token_fp16": 6.07 } ], "Unit": "FPS", @@ -5393,9 +6029,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 6.86, - "fp16": "", + "int4": 47.64, + "int8": 78.78, + "fp16": 164.54, "fp32": "", "bf16": "" } @@ -5406,19 +6042,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 169.74, - "fp16": 143.49, + "int8": 830.46, + "fp16": 585.38, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5428,7 +6068,7 @@ "Precisions": [ { "int4": "", - "int8": 7.91, + "int8": 1.23, "fp16": "", "fp32": "", "bf16": "" @@ -5440,19 +6080,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 163.17, - "fp16": 185.29, + "int8": 57.99, + "fp16": 32.18, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5462,7 +6106,7 @@ "Precisions": [ { "int4": "", - "int8": 8.23, + "int8": 26.21, "fp16": "", "fp32": "", "bf16": "" @@ -5474,19 +6118,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39.25, - "fp16": "", - "fp32": 29.57, - "bf16": "" + "int8": 485.85, + "fp16": 555.71, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5496,7 +6144,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.75, "fp16": "", "fp32": "", "bf16": "" @@ -5508,19 +6156,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 22.93, - "fp16": "", - "fp32": 23.89, - "bf16": "" + "int8": 362.75, + "fp16": 375.06, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5530,7 +6182,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.3, "fp16": "", "fp32": "", "bf16": "" @@ -5542,19 +6194,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 57.64, + "int8": 34.21, "fp16": "", - "fp32": 48.78, - "bf16": "" + "fp32": 15.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5578,17 +6234,21 @@ { "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 47.95, "fp16": "", - "fp32": 30.78, - "bf16": "" + "fp32": 29.38, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5610,19 +6270,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 74.05, + "int8": 0.5, "fp16": "", - "fp32": 48.82, - "bf16": "" + "fp32": 0.18, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5644,19 +6308,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 100.6, + "int8": 742.67, "fp16": "", - "fp32": 65.57, - "bf16": "" + "fp32": 331.98, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5678,19 +6346,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 57.55, + "int8": 162.84, "fp16": "", - "fp32": 28.28, - "bf16": "" + "fp32": 51.66, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5712,10 +6384,11 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ @@ -5723,8 +6396,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 75.36, - "bf16": "" + "fp32": 1.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5746,19 +6422,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 88.7, + "int8": 328.29, "fp16": "", - "fp32": 59.09, - "bf16": "" + "fp32": 115.41, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5780,10 +6460,11 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ @@ -5791,8 +6472,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 59.29, - "bf16": "" + "fp32": 41.68, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5814,19 +6498,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.02, + "int8": 79.4, "fp16": "", - "fp32": 28.15, - "bf16": "" + "fp32": 35.44, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5848,19 +6536,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 191.27, + "int8": 31.55, "fp16": "", - "fp32": 150.88, - "bf16": "" + "fp32": 12.38, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5870,7 +6562,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 46.55, "fp16": "", "fp32": "", "bf16": "" @@ -5882,19 +6574,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 123.24, + "int8": 43.39, "fp16": "", - "fp32": 97.13, - "bf16": "" + "fp32": 23.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5904,7 +6600,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 33.09, "fp16": "", "fp32": "", "bf16": "" @@ -5916,19 +6612,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.17, + "int8": 0.45, "fp16": "", - "fp32": 0.04, - "bf16": "" + "fp32": 0.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5938,7 +6638,7 @@ "Precisions": [ { "int4": "", - "int8": 5769.81, + "int8": 2440.72, "fp16": "", "fp32": "", "bf16": "" @@ -5950,19 +6650,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.3, + "int8": 789.02, "fp16": "", - "fp32": 0.14, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5972,7 +6676,7 @@ "Precisions": [ { "int4": "", - "int8": 3268.87, + "int8": 1.9, "fp16": "", "fp32": "", "bf16": "" @@ -5984,19 +6688,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.44, + "int8": 147.74, "fp16": "", - "fp32": 0.18, - "bf16": "" + "fp32": 38.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6006,7 +6714,7 @@ "Precisions": [ { "int4": "", - "int8": 2406.55, + "int8": 11.4, "fp16": "", "fp32": "", "bf16": "" @@ -6019,18 +6727,22 @@ }, { "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.47, + "int8": 2.66, "fp16": "", - "fp32": 0.12, - "bf16": "" + "fp32": 0.77, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6040,7 +6752,7 @@ "Precisions": [ { "int4": "", - "int8": 2337.51, + "int8": 511.09, "fp16": "", "fp32": "", "bf16": "" @@ -6052,19 +6764,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.58, + "int8": 313.17, "fp16": "", - "fp32": 0.16, - "bf16": "" + "fp32": 95.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6074,7 +6790,7 @@ "Precisions": [ { "int4": "", - "int8": 2064.41, + "int8": 4.81, "fp16": "", "fp32": "", "bf16": "" @@ -6086,19 +6802,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.71, + "int8": "", "fp16": "", - "fp32": 0.5, - "bf16": "" + "fp32": 31.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6108,7 +6828,7 @@ "Precisions": [ { "int4": "", - "int8": 708.93, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6120,19 +6840,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.48, + "int8": 67.43, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 26.68, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6142,7 +6866,7 @@ "Precisions": [ { "int4": "", - "int8": 2022.46, + "int8": 20.62, "fp16": "", "fp32": "", "bf16": "" @@ -6154,19 +6878,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.72, - "fp16": "", - "fp32": 0.19, - "bf16": "" + "int8": 46.15, + "fp16": 38.3, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6176,7 +6904,7 @@ "Precisions": [ { "int4": "", - "int8": 1351.31, + "int8": 19.82, "fp16": "", "fp32": "", "bf16": "" @@ -6188,19 +6916,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.52, - "fp16": "", - "fp32": 0.14, - "bf16": "" + "int8": 64.24, + "fp16": 50.43, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6210,7 +6942,7 @@ "Precisions": [ { "int4": "", - "int8": 1804.22, + "int8": 20.17, "fp16": "", "fp32": "", "bf16": "" @@ -6222,19 +6954,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.24, - "fp16": "", - "fp32": 0.35, - "bf16": "" + "int8": 0.5, + "fp16": 0.51, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6244,7 +6980,7 @@ "Precisions": [ { "int4": "", - "int8": 936.49, + "int8": 1499.27, "fp16": "", "fp32": "", "bf16": "" @@ -6256,19 +6992,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.64, - "fp16": "", - "fp32": 0.18, - "bf16": "" + "int8": 768.31, + "fp16": 485.7, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6278,7 +7018,7 @@ "Precisions": [ { "int4": "", - "int8": 1922.82, + "int8": 1.7, "fp16": "", "fp32": "", "bf16": "" @@ -6290,19 +7030,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.91, - "fp16": "", + "int8": 208.55, + "fp16": 117.84, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6312,7 +7056,7 @@ "Precisions": [ { "int4": "", - "int8": 1366.6, + "int8": 5.0, "fp16": "", "fp32": "", "bf16": "" @@ -6324,19 +7068,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.37, - "fp16": "", - "fp32": 0.15, - "bf16": "" + "int8": 5.64, + "fp16": 2.72, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6346,7 +7094,7 @@ "Precisions": [ { "int4": "", - "int8": 2090.62, + "int8": 172.69, "fp16": "", "fp32": "", "bf16": "" @@ -6358,19 +7106,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.47, - "fp16": "", - "fp32": 0.17, - "bf16": "" + "int8": 382.92, + "fp16": 223.39, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6380,7 +7132,7 @@ "Precisions": [ { "int4": "", - "int8": 2235.88, + "int8": 3.11, "fp16": "", "fp32": "", "bf16": "" @@ -6392,19 +7144,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.48, - "fp16": "", - "fp32": 0.71, - "bf16": "" + "int8": 126.83, + "fp16": 77.91, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6414,7 +7170,7 @@ "Precisions": [ { "int4": "", - "int8": 558.42, + "int8": 8.1, "fp16": "", "fp32": "", "bf16": "" @@ -6426,19 +7182,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.75, + "int8": 49.68, "fp16": "", - "fp32": 0.29, - "bf16": "" + "fp32": 26.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6448,7 +7208,7 @@ "Precisions": [ { "int4": "", - "int8": 1441.19, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6460,19 +7220,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.29, + "int8": 73.94, "fp16": "", - "fp32": 0.15, - "bf16": "" + "fp32": 48.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6482,7 +7246,7 @@ "Precisions": [ { "int4": "", - "int8": 3400.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6494,19 +7258,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.25, + "int8": 0.69, "fp16": "", - "fp32": 0.89, - "bf16": "" + "fp32": 0.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6516,7 +7284,7 @@ "Precisions": [ { "int4": "", - "int8": 650.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6528,19 +7296,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.7, + "int8": 1050.26, "fp16": "", - "fp32": 2.25, - "bf16": "" + "fp32": 535.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6550,7 +7322,7 @@ "Precisions": [ { "int4": "", - "int8": 251.14, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6562,19 +7334,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 14.54, + "int8": 234.19, "fp16": "", - "fp32": 3.46, - "bf16": "" + "fp32": 87.89, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6584,7 +7360,7 @@ "Precisions": [ { "int4": "", - "int8": 160.14, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6596,19 +7372,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 63.19, + "int8": 4.74, "fp16": "", - "fp32": 5.23, - "bf16": 37.95 + "fp32": 1.74, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6618,10 +7398,10 @@ "Precisions": [ { "int4": "", - "int8": 60.21, + "int8": "", "fp16": "", "fp32": "", - "bf16": 83.04 + "bf16": "" } ], "Unit": "ms", @@ -6630,19 +7410,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 75.35, + "int8": 466.65, "fp16": "", - "fp32": 6.47, - "bf16": 48.43 + "fp32": 188.83, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6652,10 +7436,10 @@ "Precisions": [ { "int4": "", - "int8": 55.97, + "int8": "", "fp16": "", "fp32": "", - "bf16": 73.23 + "bf16": "" } ], "Unit": "ms", @@ -6664,19 +7448,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.51, + "int8": "", "fp16": "", - "fp32": 1.65, - "bf16": "" + "fp32": 65.34, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6686,7 +7474,7 @@ "Precisions": [ { "int4": "", - "int8": 322.06, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6698,19 +7486,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.4, + "int8": 125.18, "fp16": "", - "fp32": 1.76, - "bf16": "" + "fp32": 58.13, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6720,7 +7512,7 @@ "Precisions": [ { "int4": "", - "int8": 286.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6732,19 +7524,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.18, + "int8": 39.97, "fp16": "", - "fp32": 0.32, - "bf16": "" + "fp32": 15.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6754,7 +7550,7 @@ "Precisions": [ { "int4": "", - "int8": 999.99, + "int8": 40.14, "fp16": "", "fp32": "", "bf16": "" @@ -6766,19 +7562,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.9, + "int8": 56.15, "fp16": "", - "fp32": 0.21, - "bf16": "" + "fp32": 35.76, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6788,7 +7588,7 @@ "Precisions": [ { "int4": "", - "int8": 1330.81, + "int8": 28.73, "fp16": "", "fp32": "", "bf16": "" @@ -6800,19 +7600,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.12, + "int8": 0.57, "fp16": "", - "fp32": 0.27, - "bf16": "" + "fp32": 0.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6822,7 +7626,7 @@ "Precisions": [ { "int4": "", - "int8": 1255.21, + "int8": 2069.28, "fp16": "", "fp32": "", "bf16": "" @@ -6834,19 +7638,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39.0, - "fp16": 21.24, - "fp32": "", - "bf16": "" + "int8": 951.93, + "fp16": "", + "fp32": 463.06, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6856,7 +7664,7 @@ "Precisions": [ { "int4": "", - "int8": 52.08, + "int8": 1.74, "fp16": "", "fp32": "", "bf16": "" @@ -6868,19 +7676,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 184.54, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 52.88, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6890,7 +7702,7 @@ "Precisions": [ { "int4": "", - "int8": 48.01, + "int8": 9.61, "fp16": "", "fp32": "", "bf16": "" @@ -6902,19 +7714,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.33, + "int8": 3.16, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 0.92, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6924,7 +7740,7 @@ "Precisions": [ { "int4": "", - "int8": 179.63, + "int8": 466.34, "fp16": "", "fp32": "", "bf16": "" @@ -6936,19 +7752,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.48, - "fp16": 0.52, - "fp32": "", - "bf16": "" + "int8": 383.62, + "fp16": "", + "fp32": 134.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6958,7 +7778,7 @@ "Precisions": [ { "int4": "", - "int8": 2109.61, + "int8": 4.16, "fp16": "", "fp32": "", "bf16": "" @@ -6970,19 +7790,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.53, - "fp16": 0.52, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 43.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6992,7 +7816,7 @@ "Precisions": [ { "int4": "", - "int8": 1494.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7004,19 +7828,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.54, - "fp16": 0.58, - "fp32": "", - "bf16": "" + "int8": 91.3, + "fp16": "", + "fp32": 36.39, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7026,7 +7854,7 @@ "Precisions": [ { "int4": "", - "int8": 1486.87, + "int8": 18.15, "fp16": "", "fp32": "", "bf16": "" @@ -7038,19 +7866,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.83, - "fp16": 0.89, + "int8": 47.17, + "fp16": 39.79, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7060,7 +7892,7 @@ "Precisions": [ { "int4": "", - "int8": 1111.35, + "int8": 18.45, "fp16": "", "fp32": "", "bf16": "" @@ -7072,19 +7904,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.57, - "fp16": 0.55, + "int8": 80.6, + "fp16": 59.92, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7094,7 +7930,7 @@ "Precisions": [ { "int4": "", - "int8": 1643.33, + "int8": 14.61, "fp16": "", "fp32": "", "bf16": "" @@ -7106,19 +7942,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.03, - "fp16": 1.14, + "int8": 0.52, + "fp16": 0.58, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7128,7 +7968,7 @@ "Precisions": [ { "int4": "", - "int8": 973.29, + "int8": 1506.76, "fp16": "", "fp32": "", "bf16": "" @@ -7140,19 +7980,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.73, - "fp16": 0.77, + "int8": 778.4, + "fp16": 509.56, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7162,7 +8006,7 @@ "Precisions": [ { "int4": "", - "int8": 1184.14, + "int8": 1.48, "fp16": "", "fp32": "", "bf16": "" @@ -7174,19 +8018,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 225.12, + "fp16": 127.27, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7196,7 +8044,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 4.31, "fp16": "", "fp32": "", "bf16": "" @@ -7208,19 +8056,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.97, - "fp16": 1.64, + "int8": 5.79, + "fp16": 2.86, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7230,7 +8082,7 @@ "Precisions": [ { "int4": "", - "int8": 481.56, + "int8": 144.71, "fp16": "", "fp32": "", "bf16": "" @@ -7242,19 +8094,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 404.76, + "fp16": 237.61, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7264,7 +8120,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.75, "fp16": "", "fp32": "", "bf16": "" @@ -7276,19 +8132,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.69, - "fp16": 7.44, + "int8": 131.89, + "fp16": 83.17, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7298,7 +8158,7 @@ "Precisions": [ { "int4": "", - "int8": 91.66, + "int8": 7.11, "fp16": "", "fp32": "", "bf16": "" @@ -7310,19 +8170,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.56, + "int8": 120.44, "fp16": "", - "fp32": 0.51, - "bf16": "" + "fp32": 47.21, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7332,7 +8196,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 13.32, "fp16": "", "fp32": "", "bf16": "" @@ -7344,19 +8208,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 148.91, "fp16": "", - "fp32": 0.19, - "bf16": "" + "fp32": 93.08, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7366,7 +8234,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 9.22, "fp16": "", "fp32": "", "bf16": "" @@ -7378,19 +8246,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.7, + "int8": "", "fp16": "", - "fp32": 0.31, - "bf16": "" + "fp32": 0.49, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7400,7 +8272,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 733.91, "fp16": "", "fp32": "", "bf16": "" @@ -7412,19 +8284,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.17, + "int8": 2974.41, "fp16": "", - "fp32": 0.65, - "bf16": "" + "fp32": 1317.04, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7434,7 +8310,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.69, "fp16": "", "fp32": "", "bf16": "" @@ -7446,19 +8322,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.67, + "int8": 537.98, "fp16": "", - "fp32": 0.33, - "bf16": "" + "fp32": 148.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7468,7 +8348,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.82, "fp16": "", "fp32": "", "bf16": "" @@ -7480,19 +8360,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.8, "fp16": "", - "fp32": 0.71, - "bf16": "" + "fp32": 2.47, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7502,7 +8386,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 133.73, "fp16": "", "fp32": "", "bf16": "" @@ -7514,19 +8398,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.81, + "int8": 1068.19, "fp16": "", - "fp32": 0.43, - "bf16": "" + "fp32": 379.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7536,7 +8424,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.33, "fp16": "", "fp32": "", "bf16": "" @@ -7548,10 +8436,11 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ @@ -7559,8 +8448,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 0.46, - "bf16": "" + "fp32": 122.62, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7582,19 +8474,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i5-13600K CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.37, + "int8": 266.57, "fp16": "", - "fp32": 1.2, - "bf16": "" + "fp32": 102.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7604,7 +8500,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 5.27, "fp16": "", "fp32": "", "bf16": "" @@ -7616,19 +8512,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 84.71, "fp16": "", - "fp32": 0.58, - "bf16": "" + "fp32": 51.06, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7650,19 +8550,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 274.01, + "int8": 98.02, "fp16": "", - "fp32": 168.87, - "bf16": "" + "fp32": 65.51, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7672,7 +8576,7 @@ "Precisions": [ { "int4": "", - "int8": 4.07, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7684,19 +8588,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 133.76, + "int8": 1.16, "fp16": "", - "fp32": 80.44, - "bf16": "" + "fp32": 0.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7706,7 +8614,7 @@ "Precisions": [ { "int4": "", - "int8": 7.82, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7718,19 +8626,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 300.95, + "int8": 1353.32, "fp16": "", - "fp32": 133.55, - "bf16": "" + "fp32": 683.15, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7740,7 +8652,7 @@ "Precisions": [ { "int4": "", - "int8": 3.62, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7752,19 +8664,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 534.91, + "int8": 365.63, "fp16": "", - "fp32": 406.53, - "bf16": "" + "fp32": 164.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7774,7 +8690,7 @@ "Precisions": [ { "int4": "", - "int8": 2.01, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7786,19 +8702,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 893.76, + "int8": 8.65, "fp16": "", - "fp32": 462.2, - "bf16": "" + "fp32": 3.77, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7808,7 +8728,7 @@ "Precisions": [ { "int4": "", - "int8": 1.62, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7820,19 +8740,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 826.85, + "int8": 657.26, "fp16": "", - "fp32": 315.6, - "bf16": "" + "fp32": 293.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7842,7 +8766,7 @@ "Precisions": [ { "int4": "", - "int8": 1.9, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7854,10 +8778,11 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ @@ -7865,8 +8790,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 462.14, - "bf16": "" + "fp32": 107.24, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7876,7 +8804,7 @@ "Precisions": [ { "int4": "", - "int8": 1.72, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7888,19 +8816,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2989.92, + "int8": 182.9, "fp16": "", - "fp32": 1328.99, - "bf16": "" + "fp32": 101.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7910,7 +8842,7 @@ "Precisions": [ { "int4": "", - "int8": 0.69, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7922,19 +8854,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 848.04, + "int8": 50.21, "fp16": "", - "fp32": 626.94, - "bf16": "" + "fp32": 18.33, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7944,7 +8880,7 @@ "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 22.66, "fp16": "", "fp32": "", "bf16": "" @@ -7957,18 +8893,22 @@ }, { "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1284.93, + "int8": 71.27, "fp16": "", - "fp32": 509.36, - "bf16": "" + "fp32": 41.39, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7978,7 +8918,7 @@ "Precisions": [ { "int4": "", - "int8": 0.96, + "int8": 14.62, "fp16": "", "fp32": "", "bf16": "" @@ -7990,19 +8930,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 975.3, + "int8": 0.71, "fp16": "", - "fp32": 312.11, - "bf16": "" + "fp32": 0.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8012,7 +8956,7 @@ "Precisions": [ { "int4": "", - "int8": 1.21, + "int8": 1361.21, "fp16": "", "fp32": "", "bf16": "" @@ -8024,19 +8968,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1966.6, + "int8": 1291.06, "fp16": "", - "fp32": 958.97, - "bf16": "" + "fp32": 507.09, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8046,7 +8994,7 @@ "Precisions": [ { "int4": "", - "int8": 0.96, + "int8": 0.95, "fp16": "", "fp32": "", "bf16": "" @@ -8058,19 +9006,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1039.53, + "int8": 224.68, "fp16": "", - "fp32": 513.13, - "bf16": "" + "fp32": 60.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8080,7 +9032,7 @@ "Precisions": [ { "int4": "", - "int8": 1.61, + "int8": 4.95, "fp16": "", "fp32": "", "bf16": "" @@ -8092,19 +9044,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.84, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1.01, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8114,7 +9070,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 250.45, "fp16": "", "fp32": "", "bf16": "" @@ -8126,19 +9082,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 737.89, + "int8": 491.99, "fp16": "", - "fp32": 488.9, - "bf16": "" + "fp32": 146.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8148,7 +9108,7 @@ "Precisions": [ { "int4": "", - "int8": 1.44, + "int8": 2.2, "fp16": "", "fp32": "", "bf16": "" @@ -8160,19 +9120,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 904.6, + "int8": "", "fp16": "", - "fp32": 570.78, - "bf16": "" + "fp32": 48.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8182,7 +9146,7 @@ "Precisions": [ { "int4": "", - "int8": 1.56, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8194,19 +9158,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4254.47, + "int8": 106.45, "fp16": "", - "fp32": 2047.25, - "bf16": "" + "fp32": 40.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8216,7 +9184,7 @@ "Precisions": [ { "int4": "", - "int8": 0.6, + "int8": 10.2, "fp16": "", "fp32": "", "bf16": "" @@ -8228,19 +9196,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 296.47, - "fp16": "", - "fp32": 183.87, - "bf16": "" + "int8": 68.4, + "fp16": 53.22, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8250,7 +9222,7 @@ "Precisions": [ { "int4": "", - "int8": 3.81, + "int8": 17.09, "fp16": "", "fp32": "", "bf16": "" @@ -8262,19 +9234,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1451.27, - "fp16": "", - "fp32": 666.39, - "bf16": "" + "int8": 91.46, + "fp16": 72.22, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8284,7 +9260,7 @@ "Precisions": [ { "int4": "", - "int8": 1.2, + "int8": 17.92, "fp16": "", "fp32": "", "bf16": "" @@ -8296,19 +9272,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 519.78, - "fp16": "", - "fp32": 425.23, - "bf16": "" + "int8": 0.82, + "fp16": 0.88, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8318,7 +9298,7 @@ "Precisions": [ { "int4": "", - "int8": 2.07, + "int8": 1113.84, "fp16": "", "fp32": "", "bf16": "" @@ -8330,19 +9310,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5407.63, - "fp16": "", - "fp32": 1924.43, - "bf16": "" + "int8": 729.72, + "fp16": 569.2, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8352,7 +9336,7 @@ "Precisions": [ { "int4": "", - "int8": 1.51, + "int8": 2.05, "fp16": "", "fp32": "", "bf16": "" @@ -8364,19 +9348,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 15016.47, - "fp16": "", - "fp32": 4645.46, - "bf16": "" + "int8": 262.94, + "fp16": 174.98, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8386,7 +9374,7 @@ "Precisions": [ { "int4": "", - "int8": 0.94, + "int8": 4.82, "fp16": "", "fp32": "", "bf16": "" @@ -8398,19 +9386,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 22842.63, - "fp16": "", - "fp32": 7144.4, - "bf16": "" + "int8": 8.29, + "fp16": 4.67, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8420,7 +9412,7 @@ "Precisions": [ { "int4": "", - "int8": 0.58, + "int8": 118.28, "fp16": "", "fp32": "", "bf16": "" @@ -8432,19 +9424,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38642.1, - "fp16": "", - "fp32": 10319.56, - "bf16": 25708.49 + "int8": 447.59, + "fp16": 299.29, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8454,10 +9450,10 @@ "Precisions": [ { "int4": "", - "int8": 0.64, + "int8": 3.33, "fp16": "", "fp32": "", - "bf16": 0.65 + "bf16": "" } ], "Unit": "ms", @@ -8466,19 +9462,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39913.42, - "fp16": "", - "fp32": 15945.06, - "bf16": "" + "int8": 161.26, + "fp16": 111.45, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8488,10 +9488,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.1, "fp16": "", "fp32": "", - "bf16": 0.75 + "bf16": "" } ], "Unit": "ms", @@ -8500,19 +9500,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10312.71, + "int8": 50.01, "fp16": "", - "fp32": 3331.01, - "bf16": "" + "fp32": 25.82, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8522,7 +9526,7 @@ "Precisions": [ { "int4": "", - "int8": 1.24, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8534,19 +9538,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 11955.69, + "int8": 57.69, "fp16": "", - "fp32": 3563.39, - "bf16": "" + "fp32": 28.41, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8568,19 +9576,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1984.45, + "int8": 0.69, "fp16": "", - "fp32": 814.84, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8590,7 +9602,7 @@ "Precisions": [ { "int4": "", - "int8": 1.35, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8602,19 +9614,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1516.8, + "int8": 958.94, "fp16": "", - "fp32": 588.51, - "bf16": "" + "fp32": 350.53, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8624,7 +9640,7 @@ "Precisions": [ { "int4": "", - "int8": 1.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8636,19 +9652,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1651.84, + "int8": 230.4, "fp16": "", - "fp32": 775.78, - "bf16": "" + "fp32": 85.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8658,7 +9678,7 @@ "Precisions": [ { "int4": "", - "int8": 1.02, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8670,19 +9690,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2423.58, - "fp16": 2444.62, - "fp32": "", - "bf16": "" + "int8": 4.44, + "fp16": "", + "fp32": 1.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8692,7 +9716,7 @@ "Precisions": [ { "int4": "", - "int8": 0.64, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8704,19 +9728,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 456.16, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 162.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8726,7 +9754,7 @@ "Precisions": [ { "int4": "", - "int8": 0.99, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8738,19 +9766,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1862.03, - "fp16": 1564.35, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 55.98, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8760,7 +9792,7 @@ "Precisions": [ { "int4": "", - "int8": 0.78, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8772,19 +9804,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1956.69, - "fp16": 1343.46, - "fp32": "", - "bf16": "" + "int8": 103.63, + "fp16": "", + "fp32": 53.56, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8794,7 +9830,7 @@ "Precisions": [ { "int4": "", - "int8": 0.82, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8806,19 +9842,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1999.97, - "fp16": 1359.27, - "fp32": "", - "bf16": "" + "int8": 38.28, + "fp16": "", + "fp32": 13.87, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8828,7 +9868,7 @@ "Precisions": [ { "int4": "", - "int8": 0.74, + "int8": 28.41, "fp16": "", "fp32": "", "bf16": "" @@ -8840,19 +9880,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3751.95, - "fp16": 2877.38, - "fp32": "", - "bf16": "" + "int8": 53.34, + "fp16": "", + "fp32": 22.26, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8862,7 +9906,7 @@ "Precisions": [ { "int4": "", - "int8": 0.4, + "int8": 20.12, "fp16": "", "fp32": "", "bf16": "" @@ -8874,19 +9918,23 @@ } }, { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 415.36, - "fp16": 324.74, - "fp32": "", - "bf16": "" + "int8": 0.52, + "fp16": "", + "fp32": 0.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8896,7 +9944,7 @@ "Precisions": [ { "int4": "", - "int8": 3.52, + "int8": 1805.69, "fp16": "", "fp32": "", "bf16": "" @@ -8908,19 +9956,23 @@ } }, { - "Platform": "Intel® Atom® X6425E iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 188.4, - "fp16": 223.08, - "fp32": "", - "bf16": "" + "int8": 972.25, + "fp16": "", + "fp32": 311.82, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8930,7 +9982,7 @@ "Precisions": [ { "int4": "", - "int8": 7.66, + "int8": 1.2, "fp16": "", "fp32": "", "bf16": "" @@ -8942,19 +9994,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 673.52, - "fp16": 505.05, - "fp32": "", - "bf16": "" + "int8": 174.69, + "fp16": "", + "fp32": 45.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8964,7 +10020,7 @@ "Precisions": [ { "int4": "", - "int8": 2.74, + "int8": 6.4, "fp16": "", "fp32": "", "bf16": "" @@ -8976,19 +10032,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 779.69, - "fp16": 496.63, - "fp32": "", - "bf16": "" + "int8": 2.72, + "fp16": "", + "fp32": 0.78, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8998,7 +10058,7 @@ "Precisions": [ { "int4": "", - "int8": 1.63, + "int8": 335.04, "fp16": "", "fp32": "", "bf16": "" @@ -9010,19 +10070,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 386.67, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 99.8, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9032,7 +10096,7 @@ "Precisions": [ { "int4": "", - "int8": 1.46, + "int8": 2.82, "fp16": "", "fp32": "", "bf16": "" @@ -9044,19 +10108,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", "int8": "", - "fp16": 574.04, - "fp32": "", - "bf16": "" + "fp16": "", + "fp32": 32.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9078,19 +10146,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 630.96, - "fp16": 442.65, - "fp32": "", - "bf16": "" + "int8": 76.54, + "fp16": "", + "fp32": 27.6, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9100,7 +10172,7 @@ "Precisions": [ { "int4": "", - "int8": 1.74, + "int8": 13.2, "fp16": "", "fp32": "", "bf16": "" @@ -9112,19 +10184,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1281.44, - "fp16": 911.94, + "int8": 45.77, + "fp16": 40.93, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9134,7 +10210,7 @@ "Precisions": [ { "int4": "", - "int8": 1.08, + "int8": 21.21, "fp16": "", "fp32": "", "bf16": "" @@ -9146,19 +10222,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 890.07, - "fp16": 624.4, + "int8": 56.2, + "fp16": 41.8, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9168,7 +10248,7 @@ "Precisions": [ { "int4": "", - "int8": 1.41, + "int8": 23.38, "fp16": "", "fp32": "", "bf16": "" @@ -9180,19 +10260,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 0.56, + "fp16": 0.54, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9202,7 +10286,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1606.31, "fp16": "", "fp32": "", "bf16": "" @@ -9214,19 +10298,23 @@ } }, { - "Platform": "Intel® Processor N100 iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 339.71, - "fp16": 267.18, + "int8": 648.66, + "fp16": 431.47, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9236,7 +10324,7 @@ "Precisions": [ { "int4": "", - "int8": 3.83, + "int8": 1.76, "fp16": "", "fp32": "", "bf16": "" @@ -9248,19 +10336,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1298.67, - "fp16": 1337.24, + "int8": 208.21, + "fp16": 122.24, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9270,7 +10362,7 @@ "Precisions": [ { "int4": "", - "int8": 0.78, + "int8": 5.47, "fp16": "", "fp32": "", "bf16": "" @@ -9282,19 +10374,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1422.4, - "fp16": 1361.15, + "int8": 5.71, + "fp16": 3.09, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9304,7 +10400,7 @@ "Precisions": [ { "int4": "", - "int8": 1.27, + "int8": 173.5, "fp16": "", "fp32": "", "bf16": "" @@ -9316,19 +10412,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1200.66, - "fp16": 1554.15, + "int8": 348.95, + "fp16": 224.45, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9338,7 +10438,7 @@ "Precisions": [ { "int4": "", - "int8": 1.23, + "int8": 3.56, "fp16": "", "fp32": "", "bf16": "" @@ -9350,19 +10450,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i7-1185GRE iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 478.43, - "fp16": "", - "fp32": 307.02, - "bf16": "" + "int8": 113.89, + "fp16": 78.71, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9372,7 +10476,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 9.49, "fp16": "", "fp32": "", "bf16": "" @@ -9384,19 +10488,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 228.63, + "int8": 111.58, "fp16": "", - "fp32": 220.31, - "bf16": "" + "fp32": 57.55, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9418,19 +10526,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 524.29, + "int8": 141.13, "fp16": "", - "fp32": 394.1, - "bf16": "" + "fp32": 75.23, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9452,19 +10564,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.63, "fp16": "", - "fp32": 350.27, - "bf16": "" + "fp32": 0.68, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9486,19 +10602,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1049.71, + "int8": 2287.47, "fp16": "", - "fp32": 538.46, - "bf16": "" + "fp32": 1150.08, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9520,19 +10640,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 532.56, "fp16": "", - "fp32": 680.55, - "bf16": "" + "fp32": 180.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9554,19 +10678,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 10.33, "fp16": "", - "fp32": 347.8, - "bf16": "" + "fp32": 3.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9589,18 +10717,22 @@ }, { "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2306.0, + "int8": 1013.57, "fp16": "", - "fp32": 1096.89, - "bf16": "" + "fp32": 403.5, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9622,19 +10754,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1219.99, + "int8": "", "fp16": "", - "fp32": 644.18, - "bf16": "" + "fp32": 133.88, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9656,19 +10792,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1610.28, + "int8": 268.57, "fp16": "", - "fp32": 845.71, - "bf16": "" + "fp32": 120.55, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9690,19 +10830,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 484.88, + "int8": 87.88, "fp16": "", - "fp32": 280.8, - "bf16": "" + "fp32": 34.76, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9712,7 +10856,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 16.26, "fp16": "", "fp32": "", "bf16": "" @@ -9724,19 +10868,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4622.78, + "int8": 113.82, "fp16": "", - "fp32": 2338.78, - "bf16": "" + "fp32": 62.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9746,7 +10894,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.46, "fp16": "", "fp32": "", "bf16": "" @@ -9758,19 +10906,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.27, "fp16": "", - "fp32": 1684.08, - "bf16": "" + "fp32": 0.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9780,7 +10932,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 886.78, "fp16": "", "fp32": "", "bf16": "" @@ -9792,19 +10944,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 45.28, + "int8": 1982.75, "fp16": "", - "fp32": 18.84, - "bf16": "" + "fp32": 968.72, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9814,7 +10970,7 @@ "Precisions": [ { "int4": "", - "int8": 23.81, + "int8": 0.89, "fp16": "", "fp32": "", "bf16": "" @@ -9826,19 +10982,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 19.87, + "int8": 429.58, "fp16": "", - "fp32": 8.15, - "bf16": "" + "fp32": 107.58, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9848,7 +11008,7 @@ "Precisions": [ { "int4": "", - "int8": 51.41, + "int8": 3.47, "fp16": "", "fp32": "", "bf16": "" @@ -9860,19 +11020,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 51.65, + "int8": 7.11, "fp16": "", - "fp32": 14.46, - "bf16": "" + "fp32": 1.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9882,7 +11046,7 @@ "Precisions": [ { "int4": "", - "int8": 19.81, + "int8": 159.25, "fp16": "", "fp32": "", "bf16": "" @@ -9894,19 +11058,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 96.85, + "int8": 854.13, "fp16": "", - "fp32": 50.34, - "bf16": "" + "fp32": 289.32, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9916,7 +11084,7 @@ "Precisions": [ { "int4": "", - "int8": 10.76, + "int8": 1.72, "fp16": "", "fp32": "", "bf16": "" @@ -9928,19 +11096,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 144.67, + "int8": "", "fp16": "", - "fp32": 72.98, - "bf16": "" + "fp32": 90.72, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9950,7 +11122,7 @@ "Precisions": [ { "int4": "", - "int8": 8.21, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -9962,19 +11134,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 154.38, + "int8": 206.32, "fp16": "", - "fp32": 40.41, - "bf16": "" + "fp32": 78.09, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9984,7 +11160,7 @@ "Precisions": [ { "int4": "", - "int8": 11.07, + "int8": 6.49, "fp16": "", "fp32": "", "bf16": "" @@ -9996,19 +11172,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 185.28, - "fp16": "", - "fp32": 53.47, - "bf16": "" + "int8": 89.81, + "fp16": 69.99, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10018,7 +11198,7 @@ "Precisions": [ { "int4": "", - "int8": 9.56, + "int8": 12.71, "fp16": "", "fp32": "", "bf16": "" @@ -10030,19 +11210,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 543.91, - "fp16": "", - "fp32": 151.19, - "bf16": "" + "int8": 128.07, + "fp16": 97.39, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10052,7 +11236,7 @@ "Precisions": [ { "int4": "", - "int8": 2.82, + "int8": 12.87, "fp16": "", "fp32": "", "bf16": "" @@ -10064,19 +11248,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 151.17, - "fp16": "", - "fp32": 75.67, - "bf16": "" + "int8": 1.04, + "fp16": 1.15, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10086,7 +11274,7 @@ "Precisions": [ { "int4": "", - "int8": 7.12, + "int8": 972.87, "fp16": "", "fp32": "", "bf16": "" @@ -10098,19 +11286,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 225.84, - "fp16": "", - "fp32": 61.28, - "bf16": "" + "int8": 1281.93, + "fp16": 912.69, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10120,7 +11312,7 @@ "Precisions": [ { "int4": "", - "int8": 4.95, + "int8": 1.08, "fp16": "", "fp32": "", "bf16": "" @@ -10132,19 +11324,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 174.35, - "fp16": "", - "fp32": 45.46, - "bf16": "" + "int8": 381.27, + "fp16": 226.42, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10154,7 +11350,7 @@ "Precisions": [ { "int4": "", - "int8": 6.41, + "int8": 3.22, "fp16": "", "fp32": "", "bf16": "" @@ -10166,19 +11362,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 413.95, - "fp16": "", - "fp32": 107.82, - "bf16": "" + "int8": 10.47, + "fp16": 6.14, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10188,7 +11388,7 @@ "Precisions": [ { "int4": "", - "int8": 3.45, + "int8": 100.17, "fp16": "", "fp32": "", "bf16": "" @@ -10200,19 +11400,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 203.93, - "fp16": "", - "fp32": 59.04, - "bf16": "" + "int8": 744.92, + "fp16": 407.72, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10222,7 +11426,7 @@ "Precisions": [ { "int4": "", - "int8": 8.97, + "int8": 1.87, "fp16": "", "fp32": "", "bf16": "" @@ -10234,19 +11438,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-12700H iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 298.27, - "fp16": "", + "int8": 215.67, + "fp16": 148.01, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10256,7 +11464,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 5.58, "fp16": "", "fp32": "", "bf16": "" @@ -10268,19 +11476,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 122.15, + "int8": 61.33, "fp16": "", - "fp32": 60.34, - "bf16": "" + "fp32": 32.27, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10290,7 +11502,7 @@ "Precisions": [ { "int4": "", - "int8": 7.21, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10302,19 +11514,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 152.19, + "int8": 88.48, "fp16": "", - "fp32": 71.16, - "bf16": "" + "fp32": 59.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10324,7 +11540,7 @@ "Precisions": [ { "int4": "", - "int8": 7.72, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10336,19 +11552,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 766.25, + "int8": 0.81, "fp16": "", - "fp32": 233.37, - "bf16": "" + "fp32": 0.43, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10358,7 +11578,7 @@ "Precisions": [ { "int4": "", - "int8": 2.16, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10370,19 +11590,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 48.79, + "int8": 1218.37, "fp16": "", - "fp32": 20.22, - "bf16": "" + "fp32": 644.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10392,7 +11616,7 @@ "Precisions": [ { "int4": "", - "int8": 21.9, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10404,19 +11628,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 245.58, + "int8": 284.91, "fp16": "", - "fp32": 121.38, - "bf16": "" + "fp32": 109.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10426,7 +11654,7 @@ "Precisions": [ { "int4": "", - "int8": 5.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10438,19 +11666,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 92.24, + "int8": 5.67, "fp16": "", - "fp32": 49.79, - "bf16": "" + "fp32": 2.15, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10460,7 +11692,7 @@ "Precisions": [ { "int4": "", - "int8": 11.19, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10472,19 +11704,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 971.95, + "int8": 554.73, "fp16": "", - "fp32": 269.81, - "bf16": "" + "fp32": 228.8, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10494,7 +11730,7 @@ "Precisions": [ { "int4": "", - "int8": 3.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10506,19 +11742,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2968.31, + "int8": "", "fp16": "", - "fp32": 754.35, - "bf16": "" + "fp32": 80.32, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10528,7 +11768,7 @@ "Precisions": [ { "int4": "", - "int8": 1.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10540,19 +11780,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4948.09, + "int8": 154.56, "fp16": "", - "fp32": 1155.67, - "bf16": "" + "fp32": 72.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10562,7 +11806,7 @@ "Precisions": [ { "int4": "", - "int8": 1.06, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10574,19 +11818,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 19506.31, + "int8": 44.62, "fp16": "", - "fp32": 1609.79, - "bf16": 7600.31 + "fp32": 17.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10596,10 +11844,10 @@ "Precisions": [ { "int4": "", - "int8": 0.99, + "int8": 37.64, "fp16": "", "fp32": "", - "bf16": 1.24 + "bf16": "" } ], "Unit": "ms", @@ -10608,19 +11856,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21749.45, + "int8": 61.85, "fp16": "", - "fp32": 2011.71, - "bf16": 13645.99 + "fp32": 39.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10630,10 +11882,10 @@ "Precisions": [ { "int4": "", - "int8": 0.98, + "int8": 26.95, "fp16": "", "fp32": "", - "bf16": 1.33 + "bf16": "" } ], "Unit": "ms", @@ -10642,19 +11894,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2122.71, + "int8": 0.64, "fp16": "", - "fp32": 565.52, - "bf16": "" + "fp32": 0.17, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10664,7 +11920,7 @@ "Precisions": [ { "int4": "", - "int8": 1.88, + "int8": 1935.64, "fp16": "", "fp32": "", "bf16": "" @@ -10676,19 +11932,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2249.41, + "int8": 1042.94, "fp16": "", - "fp32": 563.79, - "bf16": "" + "fp32": 515.99, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10698,7 +11958,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.61, "fp16": "", "fp32": "", "bf16": "" @@ -10710,19 +11970,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 392.72, + "int8": 203.02, "fp16": "", - "fp32": 95.29, - "bf16": "" + "fp32": 59.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10732,7 +11996,7 @@ "Precisions": [ { "int4": "", - "int8": 6.4, + "int8": 9.0, "fp16": "", "fp32": "", "bf16": "" @@ -10744,19 +12008,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 290.74, + "int8": 3.48, "fp16": "", - "fp32": 70.82, - "bf16": "" + "fp32": 1.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10766,7 +12034,7 @@ "Precisions": [ { "int4": "", - "int8": 7.55, + "int8": 439.19, "fp16": "", "fp32": "", "bf16": "" @@ -10778,19 +12046,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 318.23, + "int8": 422.9, "fp16": "", - "fp32": 86.35, - "bf16": "" + "fp32": 151.69, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10800,7 +12072,7 @@ "Precisions": [ { "int4": "", - "int8": 4.59, + "int8": 3.87, "fp16": "", "fp32": "", "bf16": "" @@ -10812,19 +12084,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2318.55, - "fp16": 1375.72, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 48.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10834,7 +12110,7 @@ "Precisions": [ { "int4": "", - "int8": 0.87, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10846,19 +12122,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i7-1355U Processor CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 101.73, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 40.76, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10868,7 +12148,7 @@ "Precisions": [ { "int4": "", - "int8": 1.42, + "int8": 16.99, "fp16": "", "fp32": "", "bf16": "" @@ -10880,19 +12160,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 854.09, - "fp16": 539.78, + "int8": 67.08, + "fp16": 52.9, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10902,7 +12186,7 @@ "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 14.38, "fp16": "", "fp32": "", "bf16": "" @@ -10914,19 +12198,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 768.87, - "fp16": 382.94, + "int8": 98.8, + "fp16": 73.53, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10936,7 +12224,7 @@ "Precisions": [ { "int4": "", - "int8": 1.57, + "int8": 13.41, "fp16": "", "fp32": "", "bf16": "" @@ -10948,19 +12236,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 772.54, - "fp16": 383.99, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 6.21, + "token_int8": 3.88, + "token_fp16": "" } ], "Unit": "FPS", @@ -10969,8 +12261,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.54, + "int4": 160.82, + "int8": 257.32, "fp16": "", "fp32": "", "bf16": "" @@ -10982,19 +12274,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2022.98, - "fp16": 975.93, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 7.25, + "token_int8": 4.27, + "token_fp16": "" } ], "Unit": "FPS", @@ -11003,8 +12299,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 0.75, + "int4": 137.82, + "int8": 233.92, "fp16": "", "fp32": "", "bf16": "" @@ -11016,19 +12312,23 @@ } }, { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 106.37, - "fp16": 64.66, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 8.53, + "token_int8": 5.74, + "token_fp16": "" } ], "Unit": "FPS", @@ -11037,8 +12337,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 10.57, + "int4": 117.18, + "int8": 174.01, "fp16": "", "fp32": "", "bf16": "" @@ -11050,19 +12350,23 @@ } }, { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 48.1, - "fp16": 51.69, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 8.49, + "token_int8": 5.06, + "token_fp16": "" } ], "Unit": "FPS", @@ -11071,8 +12375,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 23.01, + "int4": 117.69, + "int8": 197.3, "fp16": "", "fp32": "", "bf16": "" @@ -11084,19 +12388,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 203.75, - "fp16": 118.64, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 16.79, + "token_int8": 11.89, + "token_fp16": 6.7 } ], "Unit": "FPS", @@ -11105,9 +12413,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 6.33, - "fp16": "", + "int4": 59.54, + "int8": 84.05, + "fp16": 149.13, "fp32": "", "bf16": "" } @@ -11118,19 +12426,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 213.35, - "fp16": 119.51, + "int8": 0.73, + "fp16": 0.77, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11140,7 +12452,7 @@ "Precisions": [ { "int4": "", - "int8": 4.89, + "int8": 1191.59, "fp16": "", "fp32": "", "bf16": "" @@ -11152,19 +12464,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 225.63, - "fp16": 128.49, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 8.86, + "token_int8": 5.44, + "token_fp16": "" } ], "Unit": "FPS", @@ -11173,8 +12489,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 4.26, + "int4": 112.76, + "int8": 183.5, "fp16": "", "fp32": "", "bf16": "" @@ -11186,19 +12502,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": 174.47, + "int8": 869.88, + "fp16": 621.94, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11208,7 +12528,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.38, "fp16": "", "fp32": "", "bf16": "" @@ -11220,19 +12540,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 203.92, - "fp16": 125.66, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 14.78, + "token_int8": 9.98, + "token_fp16": 5.45 } ], "Unit": "FPS", @@ -11241,9 +12565,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 5.53, - "fp16": "", + "int4": 67.65, + "int8": 100.19, + "fp16": 183.48, "fp32": "", "bf16": "" } @@ -11254,19 +12578,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 387.63, - "fp16": 228.73, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 9.11, + "token_int8": 5.39, + "token_fp16": "" } ], "Unit": "FPS", @@ -11275,8 +12603,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 3.23, + "int4": 109.74, + "int8": 185.49, "fp16": "", "fp32": "", "bf16": "" @@ -11290,17 +12618,21 @@ { "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 278.1, - "fp16": 165.78, + "int8": 277.06, + "fp16": 164.27, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11310,7 +12642,7 @@ "Precisions": [ { "int4": "", - "int8": 3.86, + "int8": 3.85, "fp16": "", "fp32": "", "bf16": "" @@ -11322,19 +12654,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 7.1, + "fp16": 3.99, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11344,7 +12680,7 @@ "Precisions": [ { "int4": "", - "int8": 3.5, + "int8": 126.73, "fp16": "", "fp32": "", "bf16": "" @@ -11356,19 +12692,23 @@ } }, { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 81.72, - "fp16": 49.75, + "int8": 484.13, + "fp16": 298.47, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11378,7 +12718,7 @@ "Precisions": [ { "int4": "", - "int8": 13.15, + "int8": 2.49, "fp16": "", "fp32": "", "bf16": "" @@ -11390,19 +12730,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 556.79, - "fp16": 393.72, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11412,8 +12756,8 @@ "Precisions": [ { "int4": "", - "int8": 1.7, - "fp16": "", + "int8": 29.54, + "fp16": 29.97, "fp32": "", "bf16": "" } @@ -11424,19 +12768,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 568.08, - "fp16": 375.74, + "int8": 162.35, + "fp16": 106.83, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11446,7 +12794,7 @@ "Precisions": [ { "int4": "", - "int8": 2.26, + "int8": 6.38, "fp16": "", "fp32": "", "bf16": "" @@ -11458,19 +12806,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1167.79, - "fp16": 621.08, - "fp32": "", - "bf16": "" + "int8": 170.14, + "fp16": "", + "fp32": 67.07, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11480,7 +12832,7 @@ "Precisions": [ { "int4": "", - "int8": 1.49, + "int8": 10.73, "fp16": "", "fp32": "", "bf16": "" @@ -11492,19 +12844,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 129.36, + "int8": 219.8, "fp16": "", - "fp32": 55.58, - "bf16": "" + "fp32": 126.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11514,7 +12870,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 7.34, "fp16": "", "fp32": "", "bf16": "" @@ -11526,19 +12882,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 59.45, + "int8": "", "fp16": "", - "fp32": 54.27, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 9.42, + "token_int8": 6.89, + "token_fp16": 3.59 } ], "Unit": "FPS", @@ -11547,9 +12907,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 106.06, + "int8": 144.95, + "fp16": 278.42, "fp32": "", "bf16": "" } @@ -11560,19 +12920,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 196.98, + "int8": "", "fp16": "", - "fp32": 115.77, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 10.65, + "token_int8": 7.46, + "token_fp16": 3.83 } ], "Unit": "FPS", @@ -11581,9 +12945,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 93.82, + "int8": 133.88, + "fp16": 260.66, "fp32": "", "bf16": "" } @@ -11594,10 +12958,11 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ @@ -11605,8 +12970,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 53.71, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 13.44, + "token_int8": 9.29, + "token_fp16": 4.94 } ], "Unit": "FPS", @@ -11615,9 +12983,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 74.39, + "int8": 107.62, + "fp16": 202.32, "fp32": "", "bf16": "" } @@ -11628,19 +12996,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 235.76, + "int8": "", "fp16": "", - "fp32": 88.15, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 11.91, + "token_int8": 8.65, + "token_fp16": 4.48 } ], "Unit": "FPS", @@ -11649,9 +13021,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 83.93, + "int8": 115.48, + "fp16": 223.15, "fp32": "", "bf16": "" } @@ -11662,19 +13034,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 368.58, + "int8": "", "fp16": "", - "fp32": 166.93, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 25.41, + "token_int8": 18.99, + "token_fp16": 10.18 } ], "Unit": "FPS", @@ -11683,9 +13059,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 39.35, + "int8": 52.64, + "fp16": 98.23, "fp32": "", "bf16": "" } @@ -11696,19 +13072,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 224.44, + "int8": 2.49, "fp16": "", - "fp32": 81.26, - "bf16": "" + "fp32": 0.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11718,7 +13098,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 562.6, "fp16": "", "fp32": "", "bf16": "" @@ -11730,10 +13110,11 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ @@ -11742,7 +13123,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 14.41, + "token_int8": 9.12, + "token_fp16": 4.71 } ], "Unit": "FPS", @@ -11751,9 +13135,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 69.39, + "int8": 109.54, + "fp16": 211.91, "fp32": "", "bf16": "" } @@ -11764,19 +13148,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 285.59, + "int8": 4239.14, "fp16": "", - "fp32": 110.56, - "bf16": "" + "fp32": 2047.2, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11786,7 +13174,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.6, "fp16": "", "fp32": "", "bf16": "" @@ -11798,19 +13186,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 433.32, + "int8": "", "fp16": "", - "fp32": 147.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 15.66, + "token_fp16": 8.52 } ], "Unit": "FPS", @@ -11820,8 +13212,8 @@ "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 63.84, + "fp16": 117.37, "fp32": "", "bf16": "" } @@ -11832,19 +13224,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 112.49, + "int8": "", "fp16": "", - "fp32": 42.68, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 13.1, + "token_int8": 9.24, + "token_fp16": 4.75 } ], "Unit": "FPS", @@ -11853,9 +13249,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 76.33, + "int8": 108.16, + "fp16": 210.38, "fp32": "", "bf16": "" } @@ -11866,19 +13262,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", + "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1107.14, + "int8": 762.32, "fp16": "", - "fp32": 471.86, - "bf16": "" + "fp32": 234.53, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11888,7 +13288,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.17, "fp16": "", "fp32": "", "bf16": "" @@ -11900,19 +13300,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 12.97, "fp16": "", - "fp32": 257.94, - "bf16": "" + "fp32": 3.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11922,7 +13326,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 102.02, "fp16": "", "fp32": "", "bf16": "" @@ -11934,19 +13338,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.76, + "int8": 1606.89, "fp16": "", - "fp32": 0.31, - "bf16": "" + "fp32": 589.62, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11956,7 +13364,7 @@ "Precisions": [ { "int4": "", - "int8": 1318.07, + "int8": 1.08, "fp16": "", "fp32": "", "bf16": "" @@ -11968,19 +13376,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.33, + "int8": "", "fp16": "", - "fp32": 0.13, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11990,8 +13402,8 @@ "Precisions": [ { "int4": "", - "int8": 2998.35, - "fp16": "", + "int8": 40.27, + "fp16": 39.61, "fp32": "", "bf16": "" } @@ -12002,19 +13414,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.89, + "int8": "", "fp16": "", - "fp32": 0.23, - "bf16": "" + "fp32": 187.66, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12024,7 +13440,7 @@ "Precisions": [ { "int4": "", - "int8": 1117.77, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12036,19 +13452,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Core™ i9-13900K CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.67, + "int8": 389.04, "fp16": "", - "fp32": 0.89, - "bf16": "" + "fp32": 154.4, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12058,7 +13478,7 @@ "Precisions": [ { "int4": "", - "int8": 598.98, + "int8": 4.13, "fp16": "", "fp32": "", "bf16": "" @@ -12070,19 +13490,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.42, - "fp16": "", - "fp32": 1.29, - "bf16": "" + "int8": 385.87, + "fp16": 420.99, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12092,7 +13516,7 @@ "Precisions": [ { "int4": "", - "int8": 428.32, + "int8": 2.99, "fp16": "", "fp32": "", "bf16": "" @@ -12104,19 +13528,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.76, - "fp16": "", - "fp32": 0.79, - "bf16": "" + "int8": 426.56, + "fp16": 362.73, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12126,7 +13554,7 @@ "Precisions": [ { "int4": "", - "int8": 494.95, + "int8": 2.8, "fp16": "", "fp32": "", "bf16": "" @@ -12138,19 +13566,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.17, + "int8": "", "fp16": "", - "fp32": 0.92, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 22.66, + "token_int8": 18.13, + "token_fp16": "" } ], "Unit": "FPS", @@ -12159,8 +13591,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 463.7, + "int4": 44.13, + "int8": 55.13, "fp16": "", "fp32": "", "bf16": "" @@ -12172,19 +13604,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.97, + "int8": "", "fp16": "", - "fp32": 2.5, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 40.04, + "token_int8": 26.95, + "token_fp16": "" } ], "Unit": "FPS", @@ -12193,8 +13629,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 128.87, + "int4": 24.97, + "int8": 37.1, "fp16": "", "fp32": "", "bf16": "" @@ -12206,19 +13642,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.61, + "int8": "", "fp16": "", - "fp32": 1.34, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.22, + "token_int8": 33.88, + "token_fp16": 21.45 } ], "Unit": "FPS", @@ -12227,9 +13667,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 399.72, - "fp16": "", + "int4": 22.11, + "int8": 29.51, + "fp16": 46.62, "fp32": "", "bf16": "" } @@ -12240,19 +13680,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.96, + "int8": "", "fp16": "", - "fp32": 1.02, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.55, + "token_int8": 30.8, + "token_fp16": "" } ], "Unit": "FPS", @@ -12261,8 +13705,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 250.0, + "int4": 21.95, + "int8": 32.46, "fp16": "", "fp32": "", "bf16": "" @@ -12274,10 +13718,11 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ @@ -12286,7 +13731,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 69.44, + "token_int8": 57.9, + "token_fp16": 37.69 } ], "Unit": "FPS", @@ -12295,9 +13743,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 321.18, - "fp16": "", + "int4": 14.4, + "int8": 17.27, + "fp16": 26.53, "fp32": "", "bf16": "" } @@ -12308,19 +13756,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.82, - "fp16": "", - "fp32": 1.9, - "bf16": "" + "int8": 33.38, + "fp16": 19.04, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12330,7 +13782,7 @@ "Precisions": [ { "int4": "", - "int8": 169.83, + "int8": 48.67, "fp16": "", "fp32": "", "bf16": "" @@ -12342,19 +13794,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.5, + "int8": "", "fp16": "", - "fp32": 1.02, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.53, + "token_int8": 32.37, + "token_fp16": 20.21 } ], "Unit": "FPS", @@ -12363,9 +13819,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 436.59, - "fp16": "", + "int4": 21.96, + "int8": 30.89, + "fp16": 49.48, "fp32": "", "bf16": "" } @@ -12376,19 +13832,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.14, - "fp16": "", + "int8": 3134.27, + "fp16": 3004.5, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12398,7 +13858,7 @@ "Precisions": [ { "int4": "", - "int8": 247.47, + "int8": 0.57, "fp16": "", "fp32": "", "bf16": "" @@ -12410,19 +13870,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.0, + "int8": "", "fp16": "", - "fp32": 1.06, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 69.93, + "token_int8": 51.51, + "token_fp16": 32.84 } ], "Unit": "FPS", @@ -12431,9 +13895,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 374.78, - "fp16": "", + "int4": 14.3, + "int8": 19.41, + "fp16": 30.45, "fp32": "", "bf16": "" } @@ -12444,19 +13908,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.57, + "int8": "", "fp16": "", - "fp32": 1.28, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.8, + "token_int8": 32.78, + "token_fp16": "" } ], "Unit": "FPS", @@ -12465,8 +13933,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 413.46, + "int4": 21.83, + "int8": 30.5, "fp16": "", "fp32": "", "bf16": "" @@ -12478,19 +13946,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.97, - "fp16": "", - "fp32": 3.83, - "bf16": "" + "int8": 1921.18, + "fp16": 1329.28, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12500,7 +13972,7 @@ "Precisions": [ { "int4": "", - "int8": 101.21, + "int8": 0.78, "fp16": "", "fp32": "", "bf16": "" @@ -12512,19 +13984,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.82, + "int8": 133.77, "fp16": "", - "fp32": 0.32, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12534,7 +14010,7 @@ "Precisions": [ { "int4": "", - "int8": 1223.85, + "int8": 13.93, "fp16": "", "fp32": "", "bf16": "" @@ -12546,19 +14022,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.34, - "fp16": "", - "fp32": 2.29, - "bf16": "" + "int8": 2200.83, + "fp16": 1665.15, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12568,7 +14048,7 @@ "Precisions": [ { "int4": "", - "int8": 239.97, + "int8": 0.78, "fp16": "", "fp32": "", "bf16": "" @@ -12580,19 +14060,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.59, + "int8": "", "fp16": "", - "fp32": 0.85, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12602,8 +14086,8 @@ "Precisions": [ { "int4": "", - "int8": 628.98, - "fp16": "", + "int8": 2.33, + "fp16": 2.36, "fp32": "", "bf16": "" } @@ -12614,19 +14098,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Data Center GPU Flex 170 dGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 17.67, - "fp16": "", - "fp32": 4.59, - "bf16": "" + "int8": 759.93, + "fp16": 694.57, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12636,7 +14124,7 @@ "Precisions": [ { "int4": "", - "int8": 115.61, + "int8": 1.96, "fp16": "", "fp32": "", "bf16": "" @@ -12648,19 +14136,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 58.24, + "int8": 36.93, "fp16": "", - "fp32": 15.05, - "bf16": "" + "fp32": 27.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12670,7 +14162,7 @@ "Precisions": [ { "int4": "", - "int8": 36.32, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12682,19 +14174,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 85.63, + "int8": 484.32, "fp16": "", - "fp32": 21.01, - "bf16": "" + "fp32": 278.4, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12704,7 +14200,7 @@ "Precisions": [ { "int4": "", - "int8": 25.38, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12716,19 +14212,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 438.33, + "int8": 112.23, "fp16": "", - "fp32": 30.86, - "bf16": 213.33 + "fp32": 42.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12738,10 +14238,10 @@ "Precisions": [ { "int4": "", - "int8": 7.94, + "int8": "", "fp16": "", "fp32": "", - "bf16": 13.88 + "bf16": "" } ], "Unit": "ms", @@ -12750,19 +14250,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 512.76, + "int8": 2.04, "fp16": "", - "fp32": 35.43, - "bf16": 276.38 + "fp32": 0.6, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12772,10 +14276,10 @@ "Precisions": [ { "int4": "", - "int8": 7.12, + "int8": "", "fp16": "", "fp32": "", - "bf16": 11.56 + "bf16": "" } ], "Unit": "ms", @@ -12784,19 +14288,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 41.98, + "int8": 216.96, "fp16": "", - "fp32": 10.9, - "bf16": "" + "fp32": 94.92, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12806,7 +14314,7 @@ "Precisions": [ { "int4": "", - "int8": 48.76, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12818,19 +14326,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 42.23, + "int8": "", "fp16": "", - "fp32": 10.5, - "bf16": "" + "fp32": 34.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12840,7 +14352,7 @@ "Precisions": [ { "int4": "", - "int8": 49.04, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12852,19 +14364,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Processor N100 CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.45, + "int8": 61.06, "fp16": "", - "fp32": 1.66, - "bf16": "" + "fp32": 28.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12874,7 +14390,7 @@ "Precisions": [ { "int4": "", - "int8": 208.68, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12886,19 +14402,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Processor N100 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.88, + "int8": 15.44, "fp16": "", - "fp32": 1.23, - "bf16": "" + "fp32": 12.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12908,7 +14428,7 @@ "Precisions": [ { "int4": "", - "int8": 259.24, + "int8": 66.23, "fp16": "", "fp32": "", "bf16": "" @@ -12920,19 +14440,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Processor N100 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.08, + "int8": 296.53, "fp16": "", - "fp32": 1.48, - "bf16": "" + "fp32": 183.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12942,7 +14466,7 @@ "Precisions": [ { "int4": "", - "int8": 234.74, + "int8": 3.8, "fp16": "", "fp32": "", "bf16": "" @@ -12954,19 +14478,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Processor N100 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 140.65, - "fp16": 92.11, - "fp32": "", - "bf16": "" + "int8": 48.77, + "fp16": "", + "fp32": 20.13, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12976,7 +14504,7 @@ "Precisions": [ { "int4": "", - "int8": 15.32, + "int8": 21.88, "fp16": "", "fp32": "", "bf16": "" @@ -12988,19 +14516,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Platform": "Intel® Processor N100 CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.82, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 0.31, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13010,7 +14542,7 @@ "Precisions": [ { "int4": "", - "int8": 14.78, + "int8": 1224.62, "fp16": "", "fp32": "", "bf16": "" @@ -13022,19 +14554,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Processor N100 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 35.4, - "fp16": 17.45, - "fp32": "", - "bf16": "" + "int8": 106.12, + "fp16": "", + "fp32": 49.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13044,7 +14580,7 @@ "Precisions": [ { "int4": "", - "int8": 35.98, + "int8": 9.72, "fp16": "", "fp32": "", "bf16": "" @@ -13056,19 +14592,23 @@ } }, { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Processor N100 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.16, - "fp16": 1.32, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 15.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13078,7 +14618,7 @@ "Precisions": [ { "int4": "", - "int8": 472.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -13090,19 +14630,23 @@ } }, { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Processor N100 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.16, - "fp16": 1.16, - "fp32": "", - "bf16": "" + "int8": 23.65, + "fp16": "", + "fp32": 12.86, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13112,7 +14656,7 @@ "Precisions": [ { "int4": "", - "int8": 870.37, + "int8": 43.43, "fp16": "", "fp32": "", "bf16": "" @@ -13124,19 +14668,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { + "Platform": "Intel® Processor N100 iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", + "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.09, - "fp16": 2.78, + "int8": 33.69, + "fp16": 30.91, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13146,7 +14694,7 @@ "Precisions": [ { "int4": "", - "int8": 210.29, + "int8": 38.02, "fp16": "", "fp32": "", "bf16": "" @@ -13158,19 +14706,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Processor N100 iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.8, - "fp16": 2.81, + "int8": 337.95, + "fp16": 267.38, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13180,7 +14732,7 @@ "Precisions": [ { "int4": "", - "int8": 164.94, + "int8": 3.84, "fp16": "", "fp32": "", "bf16": "" @@ -13192,19 +14744,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Processor N100 iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.92, - "fp16": 2.89, + "int8": 81.72, + "fp16": 49.76, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13214,7 +14770,7 @@ "Precisions": [ { "int4": "", - "int8": 143.88, + "int8": 13.15, "fp16": "", "fp32": "", "bf16": "" @@ -13226,19 +14782,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", + "Platform": "Intel® Processor N100 iGPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.32, - "fp16": 4.74, + "int8": 1.62, + "fp16": 1.01, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13248,7 +14808,7 @@ "Precisions": [ { "int4": "", - "int8": 117.76, + "int8": 622.97, "fp16": "", "fp32": "", "bf16": "" @@ -13260,19 +14820,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Processor N100 iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.43, - "fp16": 3.17, + "int8": 164.31, + "fp16": 106.85, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13282,7 +14846,7 @@ "Precisions": [ { "int4": "", - "int8": 170.95, + "int8": 7.35, "fp16": "", "fp32": "", "bf16": "" @@ -13294,19 +14858,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Processor N100 iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (Intel® Atom™)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10.46, - "fp16": 6.1, + "int8": 47.04, + "fp16": 34.97, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13316,7 +14884,7 @@ "Precisions": [ { "int4": "", - "int8": 100.05, + "int8": 23.03, "fp16": "", "fp32": "", "bf16": "" @@ -13328,19 +14896,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.12, - "fp16": 4.0, - "fp32": "", - "bf16": "" + "int8": 218.18, + "fp16": "", + "fp32": 80.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13350,7 +14922,7 @@ "Precisions": [ { "int4": "", - "int8": 126.18, + "int8": 14.4, "fp16": "", "fp32": "", "bf16": "" @@ -13362,19 +14934,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 271.94, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 167.25, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13384,7 +14960,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.07, "fp16": "", "fp32": "", "bf16": "" @@ -13396,19 +14972,23 @@ } }, { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.62, - "fp16": 1.01, - "fp32": "", - "bf16": "" + "int8": 3.26, + "fp16": "", + "fp32": 0.9, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13418,7 +14998,7 @@ "Precisions": [ { "int4": "", - "int8": 623.25, + "int8": 637.88, "fp16": "", "fp32": "", "bf16": "" @@ -13430,19 +15010,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.37, - "fp16": 12.68, - "fp32": "", - "bf16": "" + "int8": 5417.98, + "fp16": "", + "fp32": 1926.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13452,7 +15036,7 @@ "Precisions": [ { "int4": "", - "int8": 47.25, + "int8": 1.45, "fp16": "", "fp32": "", "bf16": "" @@ -13464,19 +15048,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": 10.09, - "fp32": "", - "bf16": "" + "int8": 979.5, + "fp16": "", + "fp32": 267.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13486,7 +15074,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.06, "fp16": "", "fp32": "", "bf16": "" @@ -13498,19 +15086,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 65.86, - "fp16": 39.71, - "fp32": "", - "bf16": "" + "int8": 17.65, + "fp16": "", + "fp32": 4.58, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13520,7 +15112,7 @@ "Precisions": [ { "int4": "", - "int8": 21.36, + "int8": 116.19, "fp16": "", "fp32": "", "bf16": "" @@ -13532,19 +15124,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.53, + "int8": 2104.85, "fp16": "", - "fp32": 0.87, - "bf16": "" + "fp32": 639.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13554,7 +15150,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.56, "fp16": "", "fp32": "", "bf16": "" @@ -13566,19 +15162,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.25, + "int8": "", "fp16": "", - "fp32": 1.08, - "bf16": "" + "fp32": 206.18, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13600,19 +15200,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Xeon® Gold 5218T CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.37, + "int8": 440.56, "fp16": "", - "fp32": 2.71, - "bf16": "" + "fp32": 173.57, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13622,7 +15226,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 5.93, "fp16": "", "fp32": "", "bf16": "" @@ -13634,19 +15238,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.52, + "int8": 426.19, "fp16": "", - "fp32": 1.13, - "bf16": "" + "fp32": 162.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13656,7 +15264,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.09, "fp16": "", "fp32": "", "bf16": "" @@ -13668,19 +15276,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.77, + "int8": 411.51, "fp16": "", - "fp32": 1.75, - "bf16": "" + "fp32": 254.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13690,7 +15302,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.51, "fp16": "", "fp32": "", "bf16": "" @@ -13702,19 +15314,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.81, + "int8": 6.45, "fp16": "", - "fp32": 3.81, - "bf16": "" + "fp32": 1.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13724,7 +15340,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 321.85, "fp16": "", "fp32": "", "bf16": "" @@ -13736,19 +15352,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.33, + "int8": 10273.19, "fp16": "", - "fp32": 1.68, - "bf16": "" + "fp32": 3342.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13758,7 +15378,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.21, "fp16": "", "fp32": "", "bf16": "" @@ -13770,19 +15390,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 2125.81, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 570.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13792,7 +15416,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.84, "fp16": "", "fp32": "", "bf16": "" @@ -13804,19 +15428,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.71, + "int8": 41.83, "fp16": "", - "fp32": 2.16, - "bf16": "" + "fp32": 10.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13826,7 +15454,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 49.53, "fp16": "", "fp32": "", "bf16": "" @@ -13838,19 +15466,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 4376.71, "fp16": "", - "fp32": 3.05, - "bf16": "" + "fp32": 1244.57, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13860,7 +15492,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.22, "fp16": "", "fp32": "", "bf16": "" @@ -13872,19 +15504,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.05, + "int8": "", "fp16": "", - "fp32": 0.61, - "bf16": "" + "fp32": 383.86, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13906,19 +15542,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6238L CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 18.73, + "int8": 749.14, "fp16": "", - "fp32": 9.5, - "bf16": "" + "fp32": 338.04, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13928,7 +15568,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 4.21, "fp16": "", "fp32": "", "bf16": "" @@ -13940,19 +15580,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.04, + "int8": 622.71, "fp16": "", - "fp32": 4.95, - "bf16": "" + "fp32": 240.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13962,7 +15606,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 6.4, "fp16": "", "fp32": "", "bf16": "" @@ -13974,19 +15618,23 @@ } }, { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 97.87, + "int8": 721.9, "fp16": "", - "fp32": 45.25, - "bf16": "" + "fp32": 423.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13996,7 +15644,7 @@ "Precisions": [ { "int4": "", - "int8": 10.56, + "int8": 4.83, "fp16": "", "fp32": "", "bf16": "" @@ -14008,19 +15656,23 @@ } }, { - "Platform": "Intel® Atom® X6425E CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 45.67, + "int8": 10.46, "fp16": "", - "fp32": 21.53, - "bf16": "" + "fp32": 2.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14030,7 +15682,7 @@ "Precisions": [ { "int4": "", - "int8": 22.79, + "int8": 221.46, "fp16": "", "fp32": "", "bf16": "" @@ -14042,19 +15694,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 115.01, + "int8": 16509.95, "fp16": "", - "fp32": 36.97, - "bf16": "" + "fp32": 5201.56, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14064,7 +15720,7 @@ "Precisions": [ { "int4": "", - "int8": 9.06, + "int8": 0.59, "fp16": "", "fp32": "", "bf16": "" @@ -14076,19 +15732,23 @@ } }, { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 211.59, + "int8": 3352.09, "fp16": "", - "fp32": 119.52, - "bf16": "" + "fp32": 825.5, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14098,7 +15758,7 @@ "Precisions": [ { "int4": "", - "int8": 4.95, + "int8": 1.34, "fp16": "", "fp32": "", "bf16": "" @@ -14110,19 +15770,23 @@ } }, { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 327.96, + "int8": 60.91, "fp16": "", - "fp32": 163.65, - "bf16": "" + "fp32": 15.11, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14132,7 +15796,7 @@ "Precisions": [ { "int4": "", - "int8": 3.63, + "int8": 36.91, "fp16": "", "fp32": "", "bf16": "" @@ -14144,19 +15808,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 326.48, + "int8": 6975.09, "fp16": "", - "fp32": 98.82, - "bf16": "" + "fp32": 1755.62, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14166,7 +15834,7 @@ "Precisions": [ { "int4": "", - "int8": 4.67, + "int8": 0.77, "fp16": "", "fp32": "", "bf16": "" @@ -14178,19 +15846,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 383.99, + "int8": "", "fp16": "", - "fp32": 136.38, - "bf16": "" + "fp32": 571.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14200,7 +15872,7 @@ "Precisions": [ { "int4": "", - "int8": 4.16, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -14212,19 +15884,23 @@ } }, { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Gold 6338N CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1073.43, + "int8": 1224.86, "fp16": "", - "fp32": 385.05, - "bf16": "" + "fp32": 495.73, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14234,7 +15910,7 @@ "Precisions": [ { "int4": "", - "int8": 1.31, + "int8": 2.98, "fp16": "", "fp32": "", "bf16": "" @@ -14246,19 +15922,23 @@ } }, { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 335.03, + "int8": 587.54, "fp16": "", - "fp32": 182.29, - "bf16": "" + "fp32": 225.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14268,7 +15948,7 @@ "Precisions": [ { "int4": "", - "int8": 3.15, + "int8": 9.18, "fp16": "", "fp32": "", "bf16": "" @@ -14280,19 +15960,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 495.53, + "int8": 580.8, "fp16": "", - "fp32": 147.22, - "bf16": "" + "fp32": 343.39, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14302,7 +15986,7 @@ "Precisions": [ { "int4": "", - "int8": 2.19, + "int8": 6.9, "fp16": "", "fp32": "", "bf16": "" @@ -14314,19 +15998,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 385.05, + "int8": 8.58, "fp16": "", - "fp32": 100.34, - "bf16": "" + "fp32": 2.26, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14336,7 +16024,7 @@ "Precisions": [ { "int4": "", - "int8": 2.83, + "int8": 248.72, "fp16": "", "fp32": "", "bf16": "" @@ -14348,19 +16036,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 832.22, + "int8": 14930.31, "fp16": "", - "fp32": 284.89, - "bf16": "" + "fp32": 4646.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14370,7 +16062,7 @@ "Precisions": [ { "int4": "", - "int8": 1.73, + "int8": 0.93, "fp16": "", "fp32": "", "bf16": "" @@ -14382,19 +16074,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 423.45, + "int8": 2965.31, "fp16": "", - "fp32": 153.0, - "bf16": "" + "fp32": 761.01, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14404,7 +16100,7 @@ "Precisions": [ { "int4": "", - "int8": 3.88, + "int8": 1.59, "fp16": "", "fp32": "", "bf16": "" @@ -14416,19 +16112,23 @@ } }, { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 58.15, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 15.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14438,7 +16138,7 @@ "Precisions": [ { "int4": "", - "int8": 2.45, + "int8": 37.18, "fp16": "", "fp32": "", "bf16": "" @@ -14450,19 +16150,23 @@ } }, { - "Platform": "Intel® Core™ i7-8700T CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 275.79, + "int8": 6130.48, "fp16": "", - "fp32": 151.19, - "bf16": "" + "fp32": 1654.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14472,7 +16176,7 @@ "Precisions": [ { "int4": "", - "int8": 3.22, + "int8": 1.2, "fp16": "", "fp32": "", "bf16": "" @@ -14484,19 +16188,23 @@ } }, { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 351.76, + "int8": "", "fp16": "", - "fp32": 183.93, - "bf16": "" + "fp32": 512.57, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14506,7 +16214,7 @@ "Precisions": [ { "int4": "", - "int8": 3.41, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -14518,19 +16226,23 @@ } }, { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8280 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1607.16, + "int8": 996.59, "fp16": "", - "fp32": 587.94, - "bf16": "" + "fp32": 452.05, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14540,7 +16252,7 @@ "Precisions": [ { "int4": "", - "int8": 1.09, + "int8": 3.6, "fp16": "", "fp32": "", "bf16": "" @@ -14552,19 +16264,23 @@ } }, { - "Platform": "Intel® Processor N100 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 105.81, + "int8": 881.04, "fp16": "", - "fp32": 49.62, - "bf16": "" + "fp32": 338.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14574,7 +16290,7 @@ "Precisions": [ { "int4": "", - "int8": 9.76, + "int8": 5.18, "fp16": "", "fp32": "", "bf16": "" @@ -14586,19 +16302,23 @@ } }, { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 575.93, + "int8": 1009.71, "fp16": "", - "fp32": 272.53, - "bf16": "" + "fp32": 562.38, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14608,7 +16328,7 @@ "Precisions": [ { "int4": "", - "int8": 2.33, + "int8": 4.28, "fp16": "", "fp32": "", "bf16": "" @@ -14620,19 +16340,23 @@ } }, { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 202.32, + "int8": "", "fp16": "", - "fp32": 120.89, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 20.78, + "token_int8": 14.18, + "token_fp16": 7.72 } ], "Unit": "FPS", @@ -14641,9 +16365,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 5.11, - "fp16": "", + "int4": 48.12, + "int8": 70.5, + "fp16": 129.51, "fp32": "", "bf16": "" } @@ -14654,19 +16378,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2068.2, + "int8": "", "fp16": "", - "fp32": 637.61, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 22.79, + "token_int8": 15.56, + "token_fp16": 8.48 } ], "Unit": "FPS", @@ -14675,9 +16403,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.66, - "fp16": "", + "int4": 43.86, + "int8": 64.26, + "fp16": 117.92, "fp32": "", "bf16": "" } @@ -14688,19 +16416,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6032.65, + "int8": "", "fp16": "", - "fp32": 1652.7, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 25.41, + "token_int8": 18.68, + "token_fp16": 10.61 } ], "Unit": "FPS", @@ -14709,9 +16441,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.23, - "fp16": "", + "int4": 39.34, + "int8": 53.51, + "fp16": 94.17, "fp32": "", "bf16": "" } @@ -14723,9 +16455,10 @@ }, { "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ @@ -14733,8 +16466,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 2525.11, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.07, + "token_int8": 17.66, + "token_fp16": 9.72 } ], "Unit": "FPS", @@ -14743,9 +16479,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 0.69, - "fp16": "", + "int4": 38.35, + "int8": 56.62, + "fp16": 102.88, "fp32": "", "bf16": "" } @@ -14756,19 +16492,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 23599.57, + "int8": "", "fp16": "", - "fp32": 3405.56, - "bf16": 12205.1 + "fp32": "", + "bf16": "", + "token_int4": 46.81, + "token_int8": 33.54, + "token_fp16": 19.32 } ], "Unit": "FPS", @@ -14777,11 +16517,11 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 0.74, - "fp16": "", + "int4": 21.36, + "int8": 29.81, + "fp16": 51.74, "fp32": "", - "bf16": 0.87 + "bf16": "" } ], "Unit": "ms", @@ -14790,19 +16530,23 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 14.73, "fp16": "", - "fp32": 4698.93, - "bf16": 16520.05 + "fp32": 3.42, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14812,10 +16556,10 @@ "Precisions": [ { "int4": "", - "int8": 0.83, + "int8": "", "fp16": "", "fp32": "", - "bf16": 1.2 + "bf16": "" } ], "Unit": "ms", @@ -14824,19 +16568,23 @@ } }, { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4267.65, + "int8": "", "fp16": "", - "fp32": 1238.3, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.89, + "token_int8": 18.54, + "token_fp16": 10.22 } ], "Unit": "FPS", @@ -14845,9 +16593,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.29, - "fp16": "", + "int4": 37.18, + "int8": 53.93, + "fp16": 97.8, "fp32": "", "bf16": "" } @@ -14858,19 +16606,23 @@ } }, { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4642.89, + "int8": 22703.47, "fp16": "", - "fp32": 1227.04, - "bf16": "" + "fp32": 6937.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14880,7 +16632,7 @@ "Precisions": [ { "int4": "", - "int8": 0.93, + "int8": 0.58, "fp16": "", "fp32": "", "bf16": "" @@ -14892,19 +16644,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 682.74, + "int8": "", "fp16": "", - "fp32": 243.93, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 39.41, + "token_int8": 29.28, + "token_fp16": 17.35 } ], "Unit": "FPS", @@ -14913,9 +16669,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 2.71, - "fp16": "", + "int4": 25.37, + "int8": 34.15, + "fp16": 57.61, "fp32": "", "bf16": "" } @@ -14926,19 +16682,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 533.14, + "int8": "", "fp16": "", - "fp32": 179.15, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 28.26, + "token_int8": 19.32, + "token_fp16": 10.27 } ], "Unit": "FPS", @@ -14947,9 +16707,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 3.35, - "fp16": "", + "int4": 35.38, + "int8": 51.74, + "fp16": 97.35, "fp32": "", "bf16": "" } @@ -14960,19 +16720,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 663.32, + "int8": 4874.95, "fp16": "", - "fp32": 226.12, - "bf16": "" + "fp32": 1144.73, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14982,7 +16746,7 @@ "Precisions": [ { "int4": "", - "int8": 2.25, + "int8": 1.07, "fp16": "", "fp32": "", "bf16": "" @@ -14994,19 +16758,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2300.75, - "fp16": 1706.66, - "fp32": "", - "bf16": "" + "int8": 84.6, + "fp16": "", + "fp32": 20.95, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15016,7 +16784,7 @@ "Precisions": [ { "int4": "", - "int8": 0.91, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -15028,19 +16796,23 @@ } }, { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 10174.18, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 2524.59, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15050,7 +16822,7 @@ "Precisions": [ { "int4": "", - "int8": 1.28, + "int8": 0.7, "fp16": "", "fp32": "", "bf16": "" @@ -15062,19 +16834,23 @@ } }, { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1024.85, - "fp16": 777.67, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15084,8 +16860,8 @@ "Precisions": [ { "int4": "", - "int8": 1.33, - "fp16": "", + "int8": 13.34, + "fp16": 13.66, "fp32": "", "bf16": "" } @@ -15096,19 +16872,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 704.76, - "fp16": 451.92, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 803.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15118,7 +16898,7 @@ "Precisions": [ { "int4": "", - "int8": 1.65, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -15130,19 +16910,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® Platinum 8380 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 705.61, - "fp16": 458.2, - "fp32": "", - "bf16": "" + "int8": 1704.08, + "fp16": "", + "fp32": 697.23, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15152,7 +16936,7 @@ "Precisions": [ { "int4": "", - "int8": 1.66, + "int8": 2.36, "fp16": "", "fp32": "", "bf16": "" @@ -15164,19 +16948,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 229.58, - "fp16": 198.86, - "fp32": "", - "bf16": "" + "int8": 3023.92, + "fp16": "", + "fp32": 483.11, + "bf16": 1976.63, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15186,10 +16974,10 @@ "Precisions": [ { "int4": "", - "int8": 7.67, + "int8": 3.79, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 4.84 } ], "Unit": "ms", @@ -15198,19 +16986,23 @@ } }, { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 210.98, - "fp16": 137.08, - "fp32": "", - "bf16": "" + "int8": 1445.78, + "fp16": "", + "fp32": 861.51, + "bf16": 1021.75, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15220,10 +17012,10 @@ "Precisions": [ { "int4": "", - "int8": 6.21, + "int8": 4.69, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 5.16 } ], "Unit": "ms", @@ -15232,19 +17024,23 @@ } }, { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 92.82, - "fp16": 95.83, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 22.71, + "token_int8": 16.83, + "token_fp16": 10.76 } ], "Unit": "FPS", @@ -15253,9 +17049,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 13.65, - "fp16": "", + "int4": 44.03, + "int8": 59.39, + "fp16": 92.87, "fp32": "", "bf16": "" } @@ -15266,19 +17062,23 @@ } }, { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 396.49, - "fp16": 222.23, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 23.7, + "token_int8": 16.93, + "token_fp16": 11.27 } ], "Unit": "FPS", @@ -15287,9 +17087,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 4.31, - "fp16": "", + "int4": 42.19, + "int8": 59.04, + "fp16": 88.67, "fp32": "", "bf16": "" } @@ -15300,19 +17100,23 @@ } }, { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 392.98, - "fp16": 229.59, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 26.11, + "token_int8": 20.1, + "token_fp16": 14.19 } ], "Unit": "FPS", @@ -15321,9 +17125,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 3.07, - "fp16": "", + "int4": 38.29, + "int8": 49.73, + "fp16": 70.45, "fp32": "", "bf16": "" } @@ -15334,19 +17138,23 @@ } }, { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 405.21, - "fp16": 238.92, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 26.02, + "token_int8": 18.97, + "token_fp16": 13.23 } ], "Unit": "FPS", @@ -15355,9 +17163,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 2.72, - "fp16": "", + "int4": 38.42, + "int8": 52.71, + "fp16": 75.57, "fp32": "", "bf16": "" } @@ -15368,19 +17176,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", "int8": "", - "fp16": 298.42, + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 45.68, + "token_int8": 36.96, + "token_fp16": 27.27 } ], "Unit": "FPS", @@ -15389,9 +17201,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 21.89, + "int8": 27.05, + "fp16": 36.67, "fp32": "", "bf16": "" } @@ -15402,19 +17214,23 @@ } }, { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 360.04, - "fp16": 225.51, - "fp32": "", - "bf16": "" + "int8": 62.13, + "fp16": "", + "fp32": 5.19, + "bf16": 37.54, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15424,10 +17240,10 @@ "Precisions": [ { "int4": "", - "int8": 3.42, + "int8": 58.49, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 81.95 } ], "Unit": "ms", @@ -15436,19 +17252,23 @@ } }, { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 748.06, - "fp16": 416.28, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 27.42, + "token_int8": 19.9, + "token_fp16": 13.72 } ], "Unit": "FPS", @@ -15457,9 +17277,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.86, - "fp16": "", + "int4": 36.46, + "int8": 50.24, + "fp16": 72.84, "fp32": "", "bf16": "" } @@ -15470,19 +17290,23 @@ } }, { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 488.8, - "fp16": 302.47, - "fp32": "", - "bf16": "" + "int8": 38538.65, + "fp16": "", + "fp32": 10274.08, + "bf16": 25608.67, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15492,10 +17316,10 @@ "Precisions": [ { "int4": "", - "int8": 2.51, + "int8": 0.65, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 0.66 } ], "Unit": "ms", @@ -15504,10 +17328,11 @@ } }, { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ @@ -15516,7 +17341,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": 33.53, + "token_fp16": 23.1 } ], "Unit": "FPS", @@ -15525,9 +17353,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 24.06, + "int8": 29.82, + "fp16": 43.29, "fp32": "", "bf16": "" } @@ -15538,19 +17366,23 @@ } }, { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 164.35, - "fp16": 106.86, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 30.03, + "token_int8": 22.14, + "token_fp16": 13.95 } ], "Unit": "FPS", @@ -15559,9 +17391,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 7.36, - "fp16": "", + "int4": 33.3, + "int8": 45.16, + "fp16": 71.68, "fp32": "", "bf16": "" } @@ -15572,19 +17404,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 979.63, - "fp16": 802.16, - "fp32": "", - "bf16": "" + "int8": 19226.96, + "fp16": "", + "fp32": 1597.37, + "bf16": 7480.12, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15594,10 +17430,10 @@ "Precisions": [ { "int4": "", - "int8": 1.15, + "int8": 1.01, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 1.25 } ], "Unit": "ms", @@ -15606,9505 +17442,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": 974.05, - "fp16": 701.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 985.08, - "fp16": 990.93, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.87, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 234.86, - "fp16": "", - "fp32": 116.72, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 111.52, - "fp16": "", - "fp32": 98.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 318.36, - "fp16": "", - "fp32": 194.83, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 117.54, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 471.49, - "fp16": "", - "fp32": 189.58, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 650.89, - "fp16": "", - "fp32": 298.25, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 160.27, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 380.11, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 554.71, - "fp16": "", - "fp32": 229.14, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 350.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 217.21, - "fp16": "", - "fp32": 95.95, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1045.56, - "fp16": "", - "fp32": 608.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 748.16, - "fp16": "", - "fp32": 419.13, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1.49, - "fp16": "", - "fp32": 0.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 672.94, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.54, - "fp16": "", - "fp32": 1.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 403.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.62, - "fp16": "", - "fp32": 2.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 306.85, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.8, - "fp16": "", - "fp32": 0.82, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 348.07, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4.66, - "fp16": "", - "fp32": 0.83, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 310.23, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.67, - "fp16": "", - "fp32": 3.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 93.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.95, - "fp16": "", - "fp32": 2.06, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 247.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.54, - "fp16": "", - "fp32": 1.66, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 154.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4.97, - "fp16": "", - "fp32": 1.24, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 206.27, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 9.59, - "fp16": "", - "fp32": 2.4, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 121.76, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 5.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 292.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.22, - "fp16": "", - "fp32": 1.55, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 178.96, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.06, - "fp16": "", - "fp32": 1.66, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 254.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.79, - "fp16": "", - "fp32": 1.91, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 282.74, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.74, - "fp16": "", - "fp32": 4.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 72.32, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.13, - "fp16": "", - "fp32": 3.29, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 174.28, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.45, - "fp16": "", - "fp32": 1.35, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 415.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 28.98, - "fp16": "", - "fp32": 7.41, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 69.96, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 96.65, - "fp16": "", - "fp32": 22.64, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 23.08, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 139.91, - "fp16": "", - "fp32": 33.26, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 15.34, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 511.53, - "fp16": "", - "fp32": 48.78, - "bf16": 276.34 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.66, - "fp16": "", - "fp32": "", - "bf16": 9.97 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 617.57, - "fp16": "", - "fp32": 56.93, - "bf16": 334.64 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.71, - "fp16": "", - "fp32": "", - "bf16": 7.81 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 69.09, - "fp16": "", - "fp32": 16.42, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 29.59, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 69.98, - "fp16": "", - "fp32": 16.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 30.05, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.9, - "fp16": "", - "fp32": 1.98, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 152.81, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.79, - "fp16": "", - "fp32": 1.6, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 179.19, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.92, - "fp16": "", - "fp32": 2.48, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 185.72, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 276.55, - "fp16": 194.61, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.54, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 55.61, - "fp16": 34.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.05, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 9.86, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 9.91, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 46.18, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.4, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 294.79, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1.98, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 505.59, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.29, - "fp16": 4.48, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 121.49, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 10.02, - "fp16": 4.68, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 79.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 9.86, - "fp16": 4.73, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 82.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 14.62, - "fp16": 7.64, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 63.94, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 5.13, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 93.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.44, - "fp16": 9.72, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 54.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.84, - "fp16": 6.49, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 68.04, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.56, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 390.36, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 36.35, - "fp16": 23.49, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 25.29, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.67, - "fp16": 18.53, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 105.25, - "fp16": 74.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.37, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.85, - "fp16": "", - "fp32": 4.55, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 1.47, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.38, - "fp16": "", - "fp32": 2.41, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 15.79, - "fp16": "", - "fp32": 6.34, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.73, - "fp16": "", - "fp32": 2.82, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 6.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 9.45, - "fp16": "", - "fp32": 3.58, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 5.04, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.21, - "fp16": "", - "fp32": 16.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 7.99, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 5.65, - "fp16": "", - "fp32": 2.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 182.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.58, - "fp16": "", - "fp32": 1.04, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 393.57, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.38, - "fp16": "", - "fp32": 1.86, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 158.93, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.59, - "fp16": "", - "fp32": 6.77, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 80.57, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.57, - "fp16": "", - "fp32": 9.33, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 57.66, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.83, - "fp16": "", - "fp32": 5.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 76.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 23.43, - "fp16": "", - "fp32": 7.32, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 70.0, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 66.59, - "fp16": "", - "fp32": 20.16, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.62, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 19.4, - "fp16": "", - "fp32": 10.16, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 52.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 27.81, - "fp16": "", - "fp32": 7.85, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 37.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 20.29, - "fp16": "", - "fp32": 5.69, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 49.78, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 49.8, - "fp16": "", - "fp32": 14.8, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 25.53, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 25.87, - "fp16": "", - "fp32": 8.21, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 65.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 37.03, - "fp16": "", - "fp32": 10.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 36.87, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 15.32, - "fp16": "", - "fp32": 7.94, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 50.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 19.78, - "fp16": "", - "fp32": 9.0, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 54.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 95.49, - "fp16": "", - "fp32": 30.36, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 15.47, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.13, - "fp16": "", - "fp32": 2.62, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 167.91, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 32.67, - "fp16": "", - "fp32": 14.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 34.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.06, - "fp16": "", - "fp32": 6.75, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 83.73, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 117.81, - "fp16": "", - "fp32": 35.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.72, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 334.59, - "fp16": "", - "fp32": 102.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.17, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 533.18, - "fp16": "", - "fp32": 154.64, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.76, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1419.98, - "fp16": "", - "fp32": 214.44, - "bf16": 830.31 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.62, - "fp16": "", - "fp32": "", - "bf16": 5.98 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1755.21, - "fp16": "", - "fp32": 265.47, - "bf16": 1055.23 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.46, - "fp16": "", - "fp32": "", - "bf16": 5.21 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 251.62, - "fp16": "", - "fp32": 74.96, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 265.17, - "fp16": "", - "fp32": 76.67, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.63, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 44.74, - "fp16": "", - "fp32": 12.8, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 36.67, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 33.1, - "fp16": "", - "fp32": 9.15, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 45.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 288.05, - "fp16": 333.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.92, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 105.43, - "fp16": 93.59, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.01, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 50.31, - "fp16": 40.99, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 21.31, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 50.34, - "fp16": 41.31, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 21.03, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 14.53, - "fp16": 9.52, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 71.11, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.37, - "fp16": 7.57, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 141.78, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.36, - "fp16": 17.91, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 34.6, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 33.47, - "fp16": 18.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 24.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 35.33, - "fp16": 18.84, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 24.68, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 48.16, - "fp16": 28.25, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 22.07, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 33.73, - "fp16": 18.76, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 29.57, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 63.37, - "fp16": 36.52, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 17.02, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 44.64, - "fp16": 24.85, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 20.43, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 53.52, - "fp16": 30.53, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 17.5, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 11.12, - "fp16": 7.41, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 91.66, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 132.43, - "fp16": 70.84, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 7.23, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 120.91, - "fp16": 58.45, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.06, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 17.32, - "fp16": "", - "fp32": 7.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.67, - "fp16": "", - "fp32": 7.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 32.84, - "fp16": "", - "fp32": 18.05, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 7.54, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 32.16, - "fp16": "", - "fp32": 12.68, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 55.36, - "fp16": "", - "fp32": 25.15, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 29.16, - "fp16": "", - "fp32": 11.65, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 24.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 39.77, - "fp16": "", - "fp32": 15.13, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 52.61, - "fp16": "", - "fp32": 20.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 14.47, - "fp16": "", - "fp32": 4.84, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 120.56, - "fp16": "", - "fp32": 58.53, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 70.85, - "fp16": "", - "fp32": 32.17, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 21.65, - "fp16": "", - "fp32": 11.8, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 47.52, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 10.28, - "fp16": "", - "fp32": 5.11, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 100.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 25.98, - "fp16": "", - "fp32": 9.65, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 40.09, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-8100 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 53.64, - "fp16": "", - "fp32": 32.13, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.04, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-10500TE CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 81.71, - "fp16": "", - "fp32": 45.23, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 13.68, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 70.36, - "fp16": "", - "fp32": 27.96, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.88, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 91.48, - "fp16": "", - "fp32": 36.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 18.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-13600K CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 272.16, - "fp16": "", - "fp32": 103.27, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.12, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-8500 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 84.24, - "fp16": "", - "fp32": 49.32, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 111.42, - "fp16": "", - "fp32": 40.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 76.56, - "fp16": "", - "fp32": 27.61, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 13.37, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 200.97, - "fp16": "", - "fp32": 75.16, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.52, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 101.43, - "fp16": "", - "fp32": 40.83, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 16.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.34, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-8700T CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 70.91, - "fp16": "", - "fp32": 40.94, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-10900TE CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 89.73, - "fp16": "", - "fp32": 48.88, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 12.74, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 387.96, - "fp16": "", - "fp32": 154.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.11, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 23.66, - "fp16": "", - "fp32": 12.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 43.5, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® W1290P CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 136.65, - "fp16": "", - "fp32": 71.98, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® E-2124G CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 51.95, - "fp16": "", - "fp32": 32.5, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.42, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 5218T CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 442.04, - "fp16": "", - "fp32": 173.4, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.02, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8280 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 997.47, - "fp16": "", - "fp32": 452.11, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1718.36, - "fp16": "", - "fp32": 701.27, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.39, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2411.67, - "fp16": "", - "fp32": 959.63, - "bf16": 2381.88 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.11, - "fp16": "", - "fp32": "", - "bf16": 2.53 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 1262.79, - "bf16": 3461.09 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.15, - "fp16": "", - "fp32": "", - "bf16": 2.62 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238L CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 757.45, - "fp16": "", - "fp32": 337.98, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.31, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Silver 4316 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 859.9, - "fp16": "", - "fp32": 340.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 166.62, - "fp16": "", - "fp32": 64.56, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 12.93, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 127.76, - "fp16": "", - "fp32": 46.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 14.67, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 139.58, - "fp16": "", - "fp32": 59.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.46, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 803.05, - "fp16": 733.02, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.32, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 322.1, - "fp16": 299.84, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 126.74, - "fp16": 129.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H NPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 127.62, - "fp16": 131.24, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V NPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 390.98, - "fp16": 506.71, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.51, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 60.91, - "fp16": 44.63, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 18.54, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.72, - "fp16": 33.09, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 35.85, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 121.95, - "fp16": 81.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.32, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 130.79, - "fp16": 80.07, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.12, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 133.28, - "fp16": 83.52, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.92, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 164.13, - "fp16": 112.15, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 117.62, - "fp16": 77.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.6, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 218.62, - "fp16": 149.58, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.56, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 164.07, - "fp16": 107.64, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.83, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 47.04, - "fp16": 34.98, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 23.03, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 388.85, - "fp16": 291.51, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.61, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 386.42, - "fp16": 271.08, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.16, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 442.05, - "fp16": 412.46, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.33, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® x7425E CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 68.47, - "fp16": "", - "fp32": 36.85, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Atom® X6425E CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 36.31, - "fp16": "", - "fp32": 34.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Celeron® 6305E CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 114.66, - "fp16": "", - "fp32": 78.23, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 36.38, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 126.5, - "fp16": "", - "fp32": 58.24, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 184.9, - "fp16": "", - "fp32": 103.88, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 100.37, - "fp16": "", - "fp32": 51.71, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-12700H CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 117.6, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 154.41, - "fp16": "", - "fp32": 72.66, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 227.78, - "fp16": "", - "fp32": 103.62, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Processor N100 CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Atom™, CPU+iGPU", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 61.09, - "fp16": "", - "fp32": 28.95, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 155H CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 369.1, - "fp16": "", - "fp32": 228.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 253.52, - "fp16": "", - "fp32": 149.21, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 15.4, - "int8": 11.0, - "fp16": 5.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 64.94, - "int8": 90.95, - "fp16": 176.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 30.9, - "int8": 22.8, - "fp16": 12.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 32.39, - "int8": 43.95, - "fp16": 78.58, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 33.7, - "int8": 25.7, - "fp16": 16.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 29.69, - "int8": 38.85, - "fp16": 61.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 39.5, - "int8": 29.1, - "fp16": 18.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 25.3, - "int8": 34.31, - "fp16": 53.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.4, - "int8": 6.2, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 106.9, - "int8": 162.58, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 56.8, - "int8": 45.2, - "fp16": 26.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 17.61, - "int8": 22.14, - "fp16": 37.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 45.5, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 21.99, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.4, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 87.86, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 15.8, - "int8": 11.3, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 63.49, - "int8": 88.47, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 18.0, - "int8": 12.6, - "fp16": 7.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 55.41, - "int8": 79.46, - "fp16": 142.26, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.8, - "int8": 9.4, - "fp16": 4.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 67.35, - "int8": 106.1, - "fp16": 205.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 28.4, - "int8": 19.7, - "fp16": 10.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 35.25, - "int8": 50.64, - "fp16": 91.76, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 30.6, - "int8": 21.6, - "fp16": 14.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 32.72, - "int8": 46.36, - "fp16": 71.16, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 35.7, - "int8": 24.3, - "fp16": 15.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 28.02, - "int8": 41.12, - "fp16": 62.78, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.8, - "int8": 5.4, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 102.44, - "int8": 186.87, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 44.7, - "int8": 32.1, - "fp16": 21.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 22.37, - "int8": 31.17, - "fp16": 47.04, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 34.7, - "int8": 25.8, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 28.79, - "int8": 38.77, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.7, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 174.81, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 6.5, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 153.92, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.2, - "int8": 9.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 70.34, - "int8": 102.9, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 17.6, - "int8": 10.8, - "fp16": 6.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 56.96, - "int8": 92.92, - "fp16": 164.05, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 13.4, - "int8": 9.3, - "fp16": 5.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 74.6, - "int8": 107.69, - "fp16": 201.87, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 25.2, - "int8": 18.4, - "fp16": 10.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 39.73, - "int8": 54.31, - "fp16": 94.63, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 26.7, - "int8": 20.3, - "fp16": 14.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 37.4, - "int8": 49.27, - "fp16": 70.22, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 32.5, - "int8": 23.1, - "fp16": 16.4, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 30.75, - "int8": 43.23, - "fp16": 60.97, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 8.6, - "int8": 5.3, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 116.29, - "int8": 189.6, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 45.3, - "int8": 35.9, - "fp16": 21.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 22.08, - "int8": 27.87, - "fp16": 45.77, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 38.1, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 26.27, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.4, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 184.58, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 13.3, - "int8": 9.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 75.47, - "int8": 109.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 15.6, - "int8": 10.4, - "fp16": 5.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 64.18, - "int8": 95.84, - "fp16": 171.66, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.8, - "int8": 8.7, - "fp16": 4.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 84.87, - "int8": 115.0, - "fp16": 223.46, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 26.1, - "int8": 17.6, - "fp16": 9.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 38.25, - "int8": 56.78, - "fp16": 102.16, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 25.3, - "int8": 19.3, - "fp16": 13.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 39.49, - "int8": 51.8, - "fp16": 75.23, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 28.9, - "int8": 21.2, - "fp16": 15.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 34.63, - "int8": 47.15, - "fp16": 66.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 4.8, - "int8": 4.9, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 209.01, - "int8": 205.41, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 45.6, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 21.92, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 35.3, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 28.33, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.7, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 175.65, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.3, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 70.04, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 17.0, - "int8": 9.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 58.82, - "int8": 102.99, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.4, - "int8": 9.1, - "fp16": 4.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 69.6, - "int8": 109.3, - "fp16": 212.27, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 27.1, - "int8": 18.3, - "fp16": 10.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 36.86, - "int8": 54.59, - "fp16": 97.42, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 27.8, - "int8": 20.0, - "fp16": 13.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 35.91, - "int8": 49.99, - "fp16": 72.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 33.0, - "int8": 22.7, - "fp16": 15.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 30.3, - "int8": 44.07, - "fp16": 63.31, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.0, - "int8": 5.0, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 111.21, - "int8": 201.82, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 46.0, - "int8": 34.4, - "fp16": 21.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 21.76, - "int8": 29.06, - "fp16": 47.66, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 38.0, - "int8": "", - "fp16": 17.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 26.33, - "int8": "", - "fp16": 55.81, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.1, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 90.09, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.8, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 171.03, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.6, - "int8": 9.3, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 68.27, - "int8": 108.02, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 16.7, - "int8": 10.2, - "fp16": 5.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 60.02, - "int8": 98.33, - "fp16": 176.33, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 23.3, - "int8": 15.6, - "fp16": 8.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 42.95, - "int8": 64.22, - "fp16": 117.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 40.8, - "int8": 29.1, - "fp16": 17.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 24.53, - "int8": 34.38, - "fp16": 58.33, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 43.6, - "int8": 33.3, - "fp16": 22.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 22.94, - "int8": 30.03, - "fp16": 45.44, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 52.2, - "int8": 39.0, - "fp16": 25.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 19.15, - "int8": 25.66, - "fp16": 38.58, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.6, - "int8": 8.6, - "fp16": 6.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 86.42, - "int8": 116.68, - "fp16": 145.89, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 170 dGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 55.6, - "int8": 46.6, - "fp16": 31.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 17.98, - "int8": 21.47, - "fp16": 31.63, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 41.9, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 23.88, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 18.8, - "int8": 10.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 53.33, - "int8": 93.53, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.7, - "int8": 6.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 102.87, - "int8": 165.24, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 165H iGPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 23.1, - "int8": 15.7, - "fp16": 9.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 43.28, - "int8": 63.82, - "fp16": 109.36, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 25.6, - "int8": 16.7, - "fp16": 10.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 38.99, - "int8": 59.98, - "fp16": 96.89, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-13900K CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 42.87, - "fp16": 41.99, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8380 CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 14.24, - "fp16": 14.44, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.67, - "fp16": 4.55, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8580 CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Xeon®, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.13, - "fp16": 4.04, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Arc™ A-Series Graphics dGPU", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.55, - "fp16": 2.47, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Data Center GPU Flex 140 dGPU", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.8, - "fp16": 10.89, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-1360P iGPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 33.5, - "fp16": 32.47, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 9 processor 288V iGPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.67, - "fp16": 5.33, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 43.86, - "fp16": "", - "fp32": 19.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 30.78, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4.63, - "fp16": "", - "fp32": 1.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 281.45, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 67.33, - "fp16": "", - "fp32": 45.22, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 18.61, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 0.83, - "fp16": "", - "fp32": 0.2, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1662.21, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1206.23, - "fp16": "", - "fp32": 587.85, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.27, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 248.53, - "fp16": "", - "fp32": 60.14, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.96, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 510.87, - "fp16": "", - "fp32": 166.45, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.76, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4.55, - "fp16": "", - "fp32": 1.1, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 310.48, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.44, - "fp16": "", - "fp32": 1.82, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 213.98, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.79, - "fp16": "", - "fp32": 8.85, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 42.2, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, CPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 116.4, - "fp16": "", - "fp32": 42.84, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.51, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 279.89, - "fp16": 242.6, - "fp32": "", - "bf16": "" + "int4": "", + "int8": 434.12, + "fp16": "", + "fp32": 30.6, + "bf16": 209.11, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25114,7 +17468,7 @@ "Precisions": [ { "int4": "", - "int8": 4.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -25126,19 +17480,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 43.58, - "fp16": 40.08, - "fp32": "", - "bf16": "" + "int8": 24134.02, + "fp16": "", + "fp32": 3392.4, + "bf16": 12168.49, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25148,10 +17506,10 @@ "Precisions": [ { "int4": "", - "int8": 20.29, + "int8": 0.74, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 0.89 } ], "Unit": "ms", @@ -25160,19 +17518,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 184.73, - "fp16": 167.81, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25182,8 +17544,8 @@ "Precisions": [ { "int4": "", - "int8": 7.43, - "fp16": "", + "int8": 4.62, + "fp16": 4.55, "fp32": "", "bf16": "" } @@ -25194,19 +17556,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "yolo11", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.7, - "fp16": 5.04, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 1034.68, + "bf16": 2068.81, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25216,7 +17582,7 @@ "Precisions": [ { "int4": "", - "int8": 106.68, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -25228,19 +17594,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1530.29, - "fp16": 1450.39, - "fp32": "", - "bf16": "" + "int8": 2380.51, + "fp16": "", + "fp32": 950.6, + "bf16": 2374.89, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25250,10 +17620,10 @@ "Precisions": [ { "int4": "", - "int8": 1.42, + "int8": 3.13, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 2.54 } ], "Unit": "ms", @@ -25262,19 +17632,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1023.16, - "fp16": 466.9, - "fp32": "", - "bf16": "" + "int8": 4671.04, + "fp16": "", + "fp32": 560.3, + "bf16": 3211.93, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25284,10 +17658,10 @@ "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 3.66, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 4.77 } ], "Unit": "ms", @@ -25296,19 +17670,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1114.16, - "fp16": 946.07, - "fp32": "", - "bf16": "" + "int8": 1725.13, + "fp16": "", + "fp32": 1123.04, + "bf16": 1407.69, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25318,10 +17696,10 @@ "Precisions": [ { "int4": "", - "int8": 2.05, + "int8": 4.71, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 4.84 } ], "Unit": "ms", @@ -25330,19 +17708,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 45.06, - "fp16": 25.61, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 25.46, + "token_int8": 18.96, + "token_fp16": 12.14 } ], "Unit": "FPS", @@ -25351,9 +17733,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 21.18, - "fp16": "", + "int4": 39.27, + "int8": 52.74, + "fp16": 82.36, "fp32": "", "bf16": "" } @@ -25364,19 +17746,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 68.61, - "fp16": 48.46, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 27.1, + "token_int8": 19.33, + "token_fp16": 12.69 } ], "Unit": "FPS", @@ -25385,9 +17771,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 12.01, - "fp16": "", + "int4": 36.9, + "int8": 51.72, + "fp16": 78.77, "fp32": "", "bf16": "" } @@ -25398,19 +17784,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 112.66, - "fp16": 119.27, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 31.71, + "token_int8": 23.05, + "token_fp16": 16.64 } ], "Unit": "FPS", @@ -25419,9 +17809,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 7.63, - "fp16": "", + "int4": 31.53, + "int8": 43.37, + "fp16": 60.07, "fp32": "", "bf16": "" } @@ -25432,19 +17822,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 326.29, - "fp16": 320.34, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 30.06, + "token_int8": 21.73, + "token_fp16": 14.93 } ], "Unit": "FPS", @@ -25453,9 +17847,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 4.17, - "fp16": "", + "int4": 33.26, + "int8": 46.01, + "fp16": 66.97, "fp32": "", "bf16": "" } @@ -25466,19 +17860,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 211.04, + "int8": "", "fp16": "", - "fp32": 165.57, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 54.73, + "token_int8": 42.58, + "token_fp16": 31.51 } ], "Unit": "FPS", @@ -25487,9 +17885,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 18.27, + "int8": 23.48, + "fp16": 31.73, "fp32": "", "bf16": "" } @@ -25500,19 +17898,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 34.98, + "int8": 74.86, "fp16": "", - "fp32": 28.58, - "bf16": "" + "fp32": 6.39, + "bf16": 48.32, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25522,10 +17924,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 55.7, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 73.74 } ], "Unit": "ms", @@ -25534,19 +17936,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 166.87, + "int8": "", "fp16": "", - "fp32": 144.55, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 33.27, + "token_int8": 22.24, + "token_fp16": 15.74 } ], "Unit": "FPS", @@ -25555,9 +17961,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 30.05, + "int8": 44.96, + "fp16": 63.51, "fp32": "", "bf16": "" } @@ -25568,19 +17974,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.73, + "int8": 39894.55, "fp16": "", - "fp32": 3.15, - "bf16": "" + "fp32": 15839.75, + "bf16": 29419.55, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25590,10 +18000,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.84, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 0.72 } ], "Unit": "ms", @@ -25602,19 +18012,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1609.77, + "int8": "", "fp16": "", - "fp32": 1160.68, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 40.45, + "token_fp16": 26.95 } ], "Unit": "FPS", @@ -25623,8 +18037,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 21.43, + "int8": 24.72, "fp16": "", "fp32": "", "bf16": "" @@ -25636,19 +18050,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 738.31, + "int8": "", "fp16": "", - "fp32": 365.53, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 35.48, + "token_int8": 25.7, + "token_fp16": 16.1 } ], "Unit": "FPS", @@ -25657,9 +18075,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 28.18, + "int8": 38.91, + "fp16": 62.09, "fp32": "", "bf16": "" } @@ -25670,19 +18088,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 911.79, + "int8": 21612.82, "fp16": "", - "fp32": 627.75, - "bf16": "" + "fp32": 2002.36, + "bf16": 13669.05, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25692,10 +18114,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.0, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 1.37 } ], "Unit": "ms", @@ -25704,19 +18126,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 31.73, + "int8": 513.09, "fp16": "", - "fp32": 16.1, - "bf16": "" + "fp32": 35.2, + "bf16": 275.94, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25738,19 +18164,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 51.44, + "int8": 26748.89, "fp16": "", - "fp32": 30.42, - "bf16": "" + "fp32": 4718.18, + "bf16": 16684.87, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25760,10 +18190,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.72, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 1.15 } ], "Unit": "ms", @@ -25772,19 +18202,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 102.09, + "int8": "", "fp16": "", - "fp32": 85.74, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25794,8 +18228,8 @@ "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 4.09, + "fp16": 3.99, "fp32": "", "bf16": "" } @@ -25806,19 +18240,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "yolo11", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 315.12, + "int8": "", "fp16": "", - "fp32": 248.34, - "bf16": "" + "fp32": 1455.5, + "bf16": 2962.49, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25831,7 +18269,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": 3.19 } ], "Unit": "ms", @@ -25840,19 +18278,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® Platinum 8580 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 179.86, - "fp16": 141.28, - "fp32": "", - "bf16": "" + "int8": 3043.23, + "fp16": "", + "fp32": 1258.2, + "bf16": 3444.22, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25862,10 +18304,10 @@ "Precisions": [ { "int4": "", - "int8": 6.54, + "int8": 3.08, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 2.56 } ], "Unit": "ms", @@ -25874,19 +18316,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 31.04, + "int8": 8897.30, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1217.03, + "bf16": 6414.49, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25896,9 +18342,9 @@ "Precisions": [ { "int4": "", - "int8": 42.23, + "int8": 7.74, "fp16": "", - "fp32": "", + "fp32": 14.8, "bf16": "" } ], @@ -25908,19 +18354,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", + "Platform": "Intel® Xeon® 6979P CPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.96, - "fp16": 13.4, - "fp32": "", - "bf16": "" + "int8": 3384.23, + "fp16": "", + "fp32": 2295.4, + "bf16": 2872.84, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25930,9 +18380,9 @@ "Precisions": [ { "int4": "", - "int8": 125.1, + "int8": 9.71, "fp16": "", - "fp32": "", + "fp32": 9.43, "bf16": "" } ], @@ -25942,19 +18392,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", + "Platform": "Intel® Xeon® 6979P CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 149.52, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 11.97, + "bf16": 91.85, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25964,9 +18418,9 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 74.6, "fp16": "", - "fp32": "", + "fp32": 248.21, "bf16": "" } ], @@ -25976,19 +18430,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", + "Platform": "Intel® Xeon® 6979P CPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4151.21, - "fp16": 3149.73, - "fp32": "", - "bf16": "" + "int8": 32737.09, + "fp16": "", + "fp32": 25621.92, + "bf16": 26297.21, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25998,9 +18456,9 @@ "Precisions": [ { "int4": "", - "int8": 0.42, + "int8": 1.65, "fp16": "", - "fp32": "", + "fp32": 1.34, "bf16": "" } ], @@ -26010,19 +18468,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", + "Platform": "Intel® Xeon® 6979P CPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2038.58, - "fp16": 835.87, - "fp32": "", - "bf16": "" + "int8": 27670.82, + "fp16": "", + "fp32": 4254.94, + "bf16": 22432.74, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26032,9 +18494,9 @@ "Precisions": [ { "int4": "", - "int8": 0.77, + "int8": 2.28, "fp16": "", - "fp32": "", + "fp32": 3.69, "bf16": "" } ], @@ -26044,19 +18506,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 229.76, - "fp16": 200.09, - "fp32": "", - "bf16": "" + "int8": 1009.62, + "fp16": "", + "fp32": 77.99, + "bf16": 532.90, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26066,9 +18532,9 @@ "Precisions": [ { "int4": "", - "int8": 8.16, + "int8": 9.73, "fp16": "", - "fp32": "", + "fp32": 34.1, "bf16": "" } ], @@ -26078,19 +18544,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 29674.40, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 9800.83, + "bf16": 19479.18, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26100,9 +18570,9 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.08, "fp16": "", - "fp32": "", + "fp32": 2.45, "bf16": "" } ], @@ -26112,19 +18582,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": 37.1, - "fp32": "", - "bf16": "" + "int8": 5590.87, + "fp16": "", + "fp32": 2699.0, + "bf16": 6003.66, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26134,9 +18608,9 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 6.56, "fp16": "", - "fp32": "", + "fp32": 5.59, "bf16": "" } ], @@ -26146,30 +18620,34 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 129.88, - "fp16": 105.29, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 136.4, + "token_int8": "", + "token_fp16": 53.6 } ], - "Unit": "FPS", + "Unit": "Tokens/sec", "UnitDesc": "higher is better" }, "latency": { "Precisions": [ { - "int4": "", - "int8": 9.44, - "fp16": "", + "int4": 7.3, + "int8": "", + "fp16": 18.7, "fp32": "", "bf16": "" } @@ -26180,30 +18658,34 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V NPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Intel® Core™, NPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 399.28, - "fp16": 469.58, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 116.5, + "token_int8": "", + "token_fp16": 51.9 } ], - "Unit": "FPS", + "Unit": "Tokens/sec", "UnitDesc": "higher is better" }, "latency": { "Precisions": [ { - "int4": "", - "int8": 3.94, - "fp16": "", + "int4": 8.6, + "int8": "", + "fp16": 19, "fp32": "", "bf16": "" } @@ -26214,19 +18696,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 20.6, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 139.5, + "token_int8": "", + "token_fp16": 132 } ], "Unit": "Tokens/sec", @@ -26235,9 +18721,9 @@ "latency": { "Precisions": [ { - "int4": 48, + "int4": 7.2, "int8": "", - "fp16": "", + "fp16": 7.6, "fp32": "", "bf16": "" } @@ -26248,19 +18734,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 18.8, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 272.7, + "token_int8": 65, + "token_fp16": "" } ], "Unit": "Tokens/sec", @@ -26269,10 +18759,10 @@ "latency": { "Precisions": [ { - "int4": 53, + "int4": 3.7, "int8": "", "fp16": "", - "fp32": "", + "fp32": 15.4, "bf16": "" } ], @@ -26282,19 +18772,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 17.5, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 148.2, + "token_int8": "", + "token_fp16": 57.2 } ], "Unit": "Tokens/sec", @@ -26303,9 +18797,9 @@ "latency": { "Precisions": [ { - "int4": 57, + "int4": 6.7, "int8": "", - "fp16": "", + "fp16": 17.5, "fp32": "", "bf16": "" } @@ -26316,19 +18810,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 19.2, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 126.4, + "token_int8": "", + "token_fp16": 61.4 } ], "Unit": "Tokens/sec", @@ -26337,9 +18835,9 @@ "latency": { "Precisions": [ { - "int4": 52, + "int4": 7.9, "int8": "", - "fp16": "", + "fp16": 16.3, "fp32": "", "bf16": "" } @@ -26350,19 +18848,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 18.2, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 176.6, + "token_int8": "", + "token_fp16": 111.9 } ], "Unit": "Tokens/sec", @@ -26371,9 +18873,9 @@ "latency": { "Precisions": [ { - "int4": 54, + "int4": 5.7, "int8": "", - "fp16": "", + "fp16": 8.9, "fp32": "", "bf16": "" } @@ -26384,19 +18886,23 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "Platform": "Intel® Xeon® 6979P CPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 29.5, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 164.4, + "token_int8": "", + "token_fp16": 62.2 } ], "Unit": "Tokens/sec", @@ -26405,9 +18911,9 @@ "latency": { "Precisions": [ { - "int4": 34, + "int4": 6.1, "int8": "", - "fp16": "", + "fp16": 16.1, "fp32": "", "bf16": "" } @@ -26418,10 +18924,11 @@ } }, { - "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU", + "Platform": "Intel® Xeon® 6979P CPU-only", "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Intel® Core™, iGPU-only", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { "throughput": { "Precisions": [ @@ -26430,7 +18937,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "n/a", @@ -26440,8 +18950,8 @@ "Precisions": [ { "int4": "", - "int8": 7.2, - "fp16": 6.9, + "int8": 4.0, + "fp16": 4.1, "fp32": "", "bf16": "" } diff --git a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json index 29fbe714094b74..e5fe953b72bca1 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json +++ b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json @@ -7,6 +7,9 @@ "BF16": "bf16", "FP32_OV": "fp32_ov", "FP32_OVMS": "fp32_ovms", + "TOKEN_INT4": "token_int4", + "TOKEN_INT8": "token_int8", + "TOKEN_FP16": "token_fp16", "INT8_OV": "int8_ov", "INT8_OVMS": "int8_ovms", "THROUGHPUT": "Throughput", @@ -73,6 +76,21 @@ "color": "#00536a", "label": "INT8 OVMS" }, + "token_int4": { + "data": null, + "color": "#00536a", + "label": "T INT4" + }, + "token_int8": { + "data": null, + "color": "#00636a", + "label": "T INT8" + }, + "token_fp16": { + "data": null, + "color": "#00736a", + "label": "T INT16" + }, "Vllm": { "Latency": { "data": null, diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-155H.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-155H.csv new file mode 100644 index 00000000000000..0d16c5c4998329 --- /dev/null +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-155H.csv @@ -0,0 +1,156 @@ +Topology,Precision,Input Size,max rss memory,1st latency (ms),2nd latency (ms),2nd tok/sec +opt-125m-gptq,INT4-MIXED,32,965.9,29,7.7,129.87 +opt-125m-gptq,INT4-MIXED,1024,1507.9,113.1,7.8,128.21 +tiny-llama-1.1b-chat,INT4-MIXED,32,1831.8,46.5,16.7,59.88 +tiny-llama-1.1b-chat,INT4-MIXED,1024,1806.3,635,17.8,56.18 +qwen2-0.5b,INT4-MIXED,32,2551.7,61.4,18.3,54.64 +qwen2-0.5b,INT4-MIXED,1024,2976.6,356.1,19.2,52.08 +tiny-llama-1.1b-chat,INT8-CW,32,1987.4,56,21.6,46.30 +tiny-llama-1.1b-chat,INT8-CW,1024,2209.1,772.7,22.6,44.25 +qwen2-0.5b,INT8-CW,32,2484.9,57.3,22.8,43.86 +qwen2-0.5b,INT8-CW,1024,3102.5,407.1,23.9,41.84 +qwen2-1.5b,INT4-MIXED,32,4265.2,71.7,25.5,39.22 +qwen2-1.5b,INT4-MIXED,1024,4884.5,862.4,26.8,37.31 +dolly-v2-3b,INT4-MIXED,32,2401.3,89.6,27.5,36.36 +red-pajama-incite-chat-3b-v1,INT4-MIXED,32,2511.5,78.6,28.2,35.46 +phi-2,INT4-MIXED,32,2279.5,95.7,29.1,34.36 +minicpm-1b-sft,INT4-MIXED,31,2759.9,104.4,30.9,32.36 +phi-2,INT4-MIXED,32,2620.1,100.8,31,32.26 +stable-zephyr-3b-dpo,INT4-MIXED,30,2636.5,86.8,31.7,31.55 +dolly-v2-3b,INT4-MIXED,1024,3137.1,1782.9,32.2,31.06 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1020,3118.5,1831.7,33.3,30.03 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1024,2862.7,1821.1,33.5,29.85 +qwen2-1.5b,INT8-CW,32,4831.2,87,33.8,29.59 +opt-2.7b,INT4-MIXED,31,2898.3,73.2,33.9,29.50 +phi-2,INT4-MIXED,1024,2797.4,1887,34,29.41 +orca-mini-3b,INT4-MIXED,32,2877.8,100.3,35,28.57 +stablelm-3b-4e1t,INT4-MIXED,32,2669.4,94.7,35.3,28.33 +qwen2-1.5b,INT8-CW,1024,5455.8,1047.6,35.3,28.33 +minicpm-1b-sft,INT8-CW,31,3104.1,103.5,35.3,28.33 +phi-2,INT4-MIXED,1024,3039.8,1917.4,35.9,27.86 +stable-zephyr-3b-dpo,INT4-MIXED,946,3411.4,1695,37,27.03 +gemma-2b-it,INT4-MIXED,32,3991.7,116.1,37.9,26.39 +opt-2.7b,INT4-MIXED,937,3617.5,1764.9,38.2,26.18 +phi-3-mini-4k-instruct,INT4-MIXED,31,2935.3,111.6,38.2,26.18 +phi-3-mini-4k-instruct,INT4-MIXED,38,3102.4,134,38.4,26.04 +phi-3-mini-4k-instruct,INT4-MIXED,31,2986.1,114.1,38.9,25.71 +phi-3-mini-4k-instruct,INT4-MIXED,38,2977.4,131.1,39,25.64 +gemma-2b-it,INT4-MIXED,1024,4973.3,1249.2,39.7,25.19 +stablelm-3b-4e1t,INT4-MIXED,1024,3196.9,2045.4,39.9,25.06 +dolly-v2-3b,INT8-CW,32,3490.2,107.4,41.5,24.10 +red-pajama-incite-chat-3b-v1,INT8-CW,32,3457.9,105,42.5,23.53 +opt-2.7b,INT8-CW,31,3686.8,107.5,44.1,22.68 +phi-2,INT8-CW,32,3554.9,116.6,44.1,22.68 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3390.7,2277.1,44.2,22.62 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3643.6,2485,44.4,22.52 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3516.4,2280.9,44.5,22.47 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3537.2,2522.4,44.7,22.37 +orca-mini-3b,INT4-MIXED,1024,3557.3,1898.9,45,22.22 +minicpm-1b-sft,FP16,31,3814.4,97.9,45.4,22.03 +stablelm-3b-4e1t,INT8-CW,32,3486.9,100.5,46.1,21.69 +stable-zephyr-3b-dpo,INT8-CW,30,3516.7,101.9,46.1,21.69 +dolly-v2-3b,INT8-CW,1024,4265.9,2178.6,46.2,21.65 +red-pajama-incite-chat-3b-v1,INT8-CW,1020,3979.1,2219.7,47.2,21.19 +red-pajama-incite-chat-3b-v1,INT8-CW,1024,3975.5,2199.7,47.3,21.14 +opt-2.7b,INT8-CW,937,4358.6,1981.8,48.4,20.66 +phi-2,INT8-CW,1024,4058.1,2280.1,48.9,20.45 +gemma-2b-it,INT8-CW,32,4786.8,119.8,49.4,20.24 +chatglm3-6b,INT4-MIXED,32,4141.5,166.6,49.7,20.12 +stablelm-3b-4e1t,INT8-CW,1024,4054.8,2243.5,50.7,19.72 +stable-zephyr-3b-dpo,INT8-CW,946,4521.8,1816.4,51.3,19.49 +gemma-2b-it,INT8-CW,1024,5810.7,1580,51.3,19.49 +chatglm3-6b,INT4-MIXED,32,4651.4,164.7,51.6,19.38 +chatglm3-6b,INT4-MIXED,1024,4235.1,2818.7,52.3,19.12 +orca-mini-3b,INT8-CW,32,4162,109.2,53.3,18.76 +chatglm3-6b,INT4-MIXED,1024,4783.8,2869,54.4,18.38 +gpt-j-6b,INT4-MIXED,32,4667.3,176.7,56.3,17.76 +chatglm3-6b-gptq,INT4-MIXED,32,5369.4,173.9,58.9,16.98 +llama-2-7b-chat-hf,INT4-MIXED,32,4280,173.2,60.1,16.64 +phi-3-mini-4k-instruct,INT8-CW,31,4585.1,123,60.5,16.53 +phi-3-mini-4k-instruct,INT8-CW,38,4597,152,60.5,16.53 +chatglm2-6b,INT4-MIXED,32,4847.8,158.7,60.6,16.50 +vicuna-7b-v1.5,INT4-MIXED,32,4476.9,178.2,61.2,16.34 +chatglm3-6b-gptq,INT4-MIXED,1024,5217.6,2863.7,61.3,16.31 +mistral-7b-v0.1,INT4-MIXED,31,4413.6,194,61.7,16.21 +qwen2-7b,INT4-MIXED,32,7044.7,184.4,61.7,16.21 +mistral-7b-v0.1,INT4-MIXED,32,4427.6,193.3,61.8,16.18 +orca-mini-3b,INT8-CW,1024,4821.6,2239.1,62,16.13 +codegen25-7b,INT4-MIXED,32,4687.2,176.2,62.7,15.95 +chatglm2-6b,INT4-MIXED,1024,5165.9,3148,63,15.87 +llama-2-7b-gptq,INT4-MIXED,32,4632.8,175.2,63.4,15.77 +stablelm-7b,INT4-MIXED,32,5219.5,206.3,63.4,15.77 +qwen-7b-chat,INT4-MIXED,32,7805.6,193.8,63.6,15.72 +gpt-j-6b,INT4-MIXED,1024,5314.9,3111.8,63.6,15.72 +qwen2-7b,INT4-MIXED,1024,7716.2,3548.3,64.1,15.60 +llama-3-8b,INT4-MIXED,32,4910.9,204.8,64.7,15.46 +mistral-7b-v0.1,INT4-MIXED,1024,4720.8,3667.1,64.8,15.43 +mistral-7b-v0.1,INT4-MIXED,1007,4704.7,3685.4,64.9,15.41 +llama-3.1-8b,INT4-MIXED,31,4850.3,211.5,64.9,15.41 +phi-3-mini-4k-instruct,INT8-CW,1023,5128.6,2815.2,65.7,15.22 +phi-3-mini-4k-instruct,INT8-CW,1061,5155,3407.9,65.9,15.17 +mistral-7b-v0.1,INT4-MIXED,32,4939.3,192,66.5,15.04 +llama-3-8b,INT4-MIXED,33,4919.4,261.9,67.2,14.88 +llama-2-7b-chat-hf,INT4-MIXED,1024,4948.2,3811,67.3,14.86 +qwen1.5-7b-chat,INT4-MIXED,32,5943.1,180.5,67.7,14.77 +qwen-7b-chat-gptq,INT4-MIXED,32,8057,187,68.1,14.68 +llama-3-8b,INT4-MIXED,32,5503.5,198.4,68.1,14.68 +qwen-7b-chat,INT4-MIXED,32,8091.6,185.9,68.1,14.68 +llama-3-8b,INT4-MIXED,1024,5569.1,3920.5,68.2,14.66 +llama-3.1-8b,INT4-MIXED,31,5358.6,201,68.2,14.66 +stablelm-7b,INT4-MIXED,1020,5804.4,3726.6,68.8,14.53 +llama-3.1-8b,INT4-MIXED,31,5452.6,202.9,68.8,14.53 +llama-2-7b-chat-hf,INT4-MIXED,32,5023,165.7,69,14.49 +llama-3-8b,INT4-MIXED,32,5413.6,202,69.1,14.47 +llama-3-8b,INT4-MIXED,33,5440.4,262.1,69.2,14.45 +codegen25-7b,INT4-MIXED,1024,5434.6,3513.2,69.9,14.31 +mistral-7b-v0.1,INT4-MIXED,1024,5614.9,3819.1,70,14.29 +mistral-7b-v0.1,INT4-MIXED,31,4927.8,205,70.5,14.18 +llama-3-8b,INT4-MIXED,33,5498.9,270.7,70.6,14.16 +llama-3-8b,INT4-MIXED,1025,5577.4,4271.2,70.6,14.16 +llama-2-7b-gptq,INT4-MIXED,1024,5302.2,3529.4,70.7,14.14 +zephyr-7b-beta,INT4-MIXED,32,5212.4,190.6,71.2,14.04 +llama-3-8b,INT4-MIXED,1024,6161.1,3918,71.5,13.99 +llama-3-8b,INT4-MIXED,1025,6098,4441.8,72.3,13.83 +llama-3-8b,INT4-MIXED,1024,6071.7,3972.2,72.4,13.81 +mistral-7b-v0.1,INT4-MIXED,1007,5224.1,4153.4,73.8,13.55 +llama-3-8b,INT4-MIXED,1025,6156.9,4357,73.9,13.53 +zephyr-7b-beta,INT4-MIXED,1024,5511.6,3978,74.4,13.44 +opt-2.7b,FP16,31,9220.3,107.8,74.7,13.39 +dolly-v2-3b,FP16,32,6058.9,109.9,74.7,13.39 +qwen1.5-7b-chat,INT4-MIXED,1024,7063.2,3791.7,75,13.33 +qwen-7b-chat,INT4-MIXED,1024,8919.5,3763.9,75,13.33 +red-pajama-incite-chat-3b-v1,FP16,32,6036.5,107.5,75.9,13.18 +llama-2-7b-chat-hf,INT4-MIXED,1024,5716.8,4231.7,76.2,13.12 +phi-2,FP16,32,6090.1,115.2,77.1,12.97 +stable-zephyr-3b-dpo,FP16,30,6113.1,112.1,78.6,12.72 +qwen-7b-chat,INT4-MIXED,1024,9212.9,3857.4,78.6,12.72 +stablelm-3b-4e1t,FP16,32,6065.4,110.2,78.7,12.71 +opt-2.7b,FP16,937,9733.8,3750.8,78.8,12.69 +dolly-v2-3b,FP16,1024,6615.2,2230.9,79.1,12.64 +red-pajama-incite-chat-3b-v1,FP16,1020,6588.3,2259.4,80.2,12.47 +glm-4-9b,INT4-MIXED,33,6386.2,328,80.4,12.44 +red-pajama-incite-chat-3b-v1,FP16,1024,6570.3,2268.7,80.4,12.44 +baichuan2-7b-chat,INT4-MIXED,32,5977.9,201.7,81,12.35 +glm-4-9b,INT4-MIXED,32,6389.7,248.1,81,12.35 +phi-2,FP16,1024,6646.2,2406.7,81.4,12.29 +stable-zephyr-3b-dpo,FP16,946,6875.7,1868.2,82.9,12.06 +stablelm-3b-4e1t,FP16,1024,6636.1,2036.9,83,12.05 +chatglm2-6b,INT8-CW,32,6731.8,159.2,84.4,11.85 +glm-4-9b,INT4-MIXED,1025,7061.4,4939.2,85.2,11.74 +qwen-7b-chat-gptq,INT4-MIXED,1024,9175.3,3898,85.3,11.72 +gemma-7b-it,INT4-MIXED,32,7883.9,230.5,86,11.63 +gemma-7b-it,INT4-MIXED,32,8002.6,235,86.1,11.61 +glm-4-9b,INT4-MIXED,1024,7064.9,4411.2,86.2,11.60 +gpt-j-6b,INT8-CW,32,7009.2,176.8,86.4,11.57 +chatglm2-6b,INT8-CW,1024,7050.5,3871.6,86.8,11.52 +chatglm3-6b,INT8-CW,32,6755.9,159,86.8,11.52 +baichuan2-7b-chat,INT4-MIXED,1024,7033.3,4049,88.8,11.26 +chatglm3-6b,INT8-CW,1024,7076.5,3865.9,89.2,11.21 +qwen-7b-chat,INT4-MIXED,32,9245.7,176.3,90,11.11 +gemma-7b-it,INT4-MIXED,1024,9449.4,4305.8,93.2,10.73 +gpt-j-6b,INT8-CW,1024,7672.3,4181.1,93.5,10.70 +gemma-7b-it,INT4-MIXED,1024,9330.5,4222.5,93.7,10.67 +orca-mini-3b,FP16,32,7416.5,122.3,94.7,10.56 +codegen25-7b,INT8-CW,32,7557.6,170.7,98.4,10.16 +qwen-7b-chat,INT4-MIXED,1024,10371.1,4271.7,98.9,10.11 +llama-2-7b-chat-hf,INT8-CW,32,7390.6,171.6,99.9,10.01 diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv new file mode 100644 index 00000000000000..09799a2de31fe6 --- /dev/null +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv @@ -0,0 +1,182 @@ +Topology,Precision,Input Size,max rss memory,1st latency (ms),2nd latency (ms),2nd tok/sec +opt-125m-gptq,INT4-MIXED,1024,1513.6,81.9,7.8,128.21 +opt-125m-gptq,INT4-MIXED,32,979.9,50.4,7.9,126.58 +tiny-llama-1.1b-chat,INT4-MIXED,1024,1943.3,176.3,16.8,59.52 +tiny-llama-1.1b-chat,INT4-MIXED,32,1982.2,59.5,17.1,58.48 +qwen2-0.5b,INT4-MIXED,32,2678,117.3,18.7,53.48 +tiny-llama-1.1b-chat,INT8-CW,32,2080.9,59.4,19,52.63 +qwen2-0.5b,INT4-MIXED,1024,3036.1,165.5,19.2,52.08 +tiny-llama-1.1b-chat,INT8-CW,1024,2287,241.4,19.6,51.02 +qwen2-0.5b,INT8-CW,1024,3084.9,172.1,20,50.00 +qwen2-0.5b,INT8-CW,32,2518,105.5,21.4,46.73 +red-pajama-incite-chat-3b-v1,INT4-MIXED,32,2793.6,141.8,23.9,41.84 +qwen2-1.5b,INT4-MIXED,32,4515.4,118.7,24,41.67 +qwen2-1.5b,INT4-MIXED,1024,4930.1,229.6,24.3,41.15 +dolly-v2-3b,INT4-MIXED,32,2486.1,174,25.4,39.37 +phi-2,INT4-MIXED,32,2552.9,210.6,26.9,37.17 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1020,2934.1,464.5,27.5,36.36 +qwen2-1.5b,INT8-CW,32,4813.4,119.1,27.8,35.97 +opt-2.7b,INT4-MIXED,31,3172.5,131.9,28.5,35.09 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1024,3038.2,447.1,28.6,34.97 +dolly-v2-3b,INT4-MIXED,1024,2947.4,409,28.8,34.72 +qwen2-1.5b,INT8-CW,1024,5394.8,327.9,29.3,34.13 +stable-zephyr-3b-dpo,INT4-MIXED,30,2728.1,131.2,29.8,33.56 +phi-2,INT4-MIXED,32,2805.1,208.3,30.2,33.11 +minicpm-1b-sft,INT8-CW,31,3104.2,147.8,30.9,32.36 +phi-2,INT4-MIXED,1024,3058.9,602.9,31.1,32.15 +minicpm-1b-sft,INT4-MIXED,31,2970.1,183.7,31.1,32.15 +stablelm-3b-4e1t,INT4-MIXED,32,3077.1,183.2,31.6,31.65 +opt-2.7b,INT4-MIXED,937,3416.7,429.4,31.6,31.65 +stable-zephyr-3b-dpo,INT4-MIXED,946,3211.8,428.8,32.3,30.96 +phi-3-mini-4k-instruct,INT4-MIXED,31,3014.5,116,32.5,30.77 +phi-3-mini-4k-instruct,INT4-MIXED,38,2957.4,153.9,32.5,30.77 +phi-2,INT4-MIXED,1024,3278.9,613.3,33.4,29.94 +phi-3-mini-4k-instruct,INT4-MIXED,38,3288.5,152.9,33.4,29.94 +phi-3-mini-4k-instruct,INT4-MIXED,31,3265.1,123.6,34.1,29.33 +gemma-2b-it,INT4-MIXED,32,4162.1,208.8,34.2,29.24 +stablelm-3b-4e1t,INT4-MIXED,1024,3525.8,524.5,35,28.57 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3427.8,777.5,36.5,27.40 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3405.4,554.1,36.7,27.25 +gemma-2b-it,INT4-MIXED,1024,5053.1,354.8,36.9,27.10 +minicpm-1b-sft,FP16,31,3595.5,124.9,36.9,27.10 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3547.2,755.8,37.1,26.95 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3528.4,536.4,37.4,26.74 +red-pajama-incite-chat-3b-v1,INT8-CW,32,3747.7,189.9,38.1,26.25 +opt-2.7b,INT8-CW,31,3810.7,145.7,38.5,25.97 +chatglm3-6b,INT4-MIXED,32,4120.7,67.3,38.7,25.84 +dolly-v2-3b,INT8-CW,32,3747,188.4,39.2,25.51 +chatglm3-6b,INT4-MIXED,32,4482.9,69.9,40.7,24.57 +chatglm3-6b,INT4-MIXED,1024,4146,606.8,41,24.39 +opt-2.7b,INT8-CW,937,4458.9,587.8,41.8,23.92 +red-pajama-incite-chat-3b-v1,INT8-CW,1024,4088.4,634.1,41.9,23.87 +red-pajama-incite-chat-3b-v1,INT8-CW,1020,4086.8,653.4,42,23.81 +phi-2,INT8-CW,32,3794.6,202.7,42.1,23.75 +chatglm3-6b,INT4-MIXED,1024,4446.7,598.6,42.3,23.64 +stablelm-3b-4e1t,INT8-CW,32,3652.5,146,42.6,23.47 +stable-zephyr-3b-dpo,INT8-CW,30,3768.6,151.9,42.6,23.47 +dolly-v2-3b,INT8-CW,1024,4092,603.1,42.9,23.31 +stablelm-3b-4e1t,INT8-CW,1024,4143.2,671.7,45.2,22.12 +gemma-2b-it,INT8-CW,32,4878.4,221.6,45.6,21.93 +phi-2,INT8-CW,1024,4153.6,810.3,46,21.74 +llama-2-7b-chat-hf,INT4-MIXED,32,4394.6,109.7,46.2,21.65 +chatglm3-6b-gptq,INT4-MIXED,32,5218.9,79.7,46.7,21.41 +stable-zephyr-3b-dpo,INT8-CW,946,4360.1,627.8,46.8,21.37 +vicuna-7b-v1.5,INT4-MIXED,32,4482.3,101.2,47.2,21.19 +gemma-2b-it,INT8-CW,1024,5837.1,507.1,48,20.83 +llama-2-7b-gptq,INT4-MIXED,32,4734.3,102.8,48.1,20.79 +orca-mini-3b,INT4-MIXED,32,2720.1,132,48.1,20.79 +qwen-7b-chat,INT4-MIXED,32,7803.7,178.5,48.3,20.70 +mistral-7b-v0.1,INT4-MIXED,31,4537.5,99,48.5,20.62 +codegen25-7b,INT4-MIXED,32,4723.3,108.5,48.5,20.62 +chatglm3-6b-gptq,INT4-MIXED,1024,5150.8,614.2,48.8,20.49 +mistral-7b-v0.1,INT4-MIXED,32,4572,102.9,48.8,20.49 +llama-3-8b,INT4-MIXED,33,4991.2,252.2,50.9,19.65 +qwen-7b-chat-gptq,INT4-MIXED,32,8088.4,212.6,51,19.61 +chatglm2-6b,INT4-MIXED,32,4960.6,105.5,51.2,19.53 +gpt-j-6b,INT4-MIXED,32,4699.5,259.2,51.4,19.46 +llama-3.1-8b,INT4-MIXED,31,4897.8,106.9,51.5,19.42 +llama-3-8b,INT4-MIXED,32,4999.7,105.9,51.6,19.38 +qwen-7b-chat,INT4-MIXED,32,8085.9,193.5,51.7,19.34 +falcon-7b-instruct,INT4-MIXED,32,5416.2,175,52.5,19.05 +mistral-7b-v0.1,INT4-MIXED,1007,4772.6,803,52.6,19.01 +qwen1.5-7b-chat,INT4-MIXED,32,6027.3,174.9,53,18.87 +mistral-7b-v0.1,INT4-MIXED,1024,4775,717.6,53,18.87 +llama-2-7b-chat-hf,INT4-MIXED,1024,4976.5,992.1,53.1,18.83 +qwen2-7b,INT4-MIXED,32,7087.1,138.1,53.3,18.76 +llama-2-7b-gptq,INT4-MIXED,1024,5351.2,711.6,53.7,18.62 +llama-3-8b,INT4-MIXED,32,5472.8,109.4,53.7,18.62 +phi-3-mini-4k-instruct,INT8-CW,38,4575.3,115.9,53.7,18.62 +stablelm-7b,INT4-MIXED,32,5213.7,128.5,53.8,18.59 +phi-3-mini-4k-instruct,INT8-CW,31,4571.8,118.9,53.8,18.59 +llama-3-8b,INT4-MIXED,33,5480.4,246.8,53.9,18.55 +llama-3-8b,INT4-MIXED,32,5528.2,144.9,54.3,18.42 +llama-3.1-8b,INT4-MIXED,31,5377.3,112.8,54.3,18.42 +chatglm2-6b,INT4-MIXED,1024,5232.3,759.6,54.6,18.32 +llama-3.1-8b,INT4-MIXED,31,5440.4,126.4,54.8,18.25 +llama-3-8b,INT4-MIXED,33,5532.8,248.2,54.9,18.21 +codegen25-7b,INT4-MIXED,1024,5412.9,714.8,55,18.18 +mistral-7b-v0.1,INT4-MIXED,32,4998.5,117.3,55.2,18.12 +mistral-7b-v0.1,INT4-MIXED,31,5000.2,122.4,55.6,17.99 +llama-3-8b,INT4-MIXED,1024,5594,953.5,56.6,17.67 +gpt-j-6b,INT4-MIXED,1024,5323.8,1254,56.8,17.61 +llama-3-8b,INT4-MIXED,1025,5596.7,1192.3,56.8,17.61 +qwen2-7b,INT4-MIXED,1024,7722.1,714.2,57,17.54 +phi-3-mini-4k-instruct,INT8-CW,1023,5067.1,818.5,57.4,17.42 +phi-3-mini-4k-instruct,INT8-CW,1061,5086.1,975.1,57.4,17.42 +llama-2-7b-chat-hf,INT4-MIXED,32,5087.7,126.2,57.9,17.27 +stablelm-7b,INT4-MIXED,1020,5780.5,1248.4,59,16.95 +llama-3-8b,INT4-MIXED,1025,6088.9,1381.5,59,16.95 +llama-3-8b,INT4-MIXED,1024,6084.8,931.2,59.2,16.89 +llama-3-8b,INT4-MIXED,1025,6141.2,1494.3,59.4,16.84 +llama-3-8b,INT4-MIXED,1024,6133.8,1075.2,59.6,16.78 +mistral-7b-v0.1,INT4-MIXED,1024,5472.6,794.3,59.7,16.75 +zephyr-7b-beta,INT4-MIXED,32,5328.5,103.5,59.8,16.72 +falcon-7b-instruct,INT4-MIXED,1024,5677.5,686.2,59.8,16.72 +mistral-7b-v0.1,INT4-MIXED,1007,5243.5,1074,59.9,16.69 +qwen1.5-7b-chat,INT4-MIXED,1024,7096.7,1132.7,60,16.67 +qwen-7b-chat,INT4-MIXED,1024,8872.6,792.8,61,16.39 +qwen-7b-chat,INT4-MIXED,1024,9164.4,822.6,63.3,15.80 +orca-mini-3b,INT8-CW,32,4221.7,170.6,63.5,15.75 +llama-2-7b-chat-hf,INT4-MIXED,1024,5708.1,1397.9,63.6,15.72 +glm-4-9b,INT4-MIXED,33,6402.9,307.1,63.8,15.67 +zephyr-7b-beta,INT4-MIXED,1024,5572.4,1156.4,64.3,15.55 +glm-4-9b,INT4-MIXED,32,6383.1,256.2,64.5,15.50 +baichuan2-7b-chat,INT4-MIXED,32,5926.3,191.8,65.8,15.20 +opt-2.7b,FP16,31,5886,112.2,68,14.71 +dolly-v2-3b,FP16,32,6161.5,147.5,69.5,14.39 +red-pajama-incite-chat-3b-v1,FP16,32,6265.4,146.2,69.6,14.37 +glm-4-9b,INT4-MIXED,1024,6994.5,1013.7,69.8,14.33 +opt-2.7b,FP16,937,6345,379.5,71.6,13.97 +glm-4-9b,INT4-MIXED,1025,7014.9,1416.8,72.5,13.79 +phi-2,FP16,32,6204.7,189.2,72.9,13.72 +stable-zephyr-3b-dpo,FP16,30,6221.4,159.7,73,13.70 +dolly-v2-3b,FP16,1024,6669.9,424.3,73.3,13.64 +red-pajama-incite-chat-3b-v1,FP16,1020,6658.8,484.7,73.4,13.62 +stablelm-3b-4e1t,FP16,32,6216.3,145.4,73.5,13.61 +qwen-7b-chat,INT4-MIXED,32,9294.9,144.4,73.8,13.55 +red-pajama-incite-chat-3b-v1,FP16,1024,6755.1,469.1,73.9,13.53 +qwen-7b-chat-gptq,INT4-MIXED,1024,9152.1,827.2,75.1,13.32 +gemma-7b-it,INT4-MIXED,32,7991.4,128.6,75.8,13.19 +chatglm2-6b,INT8-CW,32,6854.4,110.2,76.3,13.11 +chatglm3-6b,INT8-CW,32,6754.8,112.3,76.4,13.09 +stable-zephyr-3b-dpo,FP16,946,6940,428.6,76.7,13.04 +baichuan2-7b-chat,INT4-MIXED,1024,6930.2,1229.5,76.7,13.04 +gemma-7b-it,INT4-MIXED,32,8061.5,125.6,76.7,13.04 +stablelm-3b-4e1t,FP16,1024,6722.9,480.8,77,12.99 +phi-2,FP16,1024,6709.4,624.1,77.2,12.95 +chatglm2-6b,INT8-CW,1024,7132.9,1361.9,78.7,12.71 +chatglm3-6b,INT8-CW,1024,7037.5,1389.2,78.7,12.71 +qwen-7b-chat,INT4-MIXED,1024,10374.1,1357.5,81.1,12.33 +gemma-7b-it,INT4-MIXED,1024,9398,1268.5,82.7,12.09 +gemma-7b-it,INT4-MIXED,1024,9469.5,1268,83.2,12.02 +gpt-j-6b,INT8-CW,32,7126.5,255.2,87.2,11.47 +falcon-7b-instruct,INT8-CW,32,8287.6,131.1,88.4,11.31 +llama-2-7b-chat-hf,INT8-CW,32,7474.9,139.5,89.7,11.15 +codegen25-7b,INT8-CW,32,7559.4,138,90.8,11.01 +vicuna-7b-v1.5,INT8-CW,32,7390.8,136.6,90.8,11.01 +falcon-7b-instruct,INT8-CW,1024,8546.8,1205.9,92.2,10.85 +stablelm-7b,INT8-CW,32,8356.4,143,92.4,10.82 +qwen2-7b,INT8-CW,32,9940.7,132,92.5,10.81 +baichuan2-13b-chat,INT4-MIXED,32,9879.2,184.9,93.3,10.72 +phi-3-mini-4k-instruct,FP16,38,8290,125.2,93.4,10.71 +phi-3-mini-4k-instruct,FP16,31,8290.5,109.5,93.5,10.70 +gpt-j-6b,INT8-CW,1024,7759,1996.8,93.9,10.65 +llama-2-7b-chat-hf,INT8-CW,1024,8097.8,1701.6,94.7,10.56 +phi-3-medium-4k-instruct,INT4-MIXED,38,8210.4,527,95.1,10.52 +mistral-7b-v0.1,INT8-CW,31,7882.4,128.6,95.1,10.52 +vicuna-7b-v1.5,INT8-CW,1024,8013.2,1558.1,95.1,10.52 +mistral-7b-v0.1,INT8-CW,32,7886.9,140.6,95.2,10.50 +qwen2-7b,INT8-CW,1024,10573.1,1564.5,95.3,10.49 +codegen25-7b,INT8-CW,1024,8253.1,1526.3,95.7,10.45 +zephyr-7b-beta,INT8-CW,32,7785.3,144.4,95.8,10.44 +stablelm-7b,INT8-CW,1020,8921.9,1845,96.9,10.32 +mistral-7b-v0.1,INT8-CW,1007,8127.4,1648.4,97.4,10.27 +qwen-7b-chat,INT8-CW,32,11083.2,140.6,97.7,10.24 +qwen1.5-7b-chat,INT8-CW,32,8870,156.4,98.1,10.19 +llama-3.1-8b,INT8-CW,31,8600.3,189.2,98.4,10.16 +mistral-7b-v0.1,INT8-CW,1024,8134.7,1554.1,98.4,10.16 +qwen-14b-chat,INT4-MIXED,32,9876.2,192.3,98.6,10.14 +zephyr-7b-beta,INT8-CW,1024,8035.2,1580.4,98.8,10.12 +llama-3-8b,INT8-CW,32,8694.2,150.7,99.5,10.05 +llama-3-8b,INT8-CW,33,8700.4,175.4,99.8,10.02 +phi-3-mini-4k-instruct,FP16,1023,8795.2,601.3,99.9,10.01 diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_9-288V.csv similarity index 100% rename from docs/sphinx_setup/_static/benchmarks_files/llm_models.csv rename to docs/sphinx_setup/_static/benchmarks_files/llm_models_9-288V.csv diff --git a/docs/sphinx_setup/_static/selector-tool/assets/index-f34d1fad.js b/docs/sphinx_setup/_static/selector-tool/assets/index-Codcw3jz.js similarity index 100% rename from docs/sphinx_setup/_static/selector-tool/assets/index-f34d1fad.js rename to docs/sphinx_setup/_static/selector-tool/assets/index-Codcw3jz.js diff --git a/docs/sphinx_setup/_static/selector-tool/assets/selector-4f9c3c09.js b/docs/sphinx_setup/_static/selector-tool/assets/selector-4f9c3c09.js deleted file mode 100644 index 0836ad1aed41f3..00000000000000 --- a/docs/sphinx_setup/_static/selector-tool/assets/selector-4f9c3c09.js +++ /dev/null @@ -1,61 +0,0 @@ -var Xf=Object.defineProperty;var Jf=(e,t,n)=>t in e?Xf(e,t,{enumerable:!0,configurable:!0,writable:!0,value:n}):e[t]=n;var Ve=(e,t,n)=>(Jf(e,typeof t!="symbol"?t+"":t,n),n);function uc(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var cc={exports:{}},po={},dc={exports:{}},D={};/** - * @license React - * react.production.min.js - * - * Copyright (c) Facebook, Inc. and its affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */var qr=Symbol.for("react.element"),Zf=Symbol.for("react.portal"),qf=Symbol.for("react.fragment"),ep=Symbol.for("react.strict_mode"),tp=Symbol.for("react.profiler"),np=Symbol.for("react.provider"),rp=Symbol.for("react.context"),ip=Symbol.for("react.forward_ref"),op=Symbol.for("react.suspense"),lp=Symbol.for("react.memo"),sp=Symbol.for("react.lazy"),Oa=Symbol.iterator;function ap(e){return e===null||typeof e!="object"?null:(e=Oa&&e[Oa]||e["@@iterator"],typeof e=="function"?e:null)}var fc={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},pc=Object.assign,hc={};function sr(e,t,n){this.props=e,this.context=t,this.refs=hc,this.updater=n||fc}sr.prototype.isReactComponent={};sr.prototype.setState=function(e,t){if(typeof e!="object"&&typeof e!="function"&&e!=null)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,e,t,"setState")};sr.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")};function mc(){}mc.prototype=sr.prototype;function hs(e,t,n){this.props=e,this.context=t,this.refs=hc,this.updater=n||fc}var ms=hs.prototype=new mc;ms.constructor=hs;pc(ms,sr.prototype);ms.isPureReactComponent=!0;var _a=Array.isArray,gc=Object.prototype.hasOwnProperty,gs={current:null},vc={key:!0,ref:!0,__self:!0,__source:!0};function yc(e,t,n){var r,i={},o=null,l=null;if(t!=null)for(r in t.ref!==void 0&&(l=t.ref),t.key!==void 0&&(o=""+t.key),t)gc.call(t,r)&&!vc.hasOwnProperty(r)&&(i[r]=t[r]);var s=arguments.length-2;if(s===1)i.children=n;else if(1{const e={type:"size",height:document.body.offsetHeight};window.parent.postMessage(e)};new ResizeObserver(kp).observe(document.body);function me(e){"@babel/helpers - typeof";return me=typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?function(t){return typeof t}:function(t){return t&&typeof Symbol=="function"&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},me(e)}function pt(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function Sp(e,t){if(me(e)!=="object"||e===null)return e;var n=e[Symbol.toPrimitive];if(n!==void 0){var r=n.call(e,t||"default");if(me(r)!=="object")return r;throw new TypeError("@@toPrimitive must return a primitive value.")}return(t==="string"?String:Number)(e)}function kc(e){var t=Sp(e,"string");return me(t)==="symbol"?t:String(t)}function Pa(e,t){for(var n=0;ne.length)&&(t=e.length);for(var n=0,r=new Array(t);n1&&arguments[1]!==void 0?arguments[1]:{};pt(this,e),this.init(t,n)}return ht(e,[{key:"init",value:function(n){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{};this.prefix=r.prefix||"i18next:",this.logger=n||Np,this.options=r,this.debug=r.debug}},{key:"setDebug",value:function(n){this.debug=n}},{key:"log",value:function(){for(var n=arguments.length,r=new Array(n),i=0;i1?r-1:0),o=1;o-1?s.replace(/###/g,"."):s}function i(){return!e||typeof e=="string"}for(var o=typeof t!="string"?[].concat(t):t.split(".");o.length>1;){if(i())return{};var l=r(o.shift());!e[l]&&n&&(e[l]=new n),Object.prototype.hasOwnProperty.call(e,l)?e=e[l]:e={}}return i()?{}:{obj:e,k:r(o.shift())}}function Ta(e,t,n){var r=ws(e,t,Object),i=r.obj,o=r.k;i[o]=n}function xp(e,t,n,r){var i=ws(e,t,Object),o=i.obj,l=i.k;o[l]=o[l]||[],r&&(o[l]=o[l].concat(n)),r||o[l].push(n)}function bi(e,t){var n=ws(e,t),r=n.obj,i=n.k;if(r)return r[i]}function Ra(e,t,n){var r=bi(e,n);return r!==void 0?r:bi(t,n)}function Nc(e,t,n){for(var r in t)r!=="__proto__"&&r!=="constructor"&&(r in e?typeof e[r]=="string"||e[r]instanceof String||typeof t[r]=="string"||t[r]instanceof String?n&&(e[r]=t[r]):Nc(e[r],t[r],n):e[r]=t[r]);return e}function Tn(e){return e.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g,"\\$&")}var Cp={"&":"&","<":"<",">":">",'"':""","'":"'","/":"/"};function Ip(e){return typeof e=="string"?e.replace(/[&<>"'\/]/g,function(t){return Cp[t]}):e}var mo=typeof window<"u"&&window.navigator&&typeof window.navigator.userAgentData>"u"&&window.navigator.userAgent&&window.navigator.userAgent.indexOf("MSIE")>-1,Tp=[" ",",","?","!",";"];function Rp(e,t,n){t=t||"",n=n||"";var r=Tp.filter(function(s){return t.indexOf(s)<0&&n.indexOf(s)<0});if(r.length===0)return!0;var i=new RegExp("(".concat(r.map(function(s){return s==="?"?"\\?":s}).join("|"),")")),o=!i.test(e);if(!o){var l=e.indexOf(n);l>0&&!i.test(e.substring(0,l))&&(o=!0)}return o}function La(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter(function(i){return Object.getOwnPropertyDescriptor(e,i).enumerable})),n.push.apply(n,r)}return n}function si(e){for(var t=1;t"u"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch{return!1}}function Pc(e,t){var n=arguments.length>2&&arguments[2]!==void 0?arguments[2]:".";if(e){if(e[t])return e[t];for(var r=t.split(n),i=e,o=0;oo+l;)l++,s=r.slice(o,o+l).join(n),a=i[s];if(a===void 0)return;if(a===null)return null;if(t.endsWith(s)){if(typeof a=="string")return a;if(s&&typeof a[s]=="string")return a[s]}var u=r.slice(o+l).join(n);return u?Pc(a,u,n):void 0}i=i[r[o]]}return i}}var jp=function(e){ho(n,e);var t=Lp(n);function n(r){var i,o=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{ns:["translation"],defaultNS:"translation"};return pt(this,n),i=t.call(this),mo&&nn.call(Gt(i)),i.data=r||{},i.options=o,i.options.keySeparator===void 0&&(i.options.keySeparator="."),i.options.ignoreJSONStructure===void 0&&(i.options.ignoreJSONStructure=!0),i}return ht(n,[{key:"addNamespaces",value:function(i){this.options.ns.indexOf(i)<0&&this.options.ns.push(i)}},{key:"removeNamespaces",value:function(i){var o=this.options.ns.indexOf(i);o>-1&&this.options.ns.splice(o,1)}},{key:"getResource",value:function(i,o,l){var s=arguments.length>3&&arguments[3]!==void 0?arguments[3]:{},a=s.keySeparator!==void 0?s.keySeparator:this.options.keySeparator,u=s.ignoreJSONStructure!==void 0?s.ignoreJSONStructure:this.options.ignoreJSONStructure,p=[i,o];l&&typeof l!="string"&&(p=p.concat(l)),l&&typeof l=="string"&&(p=p.concat(a?l.split(a):l)),i.indexOf(".")>-1&&(p=i.split("."));var d=bi(this.data,p);return d||!u||typeof l!="string"?d:Pc(this.data&&this.data[i]&&this.data[i][o],l,a)}},{key:"addResource",value:function(i,o,l,s){var a=arguments.length>4&&arguments[4]!==void 0?arguments[4]:{silent:!1},u=this.options.keySeparator;u===void 0&&(u=".");var p=[i,o];l&&(p=p.concat(u?l.split(u):l)),i.indexOf(".")>-1&&(p=i.split("."),s=o,o=p[1]),this.addNamespaces(o),Ta(this.data,p,s),a.silent||this.emit("added",i,o,l,s)}},{key:"addResources",value:function(i,o,l){var s=arguments.length>3&&arguments[3]!==void 0?arguments[3]:{silent:!1};for(var a in l)(typeof l[a]=="string"||Object.prototype.toString.apply(l[a])==="[object Array]")&&this.addResource(i,o,a,l[a],{silent:!0});s.silent||this.emit("added",i,o,l)}},{key:"addResourceBundle",value:function(i,o,l,s,a){var u=arguments.length>5&&arguments[5]!==void 0?arguments[5]:{silent:!1},p=[i,o];i.indexOf(".")>-1&&(p=i.split("."),s=l,l=o,o=p[1]),this.addNamespaces(o);var d=bi(this.data,p)||{};s?Nc(d,l,a):d=si(si({},d),l),Ta(this.data,p,d),u.silent||this.emit("added",i,o,l)}},{key:"removeResourceBundle",value:function(i,o){this.hasResourceBundle(i,o)&&delete this.data[i][o],this.removeNamespaces(o),this.emit("removed",i,o)}},{key:"hasResourceBundle",value:function(i,o){return this.getResource(i,o)!==void 0}},{key:"getResourceBundle",value:function(i,o){return o||(o=this.options.defaultNS),this.options.compatibilityAPI==="v1"?si(si({},{}),this.getResource(i,o)):this.getResource(i,o)}},{key:"getDataByLanguage",value:function(i){return this.data[i]}},{key:"hasLanguageSomeTranslations",value:function(i){var o=this.getDataByLanguage(i),l=o&&Object.keys(o)||[];return!!l.find(function(s){return o[s]&&Object.keys(o[s]).length>0})}},{key:"toJSON",value:function(){return this.data}}]),n}(nn),Ec={processors:{},addPostProcessor:function(t){this.processors[t.name]=t},handle:function(t,n,r,i,o){var l=this;return t.forEach(function(s){l.processors[s]&&(n=l.processors[s].process(n,r,i,o))}),n}};function Aa(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter(function(i){return Object.getOwnPropertyDescriptor(e,i).enumerable})),n.push.apply(n,r)}return n}function Oe(e){for(var t=1;t"u"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch{return!1}}var ja={},Fa=function(e){ho(n,e);var t=Fp(n);function n(r){var i,o=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{};return pt(this,n),i=t.call(this),mo&&nn.call(Gt(i)),Ep(["resourceStore","languageUtils","pluralResolver","interpolator","backendConnector","i18nFormat","utils"],r,Gt(i)),i.options=o,i.options.keySeparator===void 0&&(i.options.keySeparator="."),i.logger=kt.create("translator"),i}return ht(n,[{key:"changeLanguage",value:function(i){i&&(this.language=i)}},{key:"exists",value:function(i){var o=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{interpolation:{}};if(i==null)return!1;var l=this.resolve(i,o);return l&&l.res!==void 0}},{key:"extractFromKey",value:function(i,o){var l=o.nsSeparator!==void 0?o.nsSeparator:this.options.nsSeparator;l===void 0&&(l=":");var s=o.keySeparator!==void 0?o.keySeparator:this.options.keySeparator,a=o.ns||this.options.defaultNS||[],u=l&&i.indexOf(l)>-1,p=!this.options.userDefinedKeySeparator&&!o.keySeparator&&!this.options.userDefinedNsSeparator&&!o.nsSeparator&&!Rp(i,l,s);if(u&&!p){var d=i.match(this.interpolator.nestingRegexp);if(d&&d.length>0)return{key:i,namespaces:a};var h=i.split(l);(l!==s||l===s&&this.options.ns.indexOf(h[0])>-1)&&(a=h.shift()),i=h.join(s)}return typeof a=="string"&&(a=[a]),{key:i,namespaces:a}}},{key:"translate",value:function(i,o,l){var s=this;if(me(o)!=="object"&&this.options.overloadTranslationOptionHandler&&(o=this.options.overloadTranslationOptionHandler(arguments)),o||(o={}),i==null)return"";Array.isArray(i)||(i=[String(i)]);var a=o.returnDetails!==void 0?o.returnDetails:this.options.returnDetails,u=o.keySeparator!==void 0?o.keySeparator:this.options.keySeparator,p=this.extractFromKey(i[i.length-1],o),d=p.key,h=p.namespaces,g=h[h.length-1],y=o.lng||this.language,k=o.appendNamespaceToCIMode||this.options.appendNamespaceToCIMode;if(y&&y.toLowerCase()==="cimode"){if(k){var _=o.nsSeparator||this.options.nsSeparator;return a?{res:"".concat(g).concat(_).concat(d),usedKey:d,exactUsedKey:d,usedLng:y,usedNS:g}:"".concat(g).concat(_).concat(d)}return a?{res:d,usedKey:d,exactUsedKey:d,usedLng:y,usedNS:g}:d}var f=this.resolve(i,o),c=f&&f.res,m=f&&f.usedKey||d,w=f&&f.exactUsedKey||d,O=Object.prototype.toString.apply(c),S=["[object Number]","[object Function]","[object RegExp]"],P=o.joinArrays!==void 0?o.joinArrays:this.options.joinArrays,E=!this.i18nFormat||this.i18nFormat.handleAsObject,A=typeof c!="string"&&typeof c!="boolean"&&typeof c!="number";if(E&&c&&A&&S.indexOf(O)<0&&!(typeof P=="string"&&O==="[object Array]")){if(!o.returnObjects&&!this.options.returnObjects){this.options.returnedObjectHandler||this.logger.warn("accessing an object - but returnObjects options is not enabled!");var x=this.options.returnedObjectHandler?this.options.returnedObjectHandler(m,c,Oe(Oe({},o),{},{ns:h})):"key '".concat(d," (").concat(this.language,")' returned an object instead of string.");return a?(f.res=x,f):x}if(u){var H=O==="[object Array]",Te=H?[]:{},Nt=H?w:m;for(var rt in c)if(Object.prototype.hasOwnProperty.call(c,rt)){var xn="".concat(Nt).concat(u).concat(rt);Te[rt]=this.translate(xn,Oe(Oe({},o),{joinArrays:!1,ns:h})),Te[rt]===xn&&(Te[rt]=c[rt])}c=Te}}else if(E&&typeof P=="string"&&O==="[object Array]")c=c.join(P),c&&(c=this.extendTranslation(c,i,o,l));else{var mt=!1,it=!1,C=o.count!==void 0&&typeof o.count!="string",L=n.hasDefaultValue(o),j=C?this.pluralResolver.getSuffix(y,o.count,o):"",V=o["defaultValue".concat(j)]||o.defaultValue;!this.isValidLookup(c)&&L&&(mt=!0,c=V),this.isValidLookup(c)||(it=!0,c=d);var Q=o.missingKeyNoValueFallbackToKey||this.options.missingKeyNoValueFallbackToKey,Pt=Q&&it?void 0:c,ze=L&&V!==c&&this.options.updateMissing;if(it||mt||ze){if(this.logger.log(ze?"updateKey":"missingKey",y,g,d,ze?V:c),u){var Cn=this.resolve(d,Oe(Oe({},o),{},{keySeparator:!1}));Cn&&Cn.res&&this.logger.warn("Seems the loaded translations were in flat JSON format instead of nested. Either set keySeparator: false on init or make sure your translations are published in nested format.")}var be=[],Et=this.languageUtils.getFallbackCodes(this.options.fallbackLng,o.lng||this.language);if(this.options.saveMissingTo==="fallback"&&Et&&Et[0])for(var Lo=0;Lo1&&arguments[1]!==void 0?arguments[1]:{},s,a,u,p,d;return typeof i=="string"&&(i=[i]),i.forEach(function(h){if(!o.isValidLookup(s)){var g=o.extractFromKey(h,l),y=g.key;a=y;var k=g.namespaces;o.options.fallbackNS&&(k=k.concat(o.options.fallbackNS));var _=l.count!==void 0&&typeof l.count!="string",f=_&&!l.ordinal&&l.count===0&&o.pluralResolver.shouldUseIntlApi(),c=l.context!==void 0&&(typeof l.context=="string"||typeof l.context=="number")&&l.context!=="",m=l.lngs?l.lngs:o.languageUtils.toResolveHierarchy(l.lng||o.language,l.fallbackLng);k.forEach(function(w){o.isValidLookup(s)||(d=w,!ja["".concat(m[0],"-").concat(w)]&&o.utils&&o.utils.hasLoadedNamespace&&!o.utils.hasLoadedNamespace(d)&&(ja["".concat(m[0],"-").concat(w)]=!0,o.logger.warn('key "'.concat(a,'" for languages "').concat(m.join(", "),`" won't get resolved as namespace "`).concat(d,'" was not yet loaded'),"This means something IS WRONG in your setup. You access the t function before i18next.init / i18next.loadNamespace / i18next.changeLanguage was done. Wait for the callback or Promise to resolve before accessing it!!!")),m.forEach(function(O){if(!o.isValidLookup(s)){p=O;var S=[y];if(o.i18nFormat&&o.i18nFormat.addLookupKeys)o.i18nFormat.addLookupKeys(S,y,O,w,l);else{var P;_&&(P=o.pluralResolver.getSuffix(O,l.count,l));var E="".concat(o.options.pluralSeparator,"zero");if(_&&(S.push(y+P),f&&S.push(y+E)),c){var A="".concat(y).concat(o.options.contextSeparator).concat(l.context);S.push(A),_&&(S.push(A+P),f&&S.push(A+E))}}for(var x;x=S.pop();)o.isValidLookup(s)||(u=x,s=o.getResource(O,w,x,l))}}))})}}),{res:s,usedKey:a,exactUsedKey:u,usedLng:p,usedNS:d}}},{key:"isValidLookup",value:function(i){return i!==void 0&&!(!this.options.returnNull&&i===null)&&!(!this.options.returnEmptyString&&i==="")}},{key:"getResource",value:function(i,o,l){var s=arguments.length>3&&arguments[3]!==void 0?arguments[3]:{};return this.i18nFormat&&this.i18nFormat.getResource?this.i18nFormat.getResource(i,o,l,s):this.resourceStore.getResource(i,o,l,s)}}],[{key:"hasDefaultValue",value:function(i){var o="defaultValue";for(var l in i)if(Object.prototype.hasOwnProperty.call(i,l)&&o===l.substring(0,o.length)&&i[l]!==void 0)return!0;return!1}}]),n}(nn);function Do(e){return e.charAt(0).toUpperCase()+e.slice(1)}var Da=function(){function e(t){pt(this,e),this.options=t,this.supportedLngs=this.options.supportedLngs||!1,this.logger=kt.create("languageUtils")}return ht(e,[{key:"getScriptPartFromCode",value:function(n){if(!n||n.indexOf("-")<0)return null;var r=n.split("-");return r.length===2||(r.pop(),r[r.length-1].toLowerCase()==="x")?null:this.formatLanguageCode(r.join("-"))}},{key:"getLanguagePartFromCode",value:function(n){if(!n||n.indexOf("-")<0)return n;var r=n.split("-");return this.formatLanguageCode(r[0])}},{key:"formatLanguageCode",value:function(n){if(typeof n=="string"&&n.indexOf("-")>-1){var r=["hans","hant","latn","cyrl","cans","mong","arab"],i=n.split("-");return this.options.lowerCaseLng?i=i.map(function(o){return o.toLowerCase()}):i.length===2?(i[0]=i[0].toLowerCase(),i[1]=i[1].toUpperCase(),r.indexOf(i[1].toLowerCase())>-1&&(i[1]=Do(i[1].toLowerCase()))):i.length===3&&(i[0]=i[0].toLowerCase(),i[1].length===2&&(i[1]=i[1].toUpperCase()),i[0]!=="sgn"&&i[2].length===2&&(i[2]=i[2].toUpperCase()),r.indexOf(i[1].toLowerCase())>-1&&(i[1]=Do(i[1].toLowerCase())),r.indexOf(i[2].toLowerCase())>-1&&(i[2]=Do(i[2].toLowerCase()))),i.join("-")}return this.options.cleanCode||this.options.lowerCaseLng?n.toLowerCase():n}},{key:"isSupportedCode",value:function(n){return(this.options.load==="languageOnly"||this.options.nonExplicitSupportedLngs)&&(n=this.getLanguagePartFromCode(n)),!this.supportedLngs||!this.supportedLngs.length||this.supportedLngs.indexOf(n)>-1}},{key:"getBestMatchFromCodes",value:function(n){var r=this;if(!n)return null;var i;return n.forEach(function(o){if(!i){var l=r.formatLanguageCode(o);(!r.options.supportedLngs||r.isSupportedCode(l))&&(i=l)}}),!i&&this.options.supportedLngs&&n.forEach(function(o){if(!i){var l=r.getLanguagePartFromCode(o);if(r.isSupportedCode(l))return i=l;i=r.options.supportedLngs.find(function(s){if(s.indexOf(l)===0)return s})}}),i||(i=this.getFallbackCodes(this.options.fallbackLng)[0]),i}},{key:"getFallbackCodes",value:function(n,r){if(!n)return[];if(typeof n=="function"&&(n=n(r)),typeof n=="string"&&(n=[n]),Object.prototype.toString.apply(n)==="[object Array]")return n;if(!r)return n.default||[];var i=n[r];return i||(i=n[this.getScriptPartFromCode(r)]),i||(i=n[this.formatLanguageCode(r)]),i||(i=n[this.getLanguagePartFromCode(r)]),i||(i=n.default),i||[]}},{key:"toResolveHierarchy",value:function(n,r){var i=this,o=this.getFallbackCodes(r||this.options.fallbackLng||[],n),l=[],s=function(u){u&&(i.isSupportedCode(u)?l.push(u):i.logger.warn("rejecting language code not found in supportedLngs: ".concat(u)))};return typeof n=="string"&&n.indexOf("-")>-1?(this.options.load!=="languageOnly"&&s(this.formatLanguageCode(n)),this.options.load!=="languageOnly"&&this.options.load!=="currentOnly"&&s(this.getScriptPartFromCode(n)),this.options.load!=="currentOnly"&&s(this.getLanguagePartFromCode(n))):typeof n=="string"&&s(this.formatLanguageCode(n)),o.forEach(function(a){l.indexOf(a)<0&&s(i.formatLanguageCode(a))}),l}}]),e}(),Up=[{lngs:["ach","ak","am","arn","br","fil","gun","ln","mfe","mg","mi","oc","pt","pt-BR","tg","tl","ti","tr","uz","wa"],nr:[1,2],fc:1},{lngs:["af","an","ast","az","bg","bn","ca","da","de","dev","el","en","eo","es","et","eu","fi","fo","fur","fy","gl","gu","ha","hi","hu","hy","ia","it","kk","kn","ku","lb","mai","ml","mn","mr","nah","nap","nb","ne","nl","nn","no","nso","pa","pap","pms","ps","pt-PT","rm","sco","se","si","so","son","sq","sv","sw","ta","te","tk","ur","yo"],nr:[1,2],fc:2},{lngs:["ay","bo","cgg","fa","ht","id","ja","jbo","ka","km","ko","ky","lo","ms","sah","su","th","tt","ug","vi","wo","zh"],nr:[1],fc:3},{lngs:["be","bs","cnr","dz","hr","ru","sr","uk"],nr:[1,2,5],fc:4},{lngs:["ar"],nr:[0,1,2,3,11,100],fc:5},{lngs:["cs","sk"],nr:[1,2,5],fc:6},{lngs:["csb","pl"],nr:[1,2,5],fc:7},{lngs:["cy"],nr:[1,2,3,8],fc:8},{lngs:["fr"],nr:[1,2],fc:9},{lngs:["ga"],nr:[1,2,3,7,11],fc:10},{lngs:["gd"],nr:[1,2,3,20],fc:11},{lngs:["is"],nr:[1,2],fc:12},{lngs:["jv"],nr:[0,1],fc:13},{lngs:["kw"],nr:[1,2,3,4],fc:14},{lngs:["lt"],nr:[1,2,10],fc:15},{lngs:["lv"],nr:[1,2,0],fc:16},{lngs:["mk"],nr:[1,2],fc:17},{lngs:["mnk"],nr:[0,1,2],fc:18},{lngs:["mt"],nr:[1,2,11,20],fc:19},{lngs:["or"],nr:[2,1],fc:2},{lngs:["ro"],nr:[1,2,20],fc:20},{lngs:["sl"],nr:[5,1,2,3],fc:21},{lngs:["he","iw"],nr:[1,2,20,21],fc:22}],zp={1:function(t){return+(t>1)},2:function(t){return+(t!=1)},3:function(t){return 0},4:function(t){return t%10==1&&t%100!=11?0:t%10>=2&&t%10<=4&&(t%100<10||t%100>=20)?1:2},5:function(t){return t==0?0:t==1?1:t==2?2:t%100>=3&&t%100<=10?3:t%100>=11?4:5},6:function(t){return t==1?0:t>=2&&t<=4?1:2},7:function(t){return t==1?0:t%10>=2&&t%10<=4&&(t%100<10||t%100>=20)?1:2},8:function(t){return t==1?0:t==2?1:t!=8&&t!=11?2:3},9:function(t){return+(t>=2)},10:function(t){return t==1?0:t==2?1:t<7?2:t<11?3:4},11:function(t){return t==1||t==11?0:t==2||t==12?1:t>2&&t<20?2:3},12:function(t){return+(t%10!=1||t%100==11)},13:function(t){return+(t!==0)},14:function(t){return t==1?0:t==2?1:t==3?2:3},15:function(t){return t%10==1&&t%100!=11?0:t%10>=2&&(t%100<10||t%100>=20)?1:2},16:function(t){return t%10==1&&t%100!=11?0:t!==0?1:2},17:function(t){return t==1||t%10==1&&t%100!=11?0:1},18:function(t){return t==0?0:t==1?1:2},19:function(t){return t==1?0:t==0||t%100>1&&t%100<11?1:t%100>10&&t%100<20?2:3},20:function(t){return t==1?0:t==0||t%100>0&&t%100<20?1:2},21:function(t){return t%100==1?1:t%100==2?2:t%100==3||t%100==4?3:0},22:function(t){return t==1?0:t==2?1:(t<0||t>10)&&t%10==0?2:3}},bp=["v1","v2","v3"],Ua={zero:0,one:1,two:2,few:3,many:4,other:5};function Vp(){var e={};return Up.forEach(function(t){t.lngs.forEach(function(n){e[n]={numbers:t.nr,plurals:zp[t.fc]}})}),e}var $p=function(){function e(t){var n=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{};pt(this,e),this.languageUtils=t,this.options=n,this.logger=kt.create("pluralResolver"),(!this.options.compatibilityJSON||this.options.compatibilityJSON==="v4")&&(typeof Intl>"u"||!Intl.PluralRules)&&(this.options.compatibilityJSON="v3",this.logger.error("Your environment seems not to be Intl API compatible, use an Intl.PluralRules polyfill. Will fallback to the compatibilityJSON v3 format handling.")),this.rules=Vp()}return ht(e,[{key:"addRule",value:function(n,r){this.rules[n]=r}},{key:"getRule",value:function(n){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{};if(this.shouldUseIntlApi())try{return new Intl.PluralRules(n,{type:r.ordinal?"ordinal":"cardinal"})}catch{return}return this.rules[n]||this.rules[this.languageUtils.getLanguagePartFromCode(n)]}},{key:"needsPlural",value:function(n){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{},i=this.getRule(n,r);return this.shouldUseIntlApi()?i&&i.resolvedOptions().pluralCategories.length>1:i&&i.numbers.length>1}},{key:"getPluralFormsOfKey",value:function(n,r){var i=arguments.length>2&&arguments[2]!==void 0?arguments[2]:{};return this.getSuffixes(n,i).map(function(o){return"".concat(r).concat(o)})}},{key:"getSuffixes",value:function(n){var r=this,i=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{},o=this.getRule(n,i);return o?this.shouldUseIntlApi()?o.resolvedOptions().pluralCategories.sort(function(l,s){return Ua[l]-Ua[s]}).map(function(l){return"".concat(r.options.prepend).concat(l)}):o.numbers.map(function(l){return r.getSuffix(n,l,i)}):[]}},{key:"getSuffix",value:function(n,r){var i=arguments.length>2&&arguments[2]!==void 0?arguments[2]:{},o=this.getRule(n,i);return o?this.shouldUseIntlApi()?"".concat(this.options.prepend).concat(o.select(r)):this.getSuffixRetroCompatible(o,r):(this.logger.warn("no plural rule found for: ".concat(n)),"")}},{key:"getSuffixRetroCompatible",value:function(n,r){var i=this,o=n.noAbs?n.plurals(r):n.plurals(Math.abs(r)),l=n.numbers[o];this.options.simplifyPluralSuffix&&n.numbers.length===2&&n.numbers[0]===1&&(l===2?l="plural":l===1&&(l=""));var s=function(){return i.options.prepend&&l.toString()?i.options.prepend+l.toString():l.toString()};return this.options.compatibilityJSON==="v1"?l===1?"":typeof l=="number"?"_plural_".concat(l.toString()):s():this.options.compatibilityJSON==="v2"||this.options.simplifyPluralSuffix&&n.numbers.length===2&&n.numbers[0]===1?s():this.options.prepend&&o.toString()?this.options.prepend+o.toString():o.toString()}},{key:"shouldUseIntlApi",value:function(){return!bp.includes(this.options.compatibilityJSON)}}]),e}();function za(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter(function(i){return Object.getOwnPropertyDescriptor(e,i).enumerable})),n.push.apply(n,r)}return n}function ot(e){for(var t=1;t0&&arguments[0]!==void 0?arguments[0]:{};pt(this,e),this.logger=kt.create("interpolator"),this.options=t,this.format=t.interpolation&&t.interpolation.format||function(n){return n},this.init(t)}return ht(e,[{key:"init",value:function(){var n=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};n.interpolation||(n.interpolation={escapeValue:!0});var r=n.interpolation;this.escape=r.escape!==void 0?r.escape:Ip,this.escapeValue=r.escapeValue!==void 0?r.escapeValue:!0,this.useRawValueToEscape=r.useRawValueToEscape!==void 0?r.useRawValueToEscape:!1,this.prefix=r.prefix?Tn(r.prefix):r.prefixEscaped||"{{",this.suffix=r.suffix?Tn(r.suffix):r.suffixEscaped||"}}",this.formatSeparator=r.formatSeparator?r.formatSeparator:r.formatSeparator||",",this.unescapePrefix=r.unescapeSuffix?"":r.unescapePrefix||"-",this.unescapeSuffix=this.unescapePrefix?"":r.unescapeSuffix||"",this.nestingPrefix=r.nestingPrefix?Tn(r.nestingPrefix):r.nestingPrefixEscaped||Tn("$t("),this.nestingSuffix=r.nestingSuffix?Tn(r.nestingSuffix):r.nestingSuffixEscaped||Tn(")"),this.nestingOptionsSeparator=r.nestingOptionsSeparator?r.nestingOptionsSeparator:r.nestingOptionsSeparator||",",this.maxReplaces=r.maxReplaces?r.maxReplaces:1e3,this.alwaysFormat=r.alwaysFormat!==void 0?r.alwaysFormat:!1,this.resetRegExp()}},{key:"reset",value:function(){this.options&&this.init(this.options)}},{key:"resetRegExp",value:function(){var n="".concat(this.prefix,"(.+?)").concat(this.suffix);this.regexp=new RegExp(n,"g");var r="".concat(this.prefix).concat(this.unescapePrefix,"(.+?)").concat(this.unescapeSuffix).concat(this.suffix);this.regexpUnescape=new RegExp(r,"g");var i="".concat(this.nestingPrefix,"(.+?)").concat(this.nestingSuffix);this.nestingRegexp=new RegExp(i,"g")}},{key:"interpolate",value:function(n,r,i,o){var l=this,s,a,u,p=this.options&&this.options.interpolation&&this.options.interpolation.defaultVariables||{};function d(_){return _.replace(/\$/g,"$$$$")}var h=function(f){if(f.indexOf(l.formatSeparator)<0){var c=Ra(r,p,f);return l.alwaysFormat?l.format(c,void 0,i,ot(ot(ot({},o),r),{},{interpolationkey:f})):c}var m=f.split(l.formatSeparator),w=m.shift().trim(),O=m.join(l.formatSeparator).trim();return l.format(Ra(r,p,w),O,i,ot(ot(ot({},o),r),{},{interpolationkey:w}))};this.resetRegExp();var g=o&&o.missingInterpolationHandler||this.options.missingInterpolationHandler,y=o&&o.interpolation&&o.interpolation.skipOnVariables!==void 0?o.interpolation.skipOnVariables:this.options.interpolation.skipOnVariables,k=[{regex:this.regexpUnescape,safeValue:function(f){return d(f)}},{regex:this.regexp,safeValue:function(f){return l.escapeValue?d(l.escape(f)):d(f)}}];return k.forEach(function(_){for(u=0;s=_.regex.exec(n);){var f=s[1].trim();if(a=h(f),a===void 0)if(typeof g=="function"){var c=g(n,s,o);a=typeof c=="string"?c:""}else if(o&&Object.prototype.hasOwnProperty.call(o,f))a="";else if(y){a=s[0];continue}else l.logger.warn("missed to pass in variable ".concat(f," for interpolating ").concat(n)),a="";else typeof a!="string"&&!l.useRawValueToEscape&&(a=Ia(a));var m=_.safeValue(a);if(n=n.replace(s[0],m),y?(_.regex.lastIndex+=a.length,_.regex.lastIndex-=s[0].length):_.regex.lastIndex=0,u++,u>=l.maxReplaces)break}}),n}},{key:"nest",value:function(n,r){var i=this,o=arguments.length>2&&arguments[2]!==void 0?arguments[2]:{},l,s,a;function u(g,y){var k=this.nestingOptionsSeparator;if(g.indexOf(k)<0)return g;var _=g.split(new RegExp("".concat(k,"[ ]*{"))),f="{".concat(_[1]);g=_[0],f=this.interpolate(f,a);var c=f.match(/'/g),m=f.match(/"/g);(c&&c.length%2===0&&!m||m.length%2!==0)&&(f=f.replace(/'/g,'"'));try{a=JSON.parse(f),y&&(a=ot(ot({},y),a))}catch(w){return this.logger.warn("failed parsing options string in nesting for key ".concat(g),w),"".concat(g).concat(k).concat(f)}return delete a.defaultValue,g}for(;l=this.nestingRegexp.exec(n);){var p=[];a=ot({},o),a=a.replace&&typeof a.replace!="string"?a.replace:a,a.applyPostProcessor=!1,delete a.defaultValue;var d=!1;if(l[0].indexOf(this.formatSeparator)!==-1&&!/{.*}/.test(l[1])){var h=l[1].split(this.formatSeparator).map(function(g){return g.trim()});l[1]=h.shift(),p=h,d=!0}if(s=r(u.call(this,l[1].trim(),a),a),s&&l[0]===n&&typeof s!="string")return s;typeof s!="string"&&(s=Ia(s)),s||(this.logger.warn("missed to resolve ".concat(l[1]," for nesting ").concat(n)),s=""),d&&(s=p.reduce(function(g,y){return i.format(g,y,o.lng,ot(ot({},o),{},{interpolationkey:l[1].trim()}))},s.trim())),n=n.replace(l[0],s),this.regexp.lastIndex=0}return n}}]),e}();function ba(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter(function(i){return Object.getOwnPropertyDescriptor(e,i).enumerable})),n.push.apply(n,r)}return n}function xt(e){for(var t=1;t-1){var r=e.split("(");t=r[0].toLowerCase().trim();var i=r[1].substring(0,r[1].length-1);if(t==="currency"&&i.indexOf(":")<0)n.currency||(n.currency=i.trim());else if(t==="relativetime"&&i.indexOf(":")<0)n.range||(n.range=i.trim());else{var o=i.split(";");o.forEach(function(l){if(l){var s=l.split(":"),a=_p(s),u=a[0],p=a.slice(1),d=p.join(":").trim().replace(/^'+|'+$/g,"");n[u.trim()]||(n[u.trim()]=d),d==="false"&&(n[u.trim()]=!1),d==="true"&&(n[u.trim()]=!0),isNaN(d)||(n[u.trim()]=parseInt(d,10))}})}}return{formatName:t,formatOptions:n}}function Rn(e){var t={};return function(r,i,o){var l=i+JSON.stringify(o),s=t[l];return s||(s=e(i,o),t[l]=s),s(r)}}var Kp=function(){function e(){var t=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};pt(this,e),this.logger=kt.create("formatter"),this.options=t,this.formats={number:Rn(function(n,r){var i=new Intl.NumberFormat(n,xt({},r));return function(o){return i.format(o)}}),currency:Rn(function(n,r){var i=new Intl.NumberFormat(n,xt(xt({},r),{},{style:"currency"}));return function(o){return i.format(o)}}),datetime:Rn(function(n,r){var i=new Intl.DateTimeFormat(n,xt({},r));return function(o){return i.format(o)}}),relativetime:Rn(function(n,r){var i=new Intl.RelativeTimeFormat(n,xt({},r));return function(o){return i.format(o,r.range||"day")}}),list:Rn(function(n,r){var i=new Intl.ListFormat(n,xt({},r));return function(o){return i.format(o)}})},this.init(t)}return ht(e,[{key:"init",value:function(n){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{interpolation:{}},i=r.interpolation;this.formatSeparator=i.formatSeparator?i.formatSeparator:i.formatSeparator||","}},{key:"add",value:function(n,r){this.formats[n.toLowerCase().trim()]=r}},{key:"addCached",value:function(n,r){this.formats[n.toLowerCase().trim()]=Rn(r)}},{key:"format",value:function(n,r,i){var o=this,l=arguments.length>3&&arguments[3]!==void 0?arguments[3]:{},s=r.split(this.formatSeparator),a=s.reduce(function(u,p){var d=Bp(p),h=d.formatName,g=d.formatOptions;if(o.formats[h]){var y=u;try{var k=l&&l.formatParams&&l.formatParams[l.interpolationkey]||{},_=k.locale||k.lng||l.locale||l.lng||i;y=o.formats[h](u,_,xt(xt(xt({},g),l),k))}catch(f){o.logger.warn(f)}return y}else o.logger.warn("there was no format function for ".concat(h));return u},n);return a}}]),e}();function Va(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter(function(i){return Object.getOwnPropertyDescriptor(e,i).enumerable})),n.push.apply(n,r)}return n}function $a(e){for(var t=1;t"u"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch{return!1}}function Gp(e,t){e.pending[t]!==void 0&&(delete e.pending[t],e.pendingCount--)}var Yp=function(e){ho(n,e);var t=Hp(n);function n(r,i,o){var l,s=arguments.length>3&&arguments[3]!==void 0?arguments[3]:{};return pt(this,n),l=t.call(this),mo&&nn.call(Gt(l)),l.backend=r,l.store=i,l.services=o,l.languageUtils=o.languageUtils,l.options=s,l.logger=kt.create("backendConnector"),l.waitingReads=[],l.maxParallelReads=s.maxParallelReads||10,l.readingCalls=0,l.maxRetries=s.maxRetries>=0?s.maxRetries:5,l.retryTimeout=s.retryTimeout>=1?s.retryTimeout:350,l.state={},l.queue=[],l.backend&&l.backend.init&&l.backend.init(o,s.backend,s),l}return ht(n,[{key:"queueLoad",value:function(i,o,l,s){var a=this,u={},p={},d={},h={};return i.forEach(function(g){var y=!0;o.forEach(function(k){var _="".concat(g,"|").concat(k);!l.reload&&a.store.hasResourceBundle(g,k)?a.state[_]=2:a.state[_]<0||(a.state[_]===1?p[_]===void 0&&(p[_]=!0):(a.state[_]=1,y=!1,p[_]===void 0&&(p[_]=!0),u[_]===void 0&&(u[_]=!0),h[k]===void 0&&(h[k]=!0)))}),y||(d[g]=!0)}),(Object.keys(u).length||Object.keys(p).length)&&this.queue.push({pending:p,pendingCount:Object.keys(p).length,loaded:{},errors:[],callback:s}),{toLoad:Object.keys(u),pending:Object.keys(p),toLoadLanguages:Object.keys(d),toLoadNamespaces:Object.keys(h)}}},{key:"loaded",value:function(i,o,l){var s=i.split("|"),a=s[0],u=s[1];o&&this.emit("failedLoading",a,u,o),l&&this.store.addResourceBundle(a,u,l),this.state[i]=o?-1:2;var p={};this.queue.forEach(function(d){xp(d.loaded,[a],u),Gp(d,i),o&&d.errors.push(o),d.pendingCount===0&&!d.done&&(Object.keys(d.loaded).forEach(function(h){p[h]||(p[h]={});var g=d.loaded[h];g.length&&g.forEach(function(y){p[h][y]===void 0&&(p[h][y]=!0)})}),d.done=!0,d.errors.length?d.callback(d.errors):d.callback())}),this.emit("loaded",p),this.queue=this.queue.filter(function(d){return!d.done})}},{key:"read",value:function(i,o,l){var s=this,a=arguments.length>3&&arguments[3]!==void 0?arguments[3]:0,u=arguments.length>4&&arguments[4]!==void 0?arguments[4]:this.retryTimeout,p=arguments.length>5?arguments[5]:void 0;if(!i.length)return p(null,{});if(this.readingCalls>=this.maxParallelReads){this.waitingReads.push({lng:i,ns:o,fcName:l,tried:a,wait:u,callback:p});return}this.readingCalls++;var d=function(k,_){if(s.readingCalls--,s.waitingReads.length>0){var f=s.waitingReads.shift();s.read(f.lng,f.ns,f.fcName,f.tried,f.wait,f.callback)}if(k&&_&&a2&&arguments[2]!==void 0?arguments[2]:{},a=arguments.length>3?arguments[3]:void 0;if(!this.backend)return this.logger.warn("No backend was added via i18next.use. Will not load resources."),a&&a();typeof i=="string"&&(i=this.languageUtils.toResolveHierarchy(i)),typeof o=="string"&&(o=[o]);var u=this.queueLoad(i,o,s,a);if(!u.toLoad.length)return u.pending.length||a(),null;u.toLoad.forEach(function(p){l.loadOne(p)})}},{key:"load",value:function(i,o,l){this.prepareLoading(i,o,{},l)}},{key:"reload",value:function(i,o,l){this.prepareLoading(i,o,{reload:!0},l)}},{key:"loadOne",value:function(i){var o=this,l=arguments.length>1&&arguments[1]!==void 0?arguments[1]:"",s=i.split("|"),a=s[0],u=s[1];this.read(a,u,"read",void 0,void 0,function(p,d){p&&o.logger.warn("".concat(l,"loading namespace ").concat(u," for language ").concat(a," failed"),p),!p&&d&&o.logger.log("".concat(l,"loaded namespace ").concat(u," for language ").concat(a),d),o.loaded(i,p,d)})}},{key:"saveMissing",value:function(i,o,l,s,a){var u=arguments.length>5&&arguments[5]!==void 0?arguments[5]:{},p=arguments.length>6&&arguments[6]!==void 0?arguments[6]:function(){};if(this.services.utils&&this.services.utils.hasLoadedNamespace&&!this.services.utils.hasLoadedNamespace(o)){this.logger.warn('did not save key "'.concat(l,'" as the namespace "').concat(o,'" was not yet loaded'),"This means something IS WRONG in your setup. You access the t function before i18next.init / i18next.loadNamespace / i18next.changeLanguage was done. Wait for the callback or Promise to resolve before accessing it!!!");return}if(!(l==null||l==="")){if(this.backend&&this.backend.create){var d=$a($a({},u),{},{isUpdate:a}),h=this.backend.create.bind(this.backend);if(h.length<6)try{var g;h.length===5?g=h(i,o,l,s,d):g=h(i,o,l,s),g&&typeof g.then=="function"?g.then(function(y){return p(null,y)}).catch(p):p(null,g)}catch(y){p(y)}else h(i,o,l,s,p,d)}!i||!i[0]||this.store.addResource(i[0],o,l,s)}}}]),n}(nn);function Ma(){return{debug:!1,initImmediate:!0,ns:["translation"],defaultNS:["translation"],fallbackLng:["dev"],fallbackNS:!1,supportedLngs:!1,nonExplicitSupportedLngs:!1,load:"all",preload:!1,simplifyPluralSuffix:!0,keySeparator:".",nsSeparator:":",pluralSeparator:"_",contextSeparator:"_",partialBundledLanguages:!1,saveMissing:!1,updateMissing:!1,saveMissingTo:"fallback",saveMissingPlurals:!0,missingKeyHandler:!1,missingInterpolationHandler:!1,postProcess:!1,postProcessPassResolved:!1,returnNull:!0,returnEmptyString:!0,returnObjects:!1,joinArrays:!1,returnedObjectHandler:!1,parseMissingKeyHandler:!1,appendNamespaceToMissingKey:!1,appendNamespaceToCIMode:!1,overloadTranslationOptionHandler:function(t){var n={};if(me(t[1])==="object"&&(n=t[1]),typeof t[1]=="string"&&(n.defaultValue=t[1]),typeof t[2]=="string"&&(n.tDescription=t[2]),me(t[2])==="object"||me(t[3])==="object"){var r=t[3]||t[2];Object.keys(r).forEach(function(i){n[i]=r[i]})}return n},interpolation:{escapeValue:!0,format:function(t,n,r,i){return t},prefix:"{{",suffix:"}}",formatSeparator:",",unescapePrefix:"-",nestingPrefix:"$t(",nestingSuffix:")",nestingOptionsSeparator:",",maxReplaces:1e3,skipOnVariables:!0}}}function Ba(e){return typeof e.ns=="string"&&(e.ns=[e.ns]),typeof e.fallbackLng=="string"&&(e.fallbackLng=[e.fallbackLng]),typeof e.fallbackNS=="string"&&(e.fallbackNS=[e.fallbackNS]),e.supportedLngs&&e.supportedLngs.indexOf("cimode")<0&&(e.supportedLngs=e.supportedLngs.concat(["cimode"])),e}function Ka(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter(function(i){return Object.getOwnPropertyDescriptor(e,i).enumerable})),n.push.apply(n,r)}return n}function gt(e){for(var t=1;t"u"||!Reflect.construct||Reflect.construct.sham)return!1;if(typeof Proxy=="function")return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch{return!1}}function ai(){}function Jp(e){var t=Object.getOwnPropertyNames(Object.getPrototypeOf(e));t.forEach(function(n){typeof e[n]=="function"&&(e[n]=e[n].bind(e))})}var Vi=function(e){ho(n,e);var t=Qp(n);function n(){var r,i=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},o=arguments.length>1?arguments[1]:void 0;if(pt(this,n),r=t.call(this),mo&&nn.call(Gt(r)),r.options=Ba(i),r.services={},r.logger=kt,r.modules={external:[]},Jp(Gt(r)),o&&!r.isInitialized&&!i.isClone){if(!r.options.initImmediate)return r.init(i,o),ei(r,Gt(r));setTimeout(function(){r.init(i,o)},0)}return r}return ht(n,[{key:"init",value:function(){var i=this,o=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},l=arguments.length>1?arguments[1]:void 0;typeof o=="function"&&(l=o,o={}),!o.defaultNS&&o.defaultNS!==!1&&o.ns&&(typeof o.ns=="string"?o.defaultNS=o.ns:o.ns.indexOf("translation")<0&&(o.defaultNS=o.ns[0]));var s=Ma();this.options=gt(gt(gt({},s),this.options),Ba(o)),this.options.compatibilityAPI!=="v1"&&(this.options.interpolation=gt(gt({},s.interpolation),this.options.interpolation)),o.keySeparator!==void 0&&(this.options.userDefinedKeySeparator=o.keySeparator),o.nsSeparator!==void 0&&(this.options.userDefinedNsSeparator=o.nsSeparator);function a(f){return f?typeof f=="function"?new f:f:null}if(!this.options.isClone){this.modules.logger?kt.init(a(this.modules.logger),this.options):kt.init(null,this.options);var u;this.modules.formatter?u=this.modules.formatter:typeof Intl<"u"&&(u=Kp);var p=new Da(this.options);this.store=new jp(this.options.resources,this.options);var d=this.services;d.logger=kt,d.resourceStore=this.store,d.languageUtils=p,d.pluralResolver=new $p(p,{prepend:this.options.pluralSeparator,compatibilityJSON:this.options.compatibilityJSON,simplifyPluralSuffix:this.options.simplifyPluralSuffix}),u&&(!this.options.interpolation.format||this.options.interpolation.format===s.interpolation.format)&&(d.formatter=a(u),d.formatter.init(d,this.options),this.options.interpolation.format=d.formatter.format.bind(d.formatter)),d.interpolator=new Mp(this.options),d.utils={hasLoadedNamespace:this.hasLoadedNamespace.bind(this)},d.backendConnector=new Yp(a(this.modules.backend),d.resourceStore,d,this.options),d.backendConnector.on("*",function(f){for(var c=arguments.length,m=new Array(c>1?c-1:0),w=1;w1?c-1:0),w=1;w0&&h[0]!=="dev"&&(this.options.lng=h[0])}!this.services.languageDetector&&!this.options.lng&&this.logger.warn("init: no languageDetector is used and no lng is defined");var g=["getResource","hasResourceBundle","getResourceBundle","getDataByLanguage"];g.forEach(function(f){i[f]=function(){var c;return(c=i.store)[f].apply(c,arguments)}});var y=["addResource","addResources","addResourceBundle","removeResourceBundle"];y.forEach(function(f){i[f]=function(){var c;return(c=i.store)[f].apply(c,arguments),i}});var k=fr(),_=function(){var c=function(w,O){i.isInitialized&&!i.initializedStoreOnce&&i.logger.warn("init: i18next is already initialized. You should call init just once!"),i.isInitialized=!0,i.options.isClone||i.logger.log("initialized",i.options),i.emit("initialized",i.options),k.resolve(O),l(w,O)};if(i.languages&&i.options.compatibilityAPI!=="v1"&&!i.isInitialized)return c(null,i.t.bind(i));i.changeLanguage(i.options.lng,c)};return this.options.resources||!this.options.initImmediate?_():setTimeout(_,0),k}},{key:"loadResources",value:function(i){var o=this,l=arguments.length>1&&arguments[1]!==void 0?arguments[1]:ai,s=l,a=typeof i=="string"?i:this.language;if(typeof i=="function"&&(s=i),!this.options.resources||this.options.partialBundledLanguages){if(a&&a.toLowerCase()==="cimode")return s();var u=[],p=function(g){if(g){var y=o.services.languageUtils.toResolveHierarchy(g);y.forEach(function(k){u.indexOf(k)<0&&u.push(k)})}};if(a)p(a);else{var d=this.services.languageUtils.getFallbackCodes(this.options.fallbackLng);d.forEach(function(h){return p(h)})}this.options.preload&&this.options.preload.forEach(function(h){return p(h)}),this.services.backendConnector.load(u,this.options.ns,function(h){!h&&!o.resolvedLanguage&&o.language&&o.setResolvedLanguage(o.language),s(h)})}else s(null)}},{key:"reloadResources",value:function(i,o,l){var s=fr();return i||(i=this.languages),o||(o=this.options.ns),l||(l=ai),this.services.backendConnector.reload(i,o,function(a){s.resolve(),l(a)}),s}},{key:"use",value:function(i){if(!i)throw new Error("You are passing an undefined module! Please check the object you are passing to i18next.use()");if(!i.type)throw new Error("You are passing a wrong module! Please check the object you are passing to i18next.use()");return i.type==="backend"&&(this.modules.backend=i),(i.type==="logger"||i.log&&i.warn&&i.error)&&(this.modules.logger=i),i.type==="languageDetector"&&(this.modules.languageDetector=i),i.type==="i18nFormat"&&(this.modules.i18nFormat=i),i.type==="postProcessor"&&Ec.addPostProcessor(i),i.type==="formatter"&&(this.modules.formatter=i),i.type==="3rdParty"&&this.modules.external.push(i),this}},{key:"setResolvedLanguage",value:function(i){if(!(!i||!this.languages)&&!(["cimode","dev"].indexOf(i)>-1))for(var o=0;o-1)&&this.store.hasLanguageSomeTranslations(l)){this.resolvedLanguage=l;break}}}},{key:"changeLanguage",value:function(i,o){var l=this;this.isLanguageChangingTo=i;var s=fr();this.emit("languageChanging",i);var a=function(h){l.language=h,l.languages=l.services.languageUtils.toResolveHierarchy(h),l.resolvedLanguage=void 0,l.setResolvedLanguage(h)},u=function(h,g){g?(a(g),l.translator.changeLanguage(g),l.isLanguageChangingTo=void 0,l.emit("languageChanged",g),l.logger.log("languageChanged",g)):l.isLanguageChangingTo=void 0,s.resolve(function(){return l.t.apply(l,arguments)}),o&&o(h,function(){return l.t.apply(l,arguments)})},p=function(h){!i&&!h&&l.services.languageDetector&&(h=[]);var g=typeof h=="string"?h:l.services.languageUtils.getBestMatchFromCodes(h);g&&(l.language||a(g),l.translator.language||l.translator.changeLanguage(g),l.services.languageDetector&&l.services.languageDetector.cacheUserLanguage&&l.services.languageDetector.cacheUserLanguage(g)),l.loadResources(g,function(y){u(y,g)})};return!i&&this.services.languageDetector&&!this.services.languageDetector.async?p(this.services.languageDetector.detect()):!i&&this.services.languageDetector&&this.services.languageDetector.async?this.services.languageDetector.detect.length===0?this.services.languageDetector.detect().then(p):this.services.languageDetector.detect(p):p(i),s}},{key:"getFixedT",value:function(i,o,l){var s=this,a=function u(p,d){var h;if(me(d)!=="object"){for(var g=arguments.length,y=new Array(g>2?g-2:0),k=2;k1&&arguments[1]!==void 0?arguments[1]:{};if(!this.isInitialized)return this.logger.warn("hasLoadedNamespace: i18next was not initialized",this.languages),!1;if(!this.languages||!this.languages.length)return this.logger.warn("hasLoadedNamespace: i18n.languages were undefined or empty",this.languages),!1;var s=this.resolvedLanguage||this.languages[0],a=this.options?this.options.fallbackLng:!1,u=this.languages[this.languages.length-1];if(s.toLowerCase()==="cimode")return!0;var p=function(g,y){var k=o.services.backendConnector.state["".concat(g,"|").concat(y)];return k===-1||k===2};if(l.precheck){var d=l.precheck(this,p);if(d!==void 0)return d}return!!(this.hasResourceBundle(s,i)||!this.services.backendConnector.backend||this.options.resources&&!this.options.partialBundledLanguages||p(s,i)&&(!a||p(u,i)))}},{key:"loadNamespaces",value:function(i,o){var l=this,s=fr();return this.options.ns?(typeof i=="string"&&(i=[i]),i.forEach(function(a){l.options.ns.indexOf(a)<0&&l.options.ns.push(a)}),this.loadResources(function(a){s.resolve(),o&&o(a)}),s):(o&&o(),Promise.resolve())}},{key:"loadLanguages",value:function(i,o){var l=fr();typeof i=="string"&&(i=[i]);var s=this.options.preload||[],a=i.filter(function(u){return s.indexOf(u)<0});return a.length?(this.options.preload=s.concat(a),this.loadResources(function(u){l.resolve(),o&&o(u)}),l):(o&&o(),Promise.resolve())}},{key:"dir",value:function(i){if(i||(i=this.resolvedLanguage||(this.languages&&this.languages.length>0?this.languages[0]:this.language)),!i)return"rtl";var o=["ar","shu","sqr","ssh","xaa","yhd","yud","aao","abh","abv","acm","acq","acw","acx","acy","adf","ads","aeb","aec","afb","ajp","apc","apd","arb","arq","ars","ary","arz","auz","avl","ayh","ayl","ayn","ayp","bbz","pga","he","iw","ps","pbt","pbu","pst","prp","prd","ug","ur","ydd","yds","yih","ji","yi","hbo","men","xmn","fa","jpr","peo","pes","prs","dv","sam","ckb"],l=this.services&&this.services.languageUtils||new Da(Ma());return o.indexOf(l.getLanguagePartFromCode(i))>-1||i.toLowerCase().indexOf("-arab")>1?"rtl":"ltr"}},{key:"cloneInstance",value:function(){var i=this,o=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},l=arguments.length>1&&arguments[1]!==void 0?arguments[1]:ai,s=gt(gt(gt({},this.options),o),{isClone:!0}),a=new n(s);(o.debug!==void 0||o.prefix!==void 0)&&(a.logger=a.logger.clone(o));var u=["store","services","language"];return u.forEach(function(p){a[p]=i[p]}),a.services=gt({},this.services),a.services.utils={hasLoadedNamespace:a.hasLoadedNamespace.bind(a)},a.translator=new Fa(a.services,a.options),a.translator.on("*",function(p){for(var d=arguments.length,h=new Array(d>1?d-1:0),g=1;g0&&arguments[0]!==void 0?arguments[0]:{},t=arguments.length>1?arguments[1]:void 0;return new Vi(e,t)});var fe=Vi.createInstance();fe.createInstance=Vi.createInstance;fe.createInstance;fe.dir;fe.init;fe.loadResources;fe.reloadResources;fe.use;fe.changeLanguage;fe.getFixedT;fe.t;fe.exists;fe.setDefaultNamespace;fe.hasLoadedNamespace;fe.loadNamespaces;fe.loadLanguages;function Zp(e,t){if(e==null)return{};var n={},r=Object.keys(e),i,o;for(o=0;o=0)&&(n[i]=e[i]);return n}function ks(e,t){if(e==null)return{};var n=Zp(e,t),r,i;if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i=0)&&Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var qp={area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0};const eh=uc(qp);var th=/\s([^'"/\s><]+?)[\s/>]|([^\s=]+)=\s?(".*?"|'.*?')/g;function Ha(e){var t={type:"tag",name:"",voidElement:!1,attrs:{},children:[]},n=e.match(/<\/?([^\s]+?)[/\s>]/);if(n&&(t.name=n[1],(eh[n[1]]||e.charAt(e.length-2)==="/")&&(t.voidElement=!0),t.name.startsWith("!--"))){var r=e.indexOf("-->");return{type:"comment",comment:r!==-1?e.slice(4,r):""}}for(var i=new RegExp(th),o=null;(o=i.exec(e))!==null;)if(o[0].trim())if(o[1]){var l=o[1].trim(),s=[l,""];l.indexOf("=")>-1&&(s=l.split("=")),t.attrs[s[0]]=s[1],i.lastIndex--}else o[2]&&(t.attrs[o[2]]=o[3].trim().substring(1,o[3].length-1));return t}var nh=/<[a-zA-Z0-9\-\!\/](?:"[^"]*"|'[^']*'|[^'">])*>/g,rh=/^\s*$/,ih=Object.create(null);function xc(e,t){switch(t.type){case"text":return e+t.content;case"tag":return e+="<"+t.name+(t.attrs?function(n){var r=[];for(var i in n)r.push(i+'="'+n[i]+'"');return r.length?" "+r.join(" "):""}(t.attrs):"")+(t.voidElement?"/>":">"),t.voidElement?e:e+t.children.reduce(xc,"")+"";case"comment":return e+""}}var oh={parse:function(e,t){t||(t={}),t.components||(t.components=ih);var n,r=[],i=[],o=-1,l=!1;if(e.indexOf("<")!==0){var s=e.indexOf("<");r.push({type:"text",content:s===-1?e:e.substring(0,s)})}return e.replace(nh,function(a,u){if(l){if(a!=="")return;l=!1}var p,d=a.charAt(1)!=="/",h=a.startsWith("");return{type:"comment",comment:r!==-1?e.slice(4,r):""}}for(var i=new RegExp(Zp),o=null;(o=i.exec(e))!==null;)if(o[0].trim())if(o[1]){var s=o[1].trim(),l=[s,""];s.indexOf("=")>-1&&(l=s.split("=")),t.attrs[l[0]]=l[1],i.lastIndex--}else o[2]&&(t.attrs[o[2]]=o[3].trim().substring(1,o[3].length-1));return t}var qp=/<[a-zA-Z0-9\-\!\/](?:"[^"]*"|'[^']*'|[^'">])*>/g,eh=/^\s*$/,th=Object.create(null);function _c(e,t){switch(t.type){case"text":return e+t.content;case"tag":return e+="<"+t.name+(t.attrs?function(n){var r=[];for(var i in n)r.push(i+'="'+n[i]+'"');return r.length?" "+r.join(" "):""}(t.attrs):"")+(t.voidElement?"/>":">"),t.voidElement?e:e+t.children.reduce(_c,"")+"";case"comment":return e+""}}var nh={parse:function(e,t){t||(t={}),t.components||(t.components=th);var n,r=[],i=[],o=-1,s=!1;if(e.indexOf("<")!==0){var l=e.indexOf("<");r.push({type:"text",content:l===-1?e:e.substring(0,l)})}return e.replace(qp,function(a,u){if(s){if(a!=="")return;s=!1}var p,d=a.charAt(1)!=="/",h=a.startsWith(" " + << "Subgraph[" << subm_idx_to << "]/" << port_idx_to); + NPUW_ASSERT(m_subrequests[subm_idx_from]); // prod request is created + NPUW_ASSERT(m_subrequests[subm_idx_to]); // cons request is created + NPUW_ASSERT(m_subrequests[subm_idx_from]._ptr != m_subrequests[subm_idx_to]._ptr); + + const auto& iport = m_subrequests[subm_idx_to]->get_compiled_model()->inputs()[port_idx_to]; + const auto& oport = m_subrequests[subm_idx_from]->get_compiled_model()->outputs()[port_idx_from]; + const auto& tensor = m_subrequests[subm_idx_from]->get_tensor(oport); + LOG_DEBUG("Set Subgraph[" << subm_idx_to << "]/" << iport << " to Subgraph[" << subm_idx_from << "]/" << oport); + m_subrequests[subm_idx_to]->set_tensor(iport, tensor); + } // for(map) + LOG_INFO("Done"); + + init_gio(); + + for (size_t i = 0; i < m_num_submodels; i++) { + LOG_VERB("Trying to preemptively set tensors for Subgraph[" << i << "]..."); + LOG_BLOCK(); + auto& comp_model_desc = m_npuw_model->m_compiled_submodels[i]; + if (!comp_model_desc.compiled_model && !comp_model_desc.replaced_by) { + continue; // Optimized out + } + unpack_closure(i, m_subrequests[i]); + LOG_VERB("Done"); + } +} + +bool ov::npuw::UnfoldInferRequest::valid_subrequest(std::size_t idx) const { + return m_subrequests.at(idx) != nullptr; +} + +void ov::npuw::UnfoldInferRequest::infer() { + const bool do_async = m_npuw_model->m_cfg.get<::intel_npu::NPUW_FUNCALL_ASYNC>(); + + auto prepare = [&](std::size_t idx) { + if (idx >= m_subrequests.size()) { + return; + } + bind_global_params(idx, m_subrequests[idx]); + bind_global_results(idx, m_subrequests[idx]); + }; + auto wait_and_clear = [](RqPtrs& rqs) { + for (auto&& r : rqs) { + r->wait(); + } + rqs.clear(); + }; + + if (do_async) { + std::size_t past_repl_id = 0u; + RqPtrs previous_requests; + + prepare(0); + for (std::size_t idx = 0; idx < m_num_submodels; idx++) { + auto& subr = m_subrequests[idx]; + if (!subr) { + prepare(idx + 1); + continue; + } + auto& comp_model_desc = m_npuw_model->m_compiled_submodels[idx]; + const auto this_repl_id = comp_model_desc.replaced_by.value_or(idx); + if (this_repl_id != past_repl_id) { + // For non-repeating blocks, the above value_or returns idx + // For repeating blocks, it returns the function group id + // If either is not equal to the past_repl_id, make a barrier here + wait_and_clear(previous_requests); + past_repl_id = this_repl_id; + } + subr->start_async(); + previous_requests.push_back(subr); + prepare(idx + 1); + } + wait_and_clear(previous_requests); + } else { + prepare(0); + for (std::size_t idx = 0; idx < m_num_submodels; idx++) { + auto& subr = m_subrequests[idx]; + if (!subr) { + prepare(idx + 1); + continue; + } + subr->start_async(); + prepare(idx + 1); + subr->wait(); + } + } // (async) +} diff --git a/src/plugins/intel_npu/src/plugin/npuw/unfold_sync_infer_request.hpp b/src/plugins/intel_npu/src/plugin/npuw/unfold_sync_infer_request.hpp new file mode 100644 index 00000000000000..76b67571ec4c40 --- /dev/null +++ b/src/plugins/intel_npu/src/plugin/npuw/unfold_sync_infer_request.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include + +#include "base_sync_infer_request.hpp" + +namespace ov { +namespace npuw { + +class UnfoldInferRequest final : public IBaseInferRequest { +public: + explicit UnfoldInferRequest(const std::shared_ptr& compiled_model); + + //////////////////////////////////// + // implement IBaseInferRequest - nether of these are required here + // this hierarchy needs revew + void prepare_for_infer() override {} + bool valid_subrequest(std::size_t idx) const override; + void start_subrequest(std::size_t) override {} + void run_subrequest_for_success(std::size_t, bool&) override {} + void subscribe_subrequest(std::size_t, Completed cb) override {} + void complete_subrequest(std::size_t) override {} + void cancel_subrequest(std::size_t) override {} + bool supports_async_pipeline() const override { + return false; + } + void update_subrequest_links(std::size_t) override {} + +private: + void infer() override; +}; + +} // namespace npuw +} // namespace ov diff --git a/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_init.hpp b/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_init.hpp index 94f2c4ce06ee61..01b2de868e7572 100644 --- a/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_init.hpp +++ b/src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_init.hpp @@ -52,6 +52,9 @@ class ZeroInitStructsHolder final { inline uint32_t getMutableCommandListVersion() const { return mutable_command_list_version; } + inline ze_api_version_t getZeDrvApiVersion() const { + return ze_drv_api_version; + } // Helper function to check if extension with exists and its newer than inline bool isExtensionSupported(std::string ext_name, uint32_t version) const { auto iter = driver_extension_properties.find(ext_name); @@ -78,6 +81,8 @@ class ZeroInitStructsHolder final { ze_driver_properties_t driver_properties = {}; uint32_t mutable_command_list_version = 0; + + ze_api_version_t ze_drv_api_version = {}; }; } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/utils/src/zero/CMakeLists.txt b/src/plugins/intel_npu/src/utils/src/zero/CMakeLists.txt index f1e81267dc2b73..8eb9cb31db870f 100644 --- a/src/plugins/intel_npu/src/utils/src/zero/CMakeLists.txt +++ b/src/plugins/intel_npu/src/utils/src/zero/CMakeLists.txt @@ -21,10 +21,7 @@ target_include_directories(${TARGET_NAME} $ $) - -target_link_libraries( - ${TARGET_NAME} - PUBLIC openvino::runtime openvino::runtime::dev) +target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime::dev) # # targets install @@ -35,17 +32,19 @@ ov_developer_package_export_targets(TARGET openvino::npu_zero_utils INSTALL_INCLUDE_DIRECTORIES $) +ov_developer_package_export_targets(TARGET level-zero-ext) +ov_install_static_lib(level-zero-ext ${NPU_PLUGIN_COMPONENT}) + if(TARGET ze_loader) ov_developer_package_export_targets(TARGET ze_loader) - ov_developer_package_export_targets(TARGET utils) - ov_developer_package_export_targets(TARGET level-zero-ext) - ov_install_static_lib(ze_loader ${NPU_PLUGIN_COMPONENT}) + + # TODO: remove once https://github.com/oneapi-src/level-zero/pull/243 is merged + ov_developer_package_export_targets(TARGET utils) ov_install_static_lib(utils ${NPU_PLUGIN_COMPONENT}) - ov_install_static_lib(level-zero-ext ${NPU_PLUGIN_COMPONENT}) # Support tests to run with ze_loader install(TARGETS ze_loader - RUNTIME DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL - LIBRARY DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) + RUNTIME DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL + LIBRARY DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) endif() diff --git a/src/plugins/intel_npu/src/utils/src/zero/zero_init.cpp b/src/plugins/intel_npu/src/utils/src/zero/zero_init.cpp index ac6e3c2306f39c..e87f8db788b9b8 100644 --- a/src/plugins/intel_npu/src/utils/src/zero/zero_init.cpp +++ b/src/plugins/intel_npu/src/utils/src/zero/zero_init.cpp @@ -79,7 +79,6 @@ ZeroInitStructsHolder::ZeroInitStructsHolder() : log("NPUZeroInitStructsHolder", } // Check L0 API version - ze_api_version_t ze_drv_api_version = {}; THROW_ON_FAIL_FOR_LEVELZERO("zeDriverGetApiVersion", zeDriverGetApiVersion(driver_handle, &ze_drv_api_version)); if (ZE_MAJOR_VERSION(ZE_API_VERSION_CURRENT) != ZE_MAJOR_VERSION(ze_drv_api_version)) { diff --git a/src/plugins/intel_npu/src/utils/src/zero/zero_wrappers.cpp b/src/plugins/intel_npu/src/utils/src/zero/zero_wrappers.cpp index 39bd92e7ecffd0..858e65d4b5e6ee 100644 --- a/src/plugins/intel_npu/src/utils/src/zero/zero_wrappers.cpp +++ b/src/plugins/intel_npu/src/utils/src/zero/zero_wrappers.cpp @@ -107,11 +107,16 @@ CommandList::~CommandList() { } } void CommandList::updateMutableCommandList(uint32_t arg_index, const void* arg_value) const { - ze_mutable_graph_argument_exp_desc_t desc = {ZE_STRUCTURE_TYPE_MUTABLE_GRAPH_ARGUMENT_EXP_DESC, - nullptr, - _command_id, - arg_index, - arg_value}; + ze_mutable_graph_argument_exp_desc_t desc = { + (ZE_MAJOR_VERSION(_initStructs->getZeDrvApiVersion()) > 1 || + (ZE_MAJOR_VERSION(_initStructs->getZeDrvApiVersion()) == 1 && + ZE_MINOR_VERSION(_initStructs->getZeDrvApiVersion()) >= 11)) + ? ZE_STRUCTURE_TYPE_MUTABLE_GRAPH_ARGUMENT_EXP_DESC + : static_cast(ZE_STRUCTURE_TYPE_MUTABLE_GRAPH_ARGUMENT_EXP_DESC_DEPRECATED), + nullptr, + _command_id, + arg_index, + arg_value}; ze_mutable_commands_exp_desc_t mutable_commands_exp_desc_t = {ZE_STRUCTURE_TYPE_MUTABLE_COMMANDS_EXP_DESC, &desc, diff --git a/src/plugins/intel_npu/thirdparty/level-zero-ext b/src/plugins/intel_npu/thirdparty/level-zero-ext index a6487cc2c5da9a..110f48ee8eda22 160000 --- a/src/plugins/intel_npu/thirdparty/level-zero-ext +++ b/src/plugins/intel_npu/thirdparty/level-zero-ext @@ -1 +1 @@ -Subproject commit a6487cc2c5da9aa13db9e005a320a1b6a0ee5919 +Subproject commit 110f48ee8eda22d8b40daeeecdbbed0fc3b08f8b diff --git a/src/plugins/intel_npu/tools/protopipe/README.md b/src/plugins/intel_npu/tools/protopipe/README.md index 00849ad8bddc9a..807bd744851b70 100644 --- a/src/plugins/intel_npu/tools/protopipe/README.md +++ b/src/plugins/intel_npu/tools/protopipe/README.md @@ -60,6 +60,7 @@ log_level: INFO - `ol` - **Optional**. Output layer layout. - `iml` - **Optional**. Input model layout. - `oml` - **Optional**. Output model layout. +- `reshape` - **Optional**. Set shape for input layers. For example, "input1: [1,3,224,224], input2: [1,4]" or "[1,3,224,224]" in case of one input layer. Examples: ``` diff --git a/src/plugins/intel_npu/tools/protopipe/src/parser/config.cpp b/src/plugins/intel_npu/tools/protopipe/src/parser/config.cpp index c2a1bd6415d595..b9f03a97ba3f69 100644 --- a/src/plugins/intel_npu/tools/protopipe/src/parser/config.cpp +++ b/src/plugins/intel_npu/tools/protopipe/src/parser/config.cpp @@ -345,6 +345,10 @@ struct convert { params.output_model_layout = node["oml"].as>(); } + if (node["reshape"]) { + params.reshape = node["reshape"].as>> (); + } + if (node["config"]) { params.config = node["config"].as>(); } diff --git a/src/plugins/intel_npu/tools/protopipe/src/scenario/inference.hpp b/src/plugins/intel_npu/tools/protopipe/src/scenario/inference.hpp index e4568c671438bc..f9c8877b05c53e 100644 --- a/src/plugins/intel_npu/tools/protopipe/src/scenario/inference.hpp +++ b/src/plugins/intel_npu/tools/protopipe/src/scenario/inference.hpp @@ -91,6 +91,7 @@ struct OpenVINOParams { LayerVariantAttr output_layout; LayerVariantAttr input_model_layout; LayerVariantAttr output_model_layout; + LayerVariantAttr> reshape; std::map config; size_t nireq = 1u; }; diff --git a/src/plugins/intel_npu/tools/protopipe/src/simulation/ov_layers_reader.cpp b/src/plugins/intel_npu/tools/protopipe/src/simulation/ov_layers_reader.cpp index 57527cef0cc4aa..33e01e36404570 100644 --- a/src/plugins/intel_npu/tools/protopipe/src/simulation/ov_layers_reader.cpp +++ b/src/plugins/intel_npu/tools/protopipe/src/simulation/ov_layers_reader.cpp @@ -128,6 +128,15 @@ static void cfgOutputPostproc(ov::preprocess::PrePostProcessor& ppp, const std:: } } +static void cfgReshape(const std::shared_ptr& model, + const AttrMap> reshape_map) { + std::map partial_shapes; + for (const auto& [layer_name, shape] : reshape_map) { + partial_shapes.emplace(layer_name, shape); + } + model->reshape(partial_shapes); +} + static std::vector extractLayerNames(const std::vector>& nodes) { std::vector names; std::transform(nodes.begin(), nodes.end(), std::back_inserter(names), [](const auto& node) { @@ -148,6 +157,9 @@ InOutLayers OpenVINOLayersReader::Impl::readFromModel(const std::string& model_p const auto iml_map = unpackLayerAttr(params.input_model_layout, input_names, "input model layout"); cfgInputPreproc(ppp, model, ip_map, il_map, iml_map); + const auto reshape_map = unpackLayerAttr(params.reshape, input_names, "reshape"); + cfgReshape(model, reshape_map); + const auto& output_names = extractLayerNames(model->outputs()); const auto op_map = unpackLayerAttr(params.output_precision, output_names, "output precision"); const auto ol_map = unpackLayerAttr(params.output_layout, output_names, "output layout"); diff --git a/src/plugins/intel_npu/tools/protopipe/src/simulation/simulation.cpp b/src/plugins/intel_npu/tools/protopipe/src/simulation/simulation.cpp index 5b1743651b6ef1..11336c77a477e9 100644 --- a/src/plugins/intel_npu/tools/protopipe/src/simulation/simulation.cpp +++ b/src/plugins/intel_npu/tools/protopipe/src/simulation/simulation.cpp @@ -22,6 +22,11 @@ static cv::gapi::GNetPackage getNetPackage(const std::string& tag, const OpenVIN const auto& blob_path = std::get(params.path); network = std::make_unique

(tag, blob_path.blob, params.device); } + if (std::holds_alternative>>(params.reshape)) { + network->cfgReshape(std::get>>(params.reshape)); + } else { + network->cfgReshape(std::get>(params.reshape)); + } network->cfgPluginConfig(params.config); network->cfgNumRequests(params.nireq); diff --git a/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp b/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp index 999c7f5c6692ac..f9133d613401c4 100644 --- a/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp +++ b/src/plugins/template/tests/functional/op_reference/lstm_sequence.cpp @@ -3382,21 +3382,6 @@ std::vector generateCombinedParams() { return combinedParams; } -std::vector generateV1CombinedParams() { - const std::vector> generatedParams{ - generateV1Params(), - generateV1Params(), - generateV1Params(), - generateV1ParamsBF16(), - }; - std::vector combinedParams; - - for (const auto& params : generatedParams) { - combinedParams.insert(combinedParams.end(), params.begin(), params.end()); - } - return combinedParams; -} - INSTANTIATE_TEST_SUITE_P(smoke_LSTMSequence_With_Hardcoded_Refs, ReferenceLSTMSequenceTest, testing::ValuesIn(generateCombinedParams()), diff --git a/src/tests/functional/shared_test_classes/src/subgraph/perm_conv_perm_concat.cpp b/src/tests/functional/shared_test_classes/src/subgraph/perm_conv_perm_concat.cpp index 69138c3bedf3c5..397f2058034fa0 100644 --- a/src/tests/functional/shared_test_classes/src/subgraph/perm_conv_perm_concat.cpp +++ b/src/tests/functional/shared_test_classes/src/subgraph/perm_conv_perm_concat.cpp @@ -45,7 +45,7 @@ void PermConvPermConcat::SetUp() { this->GetParam(); if (element_type == ov::element::f32) { - abs_threshold = 1e-6; + abs_threshold = 1e-5; } configuration.insert(additional_config.begin(), additional_config.end()); diff --git a/src/tests/ov_helpers/ov_snippets_models/src/subgraph_mha.cpp b/src/tests/ov_helpers/ov_snippets_models/src/subgraph_mha.cpp index 34f42ec838aa6d..5f854326a47217 100644 --- a/src/tests/ov_helpers/ov_snippets_models/src/subgraph_mha.cpp +++ b/src/tests/ov_helpers/ov_snippets_models/src/subgraph_mha.cpp @@ -82,7 +82,8 @@ std::shared_ptr MHAFunction::initOriginal() const { auto softmax_out = add->output(0); if (with_reshape) { const auto interm_shape = add->get_output_shape(0); - const auto batch = std::accumulate(interm_shape.cbegin(), interm_shape.cbegin() + rank - 1, 1, std::multiplies()); + const auto batch = + std::accumulate(interm_shape.cbegin(), interm_shape.cbegin() + (rank - 1), 1, std::multiplies()); const auto reshape0ConstData = std::vector{ batch, -1 }; const auto reshape1ConstData = interm_shape; const auto reshape0Const = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{reshape0ConstData.size()}, reshape0ConstData); @@ -204,7 +205,7 @@ std::shared_ptr MHASplitMFunction::initReference() const { if (ov::shape_size(shape) > 1) { ov::Shape reshape_shape = shape; - reshape_shape.insert(reshape_shape.cbegin() + rank - 3, 1); + reshape_shape.insert(reshape_shape.cbegin() + (rank - 3), 1); const auto mulReshape = make_reshape(mulConst, reshape_shape); const auto mulParam = std::make_shared(precisions[1], mulReshape->get_shape()); matmul_parent1 = std::make_shared(transpose1, mulParam); @@ -288,7 +289,8 @@ std::shared_ptr MHAMatMul0TransposeFunction::initOriginal() const { auto softmax_out = add->output(0); if (with_reshape) { const auto interm_shape = add->get_output_shape(0); - const auto batch = std::accumulate(interm_shape.cbegin(), interm_shape.cbegin() + rank - 1, 1, std::multiplies()); + const auto batch = + std::accumulate(interm_shape.cbegin(), interm_shape.cbegin() + (rank - 1), 1, std::multiplies()); const auto reshape0ConstData = std::vector{ batch, -1 }; const auto reshape1ConstData = interm_shape; const auto reshape0Const = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{reshape0ConstData.size()}, reshape0ConstData); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e4b5fcd5d1089f..08b4308479ef03 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,5 +7,5 @@ add_subdirectory(model_hub_tests) add_subdirectory(samples_tests) add_subdirectory(e2e_tests) -install(FILES requirements_pytorch DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) -install(FILES requirements_tensorflow DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) +install(FILES requirements_pytorch requirements_tensorflow requirements_onnx + DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) diff --git a/tests/e2e_tests/requirements.txt b/tests/e2e_tests/requirements.txt index a4716e496470f4..29e1c1cf31c558 100644 --- a/tests/e2e_tests/requirements.txt +++ b/tests/e2e_tests/requirements.txt @@ -5,7 +5,7 @@ # for common utils py-cpuinfo==9.0.0 -scipy>=1.5.4,<1.12 +scipy>=1.5.4,<1.15 opencv-python>=4.5; sys_platform != "darwin" opencv-python==4.8.1.78; sys_platform == "darwin" unittest-xml-reporting==3.0.4 diff --git a/tests/layer_tests/common/mo_convert_test_class.py b/tests/layer_tests/common/mo_convert_test_class.py index 6a57339cedf111..7eff4f7fee9e8a 100644 --- a/tests/layer_tests/common/mo_convert_test_class.py +++ b/tests/layer_tests/common/mo_convert_test_class.py @@ -63,7 +63,8 @@ def _test(self, temp_dir, test_params, ref_params): core = Core() test_params.update({"model_name": 'model_test', "output_dir": temp_dir}) - ref_params.update({"model_name": 'model_ref', "output_dir": temp_dir}) + ref_output_path = Path(temp_dir, 'model_ref.xml').absolute().as_posix() + ref_params.update({"output_model": ref_output_path}) self.generate_ir_python_api(**test_params) diff --git a/tests/layer_tests/common/utils/common_utils.py b/tests/layer_tests/common/utils/common_utils.py index 0c8ad494c5cec2..620f2fee9de260 100644 --- a/tests/layer_tests/common/utils/common_utils.py +++ b/tests/layer_tests/common/utils/common_utils.py @@ -14,36 +14,24 @@ def generate_ir(coverage=False, **kwargs): - from openvino.tools.mo import mo - mo_path = Path(mo.__file__).parent - mo_runner = mo_path.joinpath('main.py').as_posix() + from openvino.tools.ovc import ovc + # Get OVC file directory + ovc_path = Path(ovc.__file__).parent + + ovc_runner = ovc_path.joinpath('main.py').as_posix() if coverage: - params = [sys.executable, '-m', 'coverage', 'run', '-p', '--source={}'.format(mo_path.parent), - '--omit=*_test.py', mo_runner] + params = [sys.executable, '-m', 'coverage', 'run', '-p', '--source={}'.format(ovc_runner.parent), + '--omit=*_test.py', ovc_runner] else: - params = [sys.executable, mo_runner] + params = [sys.executable, ovc_runner] for key, value in kwargs.items(): - if key == "batch": - params.extend(("-b", str(value))) - elif key == "k": - params.extend(("-k", str(value))) - # for FP32 set explicitly compress_to_fp16=False, - # if we omit this argument for FP32, it will be set implicitly to True as the default + if key == 'input_model': + params.append((str(value))) elif key == 'compress_to_fp16': params.append("--{}={}".format(key, value)) - elif isinstance(value, bool) and value: - params.append("--{}".format(key)) - elif isinstance(value, bool) and not value: - continue - elif (isinstance(value, tuple) and value) or (isinstance(value, str)): - params.extend(("--{}".format(key), str('"{}"'.format(value)))) - elif key == "mean_values" and (' ' in value or '(' in value): - params.extend(("--{}".format(key), str('"{}"'.format(value)))) else: params.extend(("--{}".format(key), str(value))) exit_code, stdout, stderr = shell(params) - logger.info("Model Optimizer out:\n{}".format(stdout)) - logger.error(stderr) return exit_code, stderr diff --git a/tests/layer_tests/ovc_python_api_tests/test_complex_params.py b/tests/layer_tests/ovc_python_api_tests/test_complex_params.py index 3d6df0c95f31ae..57c98db8c45e61 100644 --- a/tests/layer_tests/ovc_python_api_tests/test_complex_params.py +++ b/tests/layer_tests/ovc_python_api_tests/test_complex_params.py @@ -71,77 +71,16 @@ def create_tf_model_single_input_output(tmp_dir): return save_to_pb(tf_net, tmp_dir) test_data = [ - {'params_test': {'output': ["Sigmoid_0:0", "Sigmoid_2:0"]}, - 'params_ref': {'output': "Sigmoid_0,Sigmoid_2"}}, - {'params_test': {'output': ["Sigmoid_0:0"]}, - 'params_ref': {'output': "Sigmoid_0"}}, - {'params_test': {'input': [PartialShape([2, 3, 4]), [2, 3, 4], [Dimension(2), Dimension(3), Dimension(4)]]}, - 'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1:0,Input2:0,Input3:0'}}, - {'params_test': {'input': [PartialShape([1, 3, -1, -1]), [1, 3, -1, -1]]}, - 'params_ref': {'input_shape': "[1,3,?,?],[1,3,?,?]", 'input': 'Input1:0,Input2:0'}}, - {'params_test': {'input': [(2, 3, 4), [2, 3, 4], (Dimension(2), Dimension(3), Dimension(4))]}, - 'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1:0,Input2:0,Input3:0'}}, - {'params_test': {'input': {"Input1:0": PartialShape([2, 3, 4]), "Input2:0": [2, 3, 4], - "Input3:0": [Dimension(2), Dimension(3), Dimension(4)]}}, - 'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1:0,Input2:0,Input3:0'}}, - {'params_test': {'input': {"Input2:0": [1, -1, -1, -1], - "Input3:0": [Dimension(1), Dimension(-1), Dimension(-1), Dimension(-1)]}}, - 'params_ref': {'input_shape': "[1,?,?,?],[1,?,?,?]", 'input': 'Input2:0,Input3:0'}}, - {'params_test': {'input': [np.int32, Type(np.int32), np.int32]}, - 'params_ref': {'input': 'Input1:0{i32},Input2:0{i32},Input3:0{i32}'}}, - {'params_test': {'input': [ov.Type.f32, ov.Type.f32]}, - 'params_ref': {'input': 'Input1:0{f32},Input2:0{f32}'}}, - {'params_test': {'input': [([1, 3, -1, -1], ov.Type.i32), ov.Type.i32, ov.Type.i32]}, - 'params_ref': {'input': 'Input1:0[1,3,?,?]{i32},Input2:0{i32},Input3:0{i32}'}}, - {'params_test': {'input': (PartialShape([2, 3, 4]), [2, 3, 4], [Dimension(2), Dimension(3), Dimension(4)])}, - 'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1:0,Input2:0,Input3:0'}}, - {'params_test': {'input': (PartialShape([1, 3, -1, -1]), [1, 3, -1, -1])}, - 'params_ref': {'input_shape': "[1,3,?,?],[1,3,?,?]", 'input': 'Input1:0,Input2:0'}}, - {'params_test': {'input': ((2, 3, 4), [2, 3, 4], (Dimension(2), Dimension(3), Dimension(4)))}, - 'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1:0,Input2:0,Input3:0'}}, - {'params_test': {'input': (np.int32, Type(np.int32), np.int32)}, - 'params_ref': {'input': 'Input1:0{i32},Input2:0{i32},Input3:0{i32}'}}, - {'params_test': {'input': (ov.Type.f32, ov.Type.f32)}, - 'params_ref': {'input': 'Input1:0{f32},Input2:0{f32}'}}, - {'params_test': {'input': (([1, 3, -1, -1], ov.Type.i32), ov.Type.i32, ov.Type.i32)}, - 'params_ref': {'input': 'Input1:0[1,3,?,?]{i32},Input2:0{i32},Input3:0{i32}'}} - ] - - @pytest.mark.parametrize("params", test_data) - @pytest.mark.nightly - def test_mo_convert_tf_model(self, params, ie_device, precision, ir_version, - temp_dir, use_legacy_frontend): - tf_net_path = self.create_tf_model(temp_dir) - - test_params = params['params_test'] - ref_params = params['params_ref'] - test_params.update({'input_model': tf_net_path}) - ref_params.update({'input_model': tf_net_path}) - self._test(temp_dir, test_params, ref_params) - - test_data = [ - {'params_test': {'input': {"Input:0": ([3, 2], ov.Type.i32)}}, - 'params_ref': {'input': "Input:0[3,2]{i32}"}}, - {'params_test': {'input': {"Input:0": ov.Type.i32}}, - 'params_ref': {'input': "Input:0{i32}"}}, {'params_test': {'input': {"Input:0": [3, 2]}}, 'params_ref': {'input': "Input:0[3,2]"}}, {'params_test': {'input': (3, 2)}, 'params_ref': {'input': "Input:0[3,2]"}}, {'params_test': {'input': (3, Dimension(2))}, 'params_ref': {'input': "Input:0[3,2]"}}, - {'params_test': {'input': [3, 2]}, - 'params_ref': {'input': "Input:0[3 2]"}}, - {'params_test': {'input': [Dimension(3, 10), 2]}, - 'params_ref': {'input': "Input:0[3..10 2]"}}, {'params_test': {'input': (-1, 10)}, 'params_ref': {'input': "Input:0[?,10]"}}, {'params_test': {'input': PartialShape([-1, 10])}, 'params_ref': {'input': "Input:0[?,10]"}}, - {'params_test': {'input': np.int32}, - 'params_ref': {'input': "Input:0{i32}"}}, - {'params_test': {'input': (np.int32, [1, 2, 3])}, - 'params_ref': {'input': "Input:0[1,2,3]{i32}"}}, {'params_test': {'input': [Dimension(3, 10), 10, -1]}, 'params_ref': {'input': 'Input:0[3..10,10,?]'}}, ] diff --git a/tests/layer_tests/ovc_python_api_tests/test_pytorch.py b/tests/layer_tests/ovc_python_api_tests/test_pytorch.py index 1a49a989c11df2..02b4d569927909 100644 --- a/tests/layer_tests/ovc_python_api_tests/test_pytorch.py +++ b/tests/layer_tests/ovc_python_api_tests/test_pytorch.py @@ -366,188 +366,6 @@ def scripted_fn(x: torch.Tensor, y: torch.Tensor): return scripted_fn, ref_model, {'input': [(inp_shape, Type.f32), (inp_shape, Type.f32)]} -def create_pytorch_nn_module_layout_list(tmp_dir): - from openvino.runtime import Layout - pt_model = make_pt_model_two_inputs() - shape = [1, 3, 10, 10] - - shape = PartialShape(shape) - ref_model = make_ref_pt_model_two_inputs(shape) - ref_model.inputs[0].node.layout = Layout('nchw') - ref_model.inputs[1].node.layout = Layout('nhwc') - - return pt_model, ref_model, { - 'input': [(shape, np.float32), (shape, np.float32)], 'layout': ['nchw', Layout('nhwc')], - 'use_convert_model_from_mo': True - } - - -def create_pytorch_nn_module_layout_list_case2(tmp_dir): - from openvino.runtime import Layout - pt_model = make_pt_model_two_inputs() - shape = [1, 3, 10, 10] - - shape = PartialShape(shape) - ref_model = make_ref_pt_model_two_inputs(shape) - ref_model.inputs[0].node.layout = Layout('nchw') - ref_model.inputs[1].node.layout = Layout('nhwc') - - return pt_model, ref_model, { - 'input': [(shape, np.float32), (shape, np.float32)], 'layout': ('nchw', Layout('nhwc')), - 'use_convert_model_from_mo': True} - - -def create_pytorch_nn_module_mean_list_compression_disabled(tmp_dir): - pt_model = make_pt_model_two_inputs() - shape = [1, 10, 10, 3] - - shape = PartialShape(shape) - param1 = ov.opset8.parameter(shape) - param2 = ov.opset8.parameter(shape) - const1 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32) - const2 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32) - add1 = ov.opset8.add(param1, const1) - add2 = ov.opset8.add(param2, const2) - mul = ov.opset8.multiply(add1, add2) - relu = ov.opset8.relu(mul) - sigm = ov.opset8.sigmoid(relu) - - parameter_list = [param1, param2] - ref_model = Model([sigm], parameter_list, "test") - - return pt_model, ref_model, {'input': [(shape, np.float32), (shape, np.float32)], - 'mean_values': [[0, 0, 0], [0, 0, 0]], - 'compress_to_fp16': False, 'use_convert_model_from_mo': True} - - -def create_pytorch_nn_module_mean_list_compression_default(tmp_dir): - # when 'use_convert_model_from_mo': True by default compression in convert_model is disabled - # therefore decompression Converts will not be present - pt_model = make_pt_model_two_inputs() - shape = [1, 10, 10, 3] - - shape = PartialShape(shape) - param1 = ov.opset8.parameter(shape) - param2 = ov.opset8.parameter(shape) - const1 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32) - const2 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32) - add1 = ov.opset8.add(param1, const1) - add2 = ov.opset8.add(param2, const2) - mul = ov.opset8.multiply(add1, add2) - relu = ov.opset8.relu(mul) - sigm = ov.opset8.sigmoid(relu) - - parameter_list = [param1, param2] - ref_model = Model([sigm], parameter_list, "test") - - return pt_model, ref_model, {'input': [(shape, np.float32), (shape, np.float32)], - 'mean_values': [[0, 0, 0], [0, 0, 0]], - 'use_convert_model_from_mo': True} - - -def create_pytorch_nn_module_mean_list_compression_enabled(tmp_dir): - pt_model = make_pt_model_two_inputs() - shape = [1, 10, 10, 3] - - shape = PartialShape(shape) - param1 = ov.opset8.parameter(shape) - param2 = ov.opset8.parameter(shape) - const1 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float16) - const2 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float16) - const1_decompressed = ov.opset8.convert( - const1, destination_type=np.float32) - const2_decompressed = ov.opset8.convert( - const2, destination_type=np.float32) - - add1 = ov.opset8.add(param1, const1_decompressed) - add2 = ov.opset8.add(param2, const2_decompressed) - mul = ov.opset8.multiply(add1, add2) - relu = ov.opset8.relu(mul) - sigm = ov.opset8.sigmoid(relu) - - parameter_list = [param1, param2] - ref_model = Model([sigm], parameter_list, "test") - - return pt_model, ref_model, { - 'input': [(shape, np.float32), (shape, np.float32)], 'mean_values': [[0, 0, 0], [0, 0, 0]], - 'compress_to_fp16': True, 'use_convert_model_from_mo': True} - - -def create_pytorch_nn_module_scale_list_compression_disabled(tmp_dir): - pt_model = make_pt_model_two_inputs() - shape = [1, 10, 10, 3] - - shape = PartialShape(shape) - param1 = ov.opset8.parameter(shape) - param2 = ov.opset8.parameter(shape) - const1 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float32) - const2 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float32) - sub1 = ov.opset8.multiply(param1, const1) - sub2 = ov.opset8.multiply(param2, const2) - mul = ov.opset8.multiply(sub1, sub2) - relu = ov.opset8.relu(mul) - sigm = ov.opset8.sigmoid(relu) - - parameter_list = [param1, param2] - ref_model = Model([sigm], parameter_list, "test") - - return pt_model, ref_model, {'input': [(shape, np.float32), (shape, np.float32)], - 'scale_values': [[1, 1, 1], [1, 1, 1]], - 'compress_to_fp16': False, 'use_convert_model_from_mo': True} - - -def create_pytorch_nn_module_scale_list_compression_default(tmp_dir): - # when 'use_convert_model_from_mo': True by default compression in convert_model is disabled - # therefore decompression Converts will not be present - pt_model = make_pt_model_two_inputs() - shape = [1, 10, 10, 3] - - shape = PartialShape(shape) - param1 = ov.opset8.parameter(shape) - param2 = ov.opset8.parameter(shape) - const1 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float32) - const2 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float32) - sub1 = ov.opset8.multiply(param1, const1) - sub2 = ov.opset8.multiply(param2, const2) - mul = ov.opset8.multiply(sub1, sub2) - relu = ov.opset8.relu(mul) - sigm = ov.opset8.sigmoid(relu) - - parameter_list = [param1, param2] - ref_model = Model([sigm], parameter_list, "test") - - return pt_model, ref_model, {'input': [(shape, np.float32), (shape, np.float32)], - 'scale_values': [[1, 1, 1], [1, 1, 1]], - 'use_convert_model_from_mo': True} - - -def create_pytorch_nn_module_scale_list_compression_enabled(tmp_dir): - pt_model = make_pt_model_two_inputs() - shape = [1, 10, 10, 3] - - shape = PartialShape(shape) - param1 = ov.opset8.parameter(shape) - param2 = ov.opset8.parameter(shape) - const1 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float16) - const1_decompressed = ov.opset8.convert( - const1, destination_type=np.float32) - const2 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float16) - const2_decompressed = ov.opset8.convert( - const2, destination_type=np.float32) - mul1 = ov.opset8.multiply(param1, const1_decompressed) - mul2 = ov.opset8.multiply(param2, const2_decompressed) - mul3 = ov.opset8.multiply(mul1, mul2) - relu = ov.opset8.relu(mul3) - sigm = ov.opset8.sigmoid(relu) - - parameter_list = [param1, param2] - ref_model = Model([sigm], parameter_list, "test") - - return pt_model, ref_model, {'input': [(shape, np.float32), (shape, np.float32)], - 'scale_values': [[1, 1, 1], [1, 1, 1]], - 'compress_to_fp16': True, 'use_convert_model_from_mo': True} - - def create_pytorch_nn_module_with_compressed_constants(tmp_dir): import torch @@ -1208,14 +1026,6 @@ class TestMoConvertPyTorch(CommonMOConvertTest): 'create_pytorch_nn_module_sample_input_int32_two_inputs', 'create_pytorch_jit_script_module', 'create_pytorch_jit_script_function', - 'create_pytorch_nn_module_layout_list', - 'create_pytorch_nn_module_layout_list_case2', - 'create_pytorch_nn_module_mean_list_compression_default', - 'create_pytorch_nn_module_mean_list_compression_disabled', - 'create_pytorch_nn_module_mean_list_compression_enabled', - 'create_pytorch_nn_module_scale_list_compression_default', - 'create_pytorch_nn_module_scale_list_compression_disabled', - 'create_pytorch_nn_module_scale_list_compression_enabled', 'create_pytorch_nn_module_with_compressed_constants', 'create_pytorch_nn_module_shapes_list_static', 'create_pytorch_nn_module_shapes_list_static_via_input', diff --git a/tests/layer_tests/tensorflow2_keras_tests/test_tf2_keras_conv_2d_transpose.py b/tests/layer_tests/tensorflow2_keras_tests/test_tf2_keras_conv_2d_transpose.py index 1226c373b24fe7..d58aa72d8d0d78 100644 --- a/tests/layer_tests/tensorflow2_keras_tests/test_tf2_keras_conv_2d_transpose.py +++ b/tests/layer_tests/tensorflow2_keras_tests/test_tf2_keras_conv_2d_transpose.py @@ -1,69 +1,75 @@ # Copyright (C) 2022-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import numpy as np import pytest import tensorflow as tf from common.tf2_layer_test_class import CommonTF2LayerTest +rng = np.random.default_rng(233534) + class TestKerasConv2DTranspose(CommonTF2LayerTest): - def create_keras_conv_2d_transpose_net(self, conv_params, input_names, input_shapes, input_type, - ir_version): + def _prepare_input(self, inputs_info): + assert 'x' in inputs_info, "Test error: inputs_info must contain `x`" + x_shape = inputs_info['x'] + inputs_data = {} + inputs_data['x'] = rng.uniform(-2.0, 2.0, x_shape).astype(self.input_type) + return inputs_data + + def create_keras_conv_2d_transpose_net(self, input_shapes, input_type, + filters, kernel_size, + strides, padding, data_format, + dilation_rate, activation, + use_bias): + self.input_type = input_type activation_func_structure = { # pytest-xdist can't execute the tests in parallel because workers can't compare tests scopes before run # tf.nn. operation have no "==" operation to be compared - "relu": tf.nn.relu, - "sigmoid": tf.nn.sigmoid + 'relu': tf.nn.relu, + 'sigmoid': tf.nn.sigmoid } - conv_params = conv_params.copy() - if "activation" in conv_params: - conv_params["activation"] = activation_func_structure[conv_params["activation"]] + activation = activation_func_structure[activation] # create TensorFlow 2 model with Keras Conv2DTranspose operation - tf.keras.backend.clear_session() # For easy reset of notebook state - x = tf.keras.Input(shape=input_shapes[0][1:], dtype=input_type, - name=input_names[0]) # Variable-length sequence of ints + tf.keras.backend.clear_session() + x = tf.keras.Input(shape=input_shapes[0][1:], dtype=input_type, name='x') - y = tf.keras.layers.Conv2DTranspose(**conv_params, input_shape=input_shapes[0][1:])(x) + y = tf.keras.layers.Conv2DTranspose(filters=filters, kernel_size=kernel_size, + strides=strides, padding=padding, data_format=data_format, + dilation_rate=dilation_rate, activation=activation, + use_bias=use_bias)(x) tf2_net = tf.keras.Model(inputs=[x], outputs=[y]) - # TODO: add reference IR net. Now it is omitted and tests only inference result that is more important ref_net = None return tf2_net, ref_net - test_data_float32 = [ - dict(conv_params=dict(filters=27, kernel_size=3, padding="valid", strides=(2, 2), - data_format="channels_last"), input_names=["x"], - input_shapes=[[3, 5, 7, 6]], input_type=tf.float32), - dict(conv_params=dict(filters=10, kernel_size=5, padding="same", strides=(7, 7), - activation="relu", use_bias=True, output_padding=(3, 3)), - input_names=["x"], input_shapes=[[3, 5, 7, 8]], input_type=tf.float32), - dict(conv_params=dict(filters=10, kernel_size=5, padding="same", strides=(7, 7), - output_padding=(5, 5)), - input_names=["x"], input_shapes=[[3, 5, 7, 8]], input_type=tf.float32), - dict(conv_params=dict(filters=27, kernel_size=3, padding="valid", dilation_rate=1), - input_names=["x"], - input_shapes=[[3, 9, 7, 6]], input_type=tf.float32), - dict(conv_params=dict(filters=10, kernel_size=5, padding="same", dilation_rate=1), - input_names=["x"], - input_shapes=[[3, 9, 7, 8]], input_type=tf.float32), - dict(conv_params=dict(filters=27, kernel_size=3, padding="valid", dilation_rate=1, - activation="sigmoid", - use_bias=False), input_names=["x"], input_shapes=[[3, 9, 7, 6]], - input_type=tf.float32), - dict(conv_params=dict(filters=10, kernel_size=5, padding="same", dilation_rate=1, - use_bias=True), - input_names=["x"], input_shapes=[[3, 9, 7, 8]], input_type=tf.float32) - ] - - @pytest.mark.parametrize("params", test_data_float32) + @pytest.mark.parametrize('input_shapes', [[[3, 9, 7, 8]]]) + @pytest.mark.parametrize('input_type', [np.float32, np.float64]) + @pytest.mark.parametrize('filters', [2, 5]) + @pytest.mark.parametrize('kernel_size', [3, 5]) + @pytest.mark.parametrize('strides', [(1, 2), (2, 2)]) + @pytest.mark.parametrize('padding', ['valid', 'same']) + @pytest.mark.parametrize('data_format', ['channels_last']) + @pytest.mark.parametrize('dilation_rate', [(1, 1)]) + @pytest.mark.parametrize('activation', ['sigmoid', 'relu']) + @pytest.mark.parametrize('use_bias', [True, False]) @pytest.mark.precommit @pytest.mark.nightly - def test_keras_conv_2d_transpose_float32(self, params, ie_device, precision, ir_version, - temp_dir, use_legacy_frontend): - self._test(*self.create_keras_conv_2d_transpose_net(**params, ir_version=ir_version), + def test_keras_conv_2d_transpose(self, input_shapes, input_type, filters, kernel_size, + strides, padding, data_format, dilation_rate, activation, + use_bias, + ie_device, precision, + ir_version, temp_dir, use_legacy_frontend): + params = {} + params['input_shapes'] = input_shapes + self._test(*self.create_keras_conv_2d_transpose_net(input_shapes, input_type, + filters, kernel_size, + strides, padding, data_format, + dilation_rate, activation, + use_bias), ie_device, precision, temp_dir=temp_dir, ir_version=ir_version, use_legacy_frontend=use_legacy_frontend, **params) diff --git a/tests/model_hub_tests/transformation_tests/generate_ref_diffs.py b/tests/model_hub_tests/transformation_tests/generate_ref_diffs.py index 6823256b3ccfc5..72051783fa7422 100644 --- a/tests/model_hub_tests/transformation_tests/generate_ref_diffs.py +++ b/tests/model_hub_tests/transformation_tests/generate_ref_diffs.py @@ -43,9 +43,36 @@ from openvino._offline_transformations import paged_attention_transformation from openvino._pyopenvino.op import _PagedAttentionExtension, Parameter, Result from optimum.intel import OVModelForCausalLM +from optimum.intel.openvino import OVModelForVisualCausalLM +from typing import Type, Union nodes_to_compare = ("ScaledDotProductAttention", "PagedAttentionExtension", "Parameter", "ReadValue", "Assign") +def get_models_list_type(file_name: str, cls: Union[Type[OVModelForCausalLM], Type[OVModelForVisualCausalLM]]): + models = [] + for line_items in utils.parse_list_file(file_name): + if len(line_items) == 2: + model_name, model_link = line_items + models.append((model_name, model_link, None, None, cls)) + elif len(line_items) == 4: + model_name, model_link, mark, reason = line_items + models.append((model_name, model_link, mark, reason)) + elif len(line_items) > 4: + model_name, model_link, mark, reason, *other = line_items + if not mark: + mark = None + if not reason: + reason = None + other = line_items[4:] + transformations = [item[8:] for item in other if item.startswith('ts_name:')] + layers = [item[6:] for item in other if item.startswith('layer:')] + models.append((model_name, model_link, mark, reason, transformations, layers)) + else: + items = ','.join(line_items) + assert False, \ + f'Incorrect model info fields {items}. It must contain either 2 or 4 or more than 4 fields.' + return models + def main(): use_cache_eviction = False if len(sys.argv) >= 2: @@ -55,32 +82,37 @@ def main(): if OUTPUT_FILE.exists() and OUTPUT_FILE.is_file(): OUTPUT_FILE.unlink() - + with open(OUTPUT_FILE, 'w') as file: - model_list = utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-models-precommit")) + model_list = get_models_list_type(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-models-precommit"), OVModelForCausalLM) + model_list.extend(get_models_list_type(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-vl-models-precommit"), OVModelForVisualCausalLM)) print(OUTPUT_FILE) print('ref_diff_map_cache_eviction = {' if use_cache_eviction else 'ref_diff_map = {', file=file) - for model_id, _, _, _ in model_list: + for model_id, _, _, _, cls in model_list: # wrapping in try/catch block to continue printing models even if one has failed try: - model = OVModelForCausalLM.from_pretrained(model_id, export=True, trust_remote_code=True) + model = cls.from_pretrained(model_id, export=True, trust_remote_code=True) except: + print(f"Couldn't read {model_id}.") continue + ov_model = model.model if cls is OVModelForCausalLM else model.lm_model + before_map = {} - for op in model.model.get_ordered_ops(): + for op in ov_model.get_ordered_ops(): if op.get_type_name() in nodes_to_compare: before_map[op.get_type_name()] = before_map.get(op.get_type_name(), 0) + 1 # wrapping in try/catch block to continue printing models even if one has failed try: - paged_attention_transformation(model.model, use_cache_eviction, use_cache_eviction) + paged_attention_transformation(ov_model, use_cache_eviction, use_cache_eviction) except: + print(f"Couldn't run SDPAToPA transformation on {model_id} and generate diffs.") continue after_map = {} - for op in model.model.get_ordered_ops(): + for op in ov_model.get_ordered_ops(): if op.get_type_name() in nodes_to_compare: after_map[op.get_type_name()] = after_map.get(op.get_type_name(), 0) + 1 diff --git a/tests/model_hub_tests/transformation_tests/models/hf-tiny-random-vl-models-precommit b/tests/model_hub_tests/transformation_tests/models/hf-tiny-random-vl-models-precommit new file mode 100644 index 00000000000000..327ba3f7e8e966 --- /dev/null +++ b/tests/model_hub_tests/transformation_tests/models/hf-tiny-random-vl-models-precommit @@ -0,0 +1,4 @@ +katuni4ka/tiny-random-llava-next,https://huggingface.co/katuni4ka/tiny-random-llava-next +katuni4ka/tiny-random-minicpmv-2_6,https://huggingface.co/katuni4ka/tiny-random-minicpmv-2_6 +katuni4ka/tiny-random-llava,https://huggingface.co/katuni4ka/tiny-random-llava +katuni4ka/tiny-random-nanollava,https://huggingface.co/katuni4ka/tiny-random-nanollava,xfail,CVS-157416 \ No newline at end of file diff --git a/tests/model_hub_tests/transformation_tests/sdpa2pa_ref_diff.py b/tests/model_hub_tests/transformation_tests/sdpa2pa_ref_diff.py index 23af913d9d102f..43ef49d9b5a226 100644 --- a/tests/model_hub_tests/transformation_tests/sdpa2pa_ref_diff.py +++ b/tests/model_hub_tests/transformation_tests/sdpa2pa_ref_diff.py @@ -305,6 +305,35 @@ "ReadValue" : -12, "Assign" : -12, }, + "katuni4ka/tiny-random-llava-next" : { + "PagedAttentionExtension" : 2, + "Parameter" : 7, + "ReadValue" : -4, + "ScaledDotProductAttention" : -2, + "Assign" : -4, + }, + "katuni4ka/tiny-random-minicpmv-2_6" : { + "PagedAttentionExtension" : 2, + "Parameter" : 7, + "ReadValue" : -4, + "ScaledDotProductAttention" : -2, + "Assign" : -4, + }, + "katuni4ka/tiny-random-llava" : { + "Assign" : -4, + "Parameter" : 7, + "ReadValue" : -4, + "ScaledDotProductAttention" : -2, + "PagedAttentionExtension" : 2, + }, + + # "katuni4ka/tiny-random-nanollava" : { + # "Assign" : -4, + # "Parameter" : 7, + # "ReadValue" : -4, + # "ScaledDotProductAttention" : -2, + # "PagedAttentionExtension" : 2, + # }, } ref_diff_map_cache_eviction = { @@ -532,13 +561,13 @@ "Parameter" : 14, "Assign" : -8, }, - "katuni4ka/tiny-random-minicpm" : { - "ScaledDotProductAttention" : -4, - "Parameter" : 14, - "PagedAttentionExtension" : 4, - "ReadValue" : -8, - "Assign" : -8, - }, + "katuni4ka/tiny-random-minicpm" : { + "ScaledDotProductAttention" : -4, + "Parameter" : 14, + "PagedAttentionExtension" : 4, + "ReadValue" : -8, + "Assign" : -8, + }, "katuni4ka/tiny-random-falcon-40b" : { "ScaledDotProductAttention" : -2, "ReadValue" : -4, @@ -609,4 +638,33 @@ "Parameter" : 20, "Assign" : -12, }, + "katuni4ka/tiny-random-llava-next" : { + "Parameter" : 8, + "Assign" : -4, + "ReadValue" : -4, + "PagedAttentionExtension" : 2, + "ScaledDotProductAttention" : -2, + }, + "katuni4ka/tiny-random-minicpmv-2_6" : { + "Parameter" : 8, + "Assign" : -4, + "ReadValue" : -4, + "PagedAttentionExtension" : 2, + "ScaledDotProductAttention" : -2, + }, + "katuni4ka/tiny-random-llava" : { + "ReadValue" : -4, + "Parameter" : 8, + "ScaledDotProductAttention" : -2, + "PagedAttentionExtension" : 2, + "Assign" : -4, + }, + + # "katuni4ka/tiny-random-nanollava" : { + # "ReadValue" : -4, + # "Parameter" : 8, + # "ScaledDotProductAttention" : -2, + # "PagedAttentionExtension" : 2, + # "Assign" : -4, + # }, } diff --git a/tests/model_hub_tests/transformation_tests/test_pa_transformation.py b/tests/model_hub_tests/transformation_tests/test_pa_transformation.py index 02481439818f28..2bc6726dff030f 100644 --- a/tests/model_hub_tests/transformation_tests/test_pa_transformation.py +++ b/tests/model_hub_tests/transformation_tests/test_pa_transformation.py @@ -4,6 +4,9 @@ from openvino._offline_transformations import paged_attention_transformation from openvino._pyopenvino.op import _PagedAttentionExtension from optimum.intel import OVModelForCausalLM +from optimum.intel.openvino import OVModelForVisualCausalLM +from typing import Type, Union +import openvino as ov from models_hub_common.utils import retry import models_hub_common.utils as utils from sdpa2pa_ref_diff import ref_diff_map, ref_diff_map_cache_eviction, nodes_to_compare @@ -11,19 +14,19 @@ import os import re -@retry(3, exceptions=(OSError,), delay=1) -def run_pa(tmp_path, model_id, model_link, use_block_indices_inputs, use_score_outputs): - model = OVModelForCausalLM.from_pretrained(model_id, export=True, trust_remote_code=True) - +def compare_diffs(ov_model: ov.Model, + model_id: str, + use_block_indices_inputs: bool, + use_score_outputs: bool): before_map = {} - for op in model.model.get_ordered_ops(): + for op in ov_model.get_ordered_ops(): if op.get_type_name() in nodes_to_compare: before_map[op.get_type_name()] = before_map.get(op.get_type_name(), 0) + 1 - paged_attention_transformation(model.model, use_block_indices_inputs, use_score_outputs) + paged_attention_transformation(ov_model, use_block_indices_inputs, use_score_outputs) after_map = {} - for op in model.model.get_ordered_ops(): + for op in ov_model.get_ordered_ops(): if op.get_type_name() in nodes_to_compare: after_map[op.get_type_name()] = after_map.get(op.get_type_name(), 0) + 1 @@ -38,7 +41,7 @@ def run_pa(tmp_path, model_id, model_link, use_block_indices_inputs, use_score_o assert reference_map == resulting_map - model_inputs = model.model.inputs + model_inputs = ov_model.inputs for input in model_inputs: names = list(input.get_names()) # names stored in as set (in this case usually of 1 element) for name in names: @@ -53,7 +56,7 @@ def run_pa(tmp_path, model_id, model_link, use_block_indices_inputs, use_score_o block_indices_pattern = r'block_indices\.[0-9]+' block_indices_counter = 0 - model_inputs = model.model.inputs + model_inputs = ov_model.inputs for input in model_inputs: for name in list(input.get_names()): if re.search(block_indices_pattern, name): @@ -66,7 +69,7 @@ def run_pa(tmp_path, model_id, model_link, use_block_indices_inputs, use_score_o score_pattern = r'scores\.[0-9]+' score_outputs_counter = 0 - model_outputs = model.model.outputs + model_outputs = ov_model.outputs for output in model_outputs: for name in list(output.get_names()): if re.search(score_pattern, name): @@ -75,6 +78,18 @@ def run_pa(tmp_path, model_id, model_link, use_block_indices_inputs, use_score_o assert block_indices_counter == resulting_map["PagedAttentionExtension"], \ f"The number of scores outputs doesn't correspond to the expected value. Expected {resulting_map['PagedAttentionExtension']}, received {block_indices_counter}" +@retry(3, exceptions=(OSError,), delay=1) +def run_pa(tmp_path, + model_id, + model_link, + cls: Union[Type[OVModelForCausalLM], Type[OVModelForVisualCausalLM]], + use_block_indices_inputs, + use_score_outputs): + model = cls.from_pretrained(model_id, export=True, trust_remote_code=True) + ov_model = model.model if cls is OVModelForCausalLM else model.lm_model + + compare_diffs(ov_model, model_id, use_block_indices_inputs, use_score_outputs) + @pytest.mark.precommit @pytest.mark.parametrize("model_name, model_link, mark, reason", utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-models-precommit"))) def test_pa_precommit(tmp_path, model_name, model_link, mark, reason, ie_device): @@ -84,7 +99,7 @@ def test_pa_precommit(tmp_path, model_name, model_link, mark, reason, ie_device) pytest.skip(reason) elif mark == 'xfail': pytest.xfail(reason) - run_pa(tmp_path, model_name, model_link, False, False) + run_pa(tmp_path, model_name, model_link, OVModelForCausalLM, False, False) @pytest.mark.precommit @pytest.mark.parametrize("model_name, model_link, mark, reason", utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-models-precommit"))) @@ -95,4 +110,26 @@ def test_pa_precommit_use_cache_eviction(tmp_path, model_name, model_link, mark, pytest.skip(reason) elif mark == 'xfail': pytest.xfail(reason) - run_pa(tmp_path, model_name, model_link, True, True) \ No newline at end of file + run_pa(tmp_path, model_name, model_link, OVModelForCausalLM, True, True) + +@pytest.mark.precommit +@pytest.mark.parametrize("model_name, model_link, mark, reason", utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-vl-models-precommit"))) +def test_pa_vlm(tmp_path, model_name, model_link, mark, reason, ie_device): + assert mark is None or mark == 'skip' or mark == 'xfail', \ + "Incorrect test case: {}, {}".format(model_name, model_link) + if mark == 'skip': + pytest.skip(reason) + elif mark == 'xfail': + pytest.xfail(reason) + run_pa(tmp_path, model_name, model_link, OVModelForVisualCausalLM, False, False) + +@pytest.mark.precommit +@pytest.mark.parametrize("model_name, model_link, mark, reason", utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "hf-tiny-random-vl-models-precommit"))) +def test_pa_vlm_use_cache_eviction(tmp_path, model_name, model_link, mark, reason, ie_device): + assert mark is None or mark == 'skip' or mark == 'xfail', \ + "Incorrect test case: {}, {}".format(model_name, model_link) + if mark == 'skip': + pytest.skip(reason) + elif mark == 'xfail': + pytest.xfail(reason) + run_pa(tmp_path, model_name, model_link, OVModelForVisualCausalLM, True, True) \ No newline at end of file diff --git a/tests/requirements_onnx b/tests/requirements_onnx new file mode 100644 index 00000000000000..1dfc0077b5d075 --- /dev/null +++ b/tests/requirements_onnx @@ -0,0 +1,3 @@ +numpy>=1.16.6,<1.27 +onnx>=1.8.1,<=1.17.0 +protobuf>=3.18.1,<4.0.0 diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index c22b06bcf5863c..83f7e209bcf95b 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -18,7 +18,7 @@ find_package(PkgConfig QUIET) # cmake older than 3.18 cannot create an alias for imported non-GLOBAL targets # so, we have to use 'IMPORTED_GLOBAL' property if(CMAKE_VERSION VERSION_LESS 3.18) - set(OV_PkgConfig_VISILITY GLOBAL) + set(OV_PkgConfig_VISIBILITY GLOBAL) endif() if(SUGGEST_OVERRIDE_SUPPORTED) @@ -69,9 +69,21 @@ endif() # if(ENABLE_INTEL_NPU) - add_subdirectory(thirdparty/level_zero EXCLUDE_FROM_ALL) + if(ENABLE_SYSTEM_LEVEL_ZERO) + pkg_search_module(level_zero QUIET + IMPORTED_TARGET + ${OV_PkgConfig_VISIBILITY} + level-zero) + if(level_zero_FOUND) + add_library(LevelZero::LevelZero ALIAS PkgConfig::level_zero) + message(STATUS "${PKG_CONFIG_EXECUTABLE}: level_zero (${level_zero_VERSION}) is found at ${level_zero_PREFIX}") + endif() + endif() - add_library(LevelZero::LevelZero ALIAS ze_loader) + if(NOT libze_loader_FOUND) + add_subdirectory(thirdparty/level_zero EXCLUDE_FROM_ALL) + add_library(LevelZero::LevelZero ALIAS ze_loader) + endif() endif() # @@ -190,7 +202,7 @@ if(ENABLE_SYSTEM_PUGIXML) # Ubuntu 18.04 case when cmake interface is not available pkg_search_module(pugixml QUIET IMPORTED_TARGET - ${OV_PkgConfig_VISILITY} + ${OV_PkgConfig_VISIBILITY} pugixml) if(pugixml_FOUND) set(pugixml_target PkgConfig::pugixml) @@ -249,7 +261,7 @@ if(ENABLE_SYSTEM_PUGIXML) message(FATAL_ERROR "Debian | RPM package build requires shared Pugixml library") endif() - if(OV_PkgConfig_VISILITY) + if(OV_PkgConfig_VISIBILITY) # need to set GLOBAL visibility in order to create ALIAS for this target set_target_properties(${pugixml_target} PROPERTIES IMPORTED_GLOBAL ON) endif() @@ -299,7 +311,7 @@ if(ENABLE_TESTS) if(GTest_FOUND) foreach(gtest_target gtest gtest_main gmock gmock_main) - if(OV_PkgConfig_VISILITY) + if(OV_PkgConfig_VISIBILITY) # need to set GLOBAL visibility in order to create ALIAS for this target set_target_properties(GTest::${gtest_target} PROPERTIES IMPORTED_GLOBAL ON) endif() @@ -448,7 +460,7 @@ if(ENABLE_SNAPPY_COMPRESSION) set(ov_snappy_lib Snappy::snappy-static) endif() - if(OV_PkgConfig_VISILITY) + if(OV_PkgConfig_VISIBILITY) # need to set GLOBAL visibility in order to create ALIAS for this target set_target_properties(${ov_snappy_lib} PROPERTIES IMPORTED_GLOBAL ON) endif() diff --git a/thirdparty/level_zero/level-zero b/thirdparty/level_zero/level-zero index fd459f28041c17..91e28669b464c3 160000 --- a/thirdparty/level_zero/level-zero +++ b/thirdparty/level_zero/level-zero @@ -1 +1 @@ -Subproject commit fd459f28041c176df23a0b7b791ff20a7689d237 +Subproject commit 91e28669b464c32eced6b0afc84bd08ce77d17c6