diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e7eb099540439..3598e32166a809 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -108,8 +108,6 @@ /tools/ @openvinotoolkit/openvino-tools-maintainers /tools/benchmark_tool/ @openvinotoolkit/openvino-ie-python-api-maintainers /tools/legacy/ @openvinotoolkit/openvino-samples-maintainers -/tools/openvino_dev/ @openvinotoolkit/openvino-tools-maintainers @openvinotoolkit/openvino-ie-python-api-maintainers -/tools/mo/ @openvinotoolkit/openvino-mo-maintainers /tools/ovc/ @openvinotoolkit/openvino-ovc-maintainers /thirdparty/open_model_zoo/ @openvinotoolkit/omz-maintainers diff --git a/.github/actions/common/constants.py b/.github/actions/common/constants.py index da55ece2ee4258..6a1d165fc7df13 100644 --- a/.github/actions/common/constants.py +++ b/.github/actions/common/constants.py @@ -16,6 +16,7 @@ class EventType(Enum): 'public_linux_ubuntu_24_04_x86_64_release', 'public_windows_vs2019_Release', 'public_windows_vs2019_Debug', + 'public_manylinux2014_x86_64_release', ) ProductType = Enum('ProductType', {t.upper(): t for t in productTypes}) diff --git a/.github/actions/setup_python/action.yml b/.github/actions/setup_python/action.yml index 80e22866170535..96968f55636df9 100644 --- a/.github/actions/setup_python/action.yml +++ b/.github/actions/setup_python/action.yml @@ -62,7 +62,7 @@ runs: env: PIP_CACHE_DIR: ${{ inputs.self-hosted-runner == 'true' && inputs.pip-cache-path || '' }} - - if: ${{ inputs.should-setup-pip-paths == 'true' }} + - if: ${{ inputs.should-setup-pip-paths == 'true' && runner.os != 'Windows' }} name: Setup pip variables (cache and install path) shell: bash run: | @@ -71,6 +71,14 @@ runs: echo "PIP_CACHE_DIR=${{ inputs.pip-cache-path }}/${PIP_VER}" >> $GITHUB_ENV echo "PIP_INSTALL_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" >> $GITHUB_ENV + - if: ${{ inputs.should-setup-pip-paths == 'true' && runner.os == 'Windows' }} + name: Setup pip cache path + shell: pwsh + run: | + $pipVersion = python3 -c "import pip; print(pip.__version__)" + Write-Host "Using pip version: $pipVersion" + "PIP_CACHE_DIR=${{ inputs.pip-cache-path }}/$pipVersion" >> $env:GITHUB_ENV + - if: ${{ inputs.show-cache-info == 'true' }} name: Get pip cache info shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1511e6a2c30170..359ff683c9b22a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -41,7 +41,7 @@ updates: - "rkazants" versioning-strategy: increase-if-necessary - # Model Optimizer, openvino_dev and Benchmark tool + # ovc and Benchmark tools - package-ecosystem: pip directory: "/tools" schedule: diff --git a/.github/dependency_review.yml b/.github/dependency_review.yml index 5636a441501fc8..3240502f4e94b5 100644 --- a/.github/dependency_review.yml +++ b/.github/dependency_review.yml @@ -2,13 +2,10 @@ fail-on-severity: 'low' allow-licenses: - 'BSD-2-Clause' - 'BSD-3-Clause' - - 'BSD-2-Clause AND BSD-3-Clause' - - 'BSD-3-Clause AND BSD-3-Clause-Clear' + - 'BSD-3-Clause-Clear' - 'MIT' - 'Apache-2.0' - - 'Apache-2.0 AND BSD-3-Clause' - 'ISC' - - 'Apache-2.0 AND MIT' - 'BlueOak-1.0.0' - '0BSD' - 'Python-2.0' diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 5a4f7795ea4a44..3783a7e8d5600a 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-27384 +pr-27430 diff --git a/.github/dockerfiles/ov_build/manylinux2014_x86_64/Dockerfile b/.github/dockerfiles/ov_build/manylinux2014_x86_64/Dockerfile new file mode 100644 index 00000000000000..59239575be329c --- /dev/null +++ b/.github/dockerfiles/ov_build/manylinux2014_x86_64/Dockerfile @@ -0,0 +1,20 @@ +ARG REGISTRY="quay.io" +FROM openvinogithubactions.azurecr.io/quayio/pypa/manylinux2014_x86_64 + +USER root + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && /install_build_dependencies.sh + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +ENV PATH="$SCCACHE_HOME:$PATH" diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_x64_docker/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_docker/Dockerfile new file mode 100644 index 00000000000000..2d5bc1c878069a --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_x64_docker/Dockerfile @@ -0,0 +1,42 @@ +ARG REGISTRY="docker.io" +FROM ${REGISTRY}/library/ubuntu:22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + git \ + gpg-agent \ + tzdata \ + # parallel gzip + pigz \ + python3 \ + python3-pip \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install docker +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ + gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] \ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + +RUN apt-get update && \ + apt-get install -y docker-ce docker-ce-cli containerd.io + +ENV DOCKER_BUILDKIT=1 \ No newline at end of file diff --git a/.github/labeler.yml b/.github/labeler.yml index daa5375b175bd3..e9b2acb26c9072 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -100,10 +100,6 @@ 'category: LP transformations': - 'src/common/low_precision_transformations/**/*' -'category: MO': -- 'tools/mo/**/*' -- 'tests/layer_tests/mo_python_api_tests/**/*' - 'category: OVC': - 'tools/ovc/**/*' - 'tests/layer_tests/ovc_python_api_tests/**/*' @@ -119,7 +115,6 @@ - any: ['src/bindings/js/node/CMakeLists.txt', 'src/bindings/js/node/package.json', 'src/bindings/js/node/package-lock.json'] -- 'tools/openvino_dev/**/*' 'category: PDPD FE': - 'src/frontends/paddle/**/*' @@ -183,7 +178,6 @@ 'category: tools': - any: ['tools/**', - '!tools/mo/**/*', '!tools/ovc/**/*'] 'category: transformations': diff --git a/.github/workflows/android_arm64.yml b/.github/workflows/android_arm64.yml index fca16f2848f7bb..e0954871f4b51e 100644 --- a/.github/workflows/android_arm64.yml +++ b/.github/workflows/android_arm64.yml @@ -169,7 +169,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake - build - run: cmake --build ${BUILD_DIR} --parallel + run: cmake --build ${BUILD_DIR} --parallel $(nproc) - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/android_x64.yml b/.github/workflows/android_x64.yml index 1cdb2023784979..b0b46c662abdbb 100644 --- a/.github/workflows/android_x64.yml +++ b/.github/workflows/android_x64.yml @@ -144,7 +144,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake - build - run: cmake --build ${BUILD_DIR} --parallel + run: cmake --build ${BUILD_DIR} --parallel $(nproc) - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index db5ba3de1a3c85..cde1b9cf67e2fc 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -50,12 +50,6 @@ jobs: python3 -m pip install -r ${{ github.workspace }}/src/frontends/onnx/tests/requirements.txt # For running TensorFlow frontend unit tests python3 -m pip install -r ${{ github.workspace }}/src/frontends/tensorflow/tests/requirements.txt - # For MO unit tests - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_caffe.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_kaldi.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_onnx.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_tf2.txt - python3 -m pip install -r ${{ github.workspace }}/tools/mo/requirements_dev.txt - name: Build OpenVINO with CMake uses: ashutoshvarma/action-cmake-build@ade188313bc7eaa6f14349569a64d8bc716342ff # master @@ -84,9 +78,6 @@ jobs: - name: Install wheel packages run: cmake -DCOMPONENT=python_wheels -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install_pkg -P '${{ github.workspace }}/build/cmake_install.cmake' - - name: Install python wheels - run: python3 -m pip install openvino-dev --find-links=${{ github.workspace }}/install_pkg/tools - - name: List binaries run: ls -la ${{ github.workspace }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }} @@ -138,6 +129,6 @@ jobs: lcov --capture --directory ${{ github.workspace }}/. --output-file coverage.info genhtml coverage.info --output-directory coverage-report - name: Collect coverage - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: verbose: true diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 9797414cde56c8..5a08ec084dadac 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -85,7 +85,6 @@ jobs: image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} volumes: - /mount:/mount - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' @@ -95,14 +94,6 @@ jobs: OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib BUILD_DIR: /__w/openvino/openvino/openvino_build COVERITY_TOOL_DIR: /__w/openvino/openvino/coverity_tool - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt - SCCACHE_LOG: warn - SCCACHE_AZURE_KEY_PREFIX: coverity_lin - SCCACHE_CACHE_SIZE: 50G steps: - name: Clone OpenVINO @@ -141,8 +132,6 @@ jobs: # # Build # - - name: Clean sccache stats - run: ${SCCACHE_PATH} --zero-stats - name: CMake configure - OpenVINO run: | @@ -164,10 +153,7 @@ jobs: popd - name: Cmake build - OpenVINO with Coverity - run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats + run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Pack Artefacts run: tar -cvf - cov-int | pigz > openvino.tgz diff --git a/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml b/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml index 2f6d646f8e271d..ba458da5d3ec1a 100644 --- a/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml +++ b/.github/workflows/dev_cpu_linux_snippets_libxsmm.yml @@ -143,7 +143,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/fedora_29.yml b/.github/workflows/fedora_29.yml index 3f685502747a19..f3b101327f76dc 100644 --- a/.github/workflows/fedora_29.yml +++ b/.github/workflows/fedora_29.yml @@ -5,7 +5,7 @@ on: merge_group: push: branches: - # - master + - master - 'releases/**' concurrency: diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index e491388a4b3606..3964f049be2abb 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -168,7 +168,7 @@ jobs: run: ${SCCACHE_PATH} --zero-stats - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats @@ -210,7 +210,7 @@ jobs: -DPython3_EXECUTABLE=$python_exec \ -DCPACK_GENERATOR=DEB \ ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --target package - name: Cmake & Build - OpenVINO Contrib if: ${{ inputs.build-contrib }} @@ -221,7 +221,7 @@ jobs: -DENABLE_WHEEL=OFF \ -S ${OPENVINO_REPO} \ -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: CMake configure, build and install - OpenVINO JS API if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }} @@ -230,7 +230,7 @@ jobs: -DCPACK_GENERATOR=NPM \ -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF - cmake --build ${BUILD_DIR} --parallel + cmake --build ${BUILD_DIR} --parallel $(nproc) cmake --install ${BUILD_DIR} --prefix ${INSTALL_DIR_JS} - name: Build RPM packages @@ -243,7 +243,7 @@ jobs: -DENABLE_WHEEL=OFF \ -DENABLE_TESTS=OFF \ ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --target package --verbose + cmake --build ${BUILD_DIR} --parallel $(nproc) --target package --verbose # # Upload build artifacts and logs diff --git a/.github/workflows/job_build_windows.yml b/.github/workflows/job_build_windows.yml index 0567109e0dea46..8a39f348d824c3 100644 --- a/.github/workflows/job_build_windows.yml +++ b/.github/workflows/job_build_windows.yml @@ -25,7 +25,7 @@ on: permissions: read-all env: - PIP_CACHE_PATH: /mount/caches/pip/win + PIP_CACHE_PATH: "C:\\mount\\caches\\pip\\win" PYTHON_VERSION: '3.11' jobs: @@ -164,7 +164,7 @@ jobs: run: '& ccache --zero-stats' - name: Cmake build - OpenVINO - run: cmake --build ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --parallel --verbose + run: cmake --build ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --parallel $ENV:NUMBER_OF_PROCESSORS --verbose - name: Show ccache stats run: '& ccache --show-stats' @@ -201,7 +201,7 @@ jobs: -DCPACK_GENERATOR=NPM ` -DENABLE_SYSTEM_TBB=OFF ` -DENABLE_WHEEL=OFF - cmake --build ${{ env.BUILD_DIR }} --parallel + cmake --build ${{ env.BUILD_DIR }} --parallel $ENV:NUMBER_OF_PROCESSORS cmake --install ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_DIR_JS }} # diff --git a/.github/workflows/job_cxx_unit_tests.yml b/.github/workflows/job_cxx_unit_tests.yml index 8fab17043b7465..52a2b3f4d287c8 100644 --- a/.github/workflows/job_cxx_unit_tests.yml +++ b/.github/workflows/job_cxx_unit_tests.yml @@ -29,7 +29,7 @@ on: description: 'Timeout in minutes for the job' type: number required: false - default: 35 + default: 45 permissions: read-all diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index d7d7bf089b64ec..1aa13efea090ec 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -50,14 +50,14 @@ jobs: # Install debian packages from previous release apt-get install --no-install-recommends -y gnupg wget ca-certificates lsb-release wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - + gpg --output /etc/apt/trusted.gpg.d/intel.gpg --dearmor GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + # Yields only the number, e.g., "20" ubuntu_version=$(lsb_release -r -s) ubuntu_version=${ubuntu_version%.*} - + echo "deb https://apt.repos.intel.com/openvino/2024 ubuntu$ubuntu_version main" | tee /etc/apt/sources.list.d/intel-openvino-2024.list - + apt-get update -y apt-get install -y openvino fi diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml index c879f0cb6a1efc..39a4b467e74fc1 100644 --- a/.github/workflows/job_onnx_models_tests.yml +++ b/.github/workflows/job_onnx_models_tests.yml @@ -86,14 +86,11 @@ jobs: run: | # Install the core OV wheel python3 -m pip install ./openvino-*.whl - - extras_to_install="onnx" - - # Find and install OV dev wheel - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[$extras_to_install] working-directory: ${{ env.INSTALL_WHEELS_DIR }} + - name: Install ONNX Models tests requirements + run: python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_onnx + - name: Install Python tests dependencies run: | # To enable pytest parallel features diff --git a/.github/workflows/job_onnx_runtime.yml b/.github/workflows/job_onnx_runtime.yml index 966d258a2fc609..df50c4f3e2ad3c 100644 --- a/.github/workflows/job_onnx_runtime.yml +++ b/.github/workflows/job_onnx_runtime.yml @@ -96,7 +96,7 @@ jobs: --config RelWithDebInfo \ --use_openvino CPU \ --build_shared_lib \ - --parallel \ + --parallel $(nproc) \ --skip_tests \ --compile_no_warning_as_error \ --allow_running_as_root \ diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index 64be9ef4bbcc44..8075f3299fe063 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -91,20 +91,11 @@ jobs: should-setup-pip-paths: ${{ runner.os == 'Linux' }} self-hosted-runner: ${{ runner.os == 'Linux' }} - # - # Tests - # - - name: Install OpenVINO Python wheels run: | # Install the core OV wheel python3 -m pip install ./openvino-*.whl - extras_to_install="caffe,kaldi,onnx,tensorflow2,pytorch" - - # Find and install OV dev wheel - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[$extras_to_install] working-directory: ${{ env.INSTALL_WHEELS_DIR }} - name: Install Python API tests dependencies @@ -112,7 +103,19 @@ jobs: # To enable pytest parallel features python3 -m pip install pytest-xdist[psutil] python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt - python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt + + - name: Install Python Layer tests dependencies and for OVC unit tests + run: | + # For torchvision to OpenVINO preprocessing converter + python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt + + # layer test requirements + python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt + + - name: Install ONNX tests dependencies + run: | + # ONNX tests requirements + python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_onnx # # Tests @@ -127,18 +130,6 @@ jobs: --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ --ignore=${INSTALL_TEST_DIR}/pyopenvino/tests/test_utils/test_utils.py - - name: Model Optimizer unit tests - if: fromJSON(inputs.affected-components).MO.test - run: | - if [[ "${{ runner.os }}" == "Linux" ]] && [[ "${{ runner.arch }}" != "ARM64" ]]; then - # required for MxNet - apt-get install -y libgomp1 libquadmath0 - fi - - # Skips under tickets: 133405, 122666 - python3 -m pytest -s ${INSTALL_TEST_DIR}/mo/unit_tests \ - --junitxml=${INSTALL_TEST_DIR}/TEST-ModelOptimizer.xml - - name: Python ONNX operators tests if: (fromJSON(inputs.affected-components).Python_API.test || fromJSON(inputs.affected-components).ONNX_FE.test) && @@ -153,32 +144,6 @@ jobs: if: fromJSON(inputs.affected-components).MO.test run: python3 -m pytest -s ${INSTALL_TEST_DIR}/ovc/unit_tests --junitxml=${INSTALL_TEST_DIR}/TEST-OpenVinoConversion.xml - - name: Install Python Layer tests dependencies - run: | - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt - - # layer test requirements - python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt - - - name: MO Python API Tests - if: fromJSON(inputs.affected-components).MO.test - run: | - # Import 'test_utils' installed in '/tests/python/openvino' - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - export PYTHONPATH=${INSTALL_TEST_DIR}/python - - if [[ "${{ runner.os }}" == "Linux" ]] && [[ "${{ runner.arch }}" == "ARM64" ]]; then - # Find gomp lib - GOMP_LIB=$(find "${PIP_INSTALL_PATH}/torch/lib/../../torch.libs/" -name '*libgomp-*so*') - export LD_PRELOAD=${GOMP_LIB} - fi - - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests -n logical --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: OVC Python API Tests if: fromJSON(inputs.affected-components).MO.test run: | @@ -205,16 +170,6 @@ jobs: export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/py_frontend_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_py_fontend.xml - - name: ONNX Layer Tests - if: ${{ fromJSON(inputs.affected-components).ONNX_FE.test }} - run: | - # requires 'unit_tests' from 'tools/mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo:$PYTHONPATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/onnx_tests -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: JAX Layer Tests - JAX FE if: ${{ fromJSON(inputs.affected-components).JAX_FE.test && runner.arch != 'ARM64' && runner.os != 'macOS' }} run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit_jax_fe --junitxml=${INSTALL_TEST_DIR}/TEST-jax_fe.xml @@ -230,22 +185,6 @@ jobs: TEST_DEVICE: CPU TEST_PRECISION: FP16 - - name: TensorFlow 1 Layer Tests - Legacy FE - if: fromJSON(inputs.affected-components).TF_FE.test - run: python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --use_legacy_frontend --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 2 Layer Tests - Legacy FE - # no longer workable since TF 2.17 - # will be removed in 2024.5 - if: ${{ 'false' }} - run: python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/test_tf2_keras_activation.py --use_legacy_frontend --ir_version=11 -k "sigmoid" --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_Activation.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: Clone API snippets if: runner.os != 'macOS' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/job_pytorch_layer_tests.yml b/.github/workflows/job_pytorch_layer_tests.yml index 4ea6d33336fc87..b0eba0a278e582 100644 --- a/.github/workflows/job_pytorch_layer_tests.yml +++ b/.github/workflows/job_pytorch_layer_tests.yml @@ -24,7 +24,8 @@ on: permissions: read-all env: - PIP_CACHE_PATH: /mount/caches/pip/linux + PIP_CACHE_PATH_LINUX: /mount/caches/pip/linux + PIP_CACHE_PATH_WIN: "C:\\mount\\caches\\pip\\win" jobs: PyTorch_Layer_Tests: @@ -94,9 +95,9 @@ jobs: uses: ./openvino/.github/actions/setup_python with: version: ${{ inputs.python-version }} - pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }} - should-setup-pip-paths: ${{ runner.os == 'Linux' }} - self-hosted-runner: ${{ runner.os == 'Linux' }} + pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH_LINUX || env.PIP_CACHE_PATH_WIN }} + should-setup-pip-paths: ${{ runner.os != 'macOS' }} + self-hosted-runner: ${{ runner.os != 'macOS' }} - name: Install OpenVINO Python wheels (Linux and macOS) if: runner.os != 'Windows' diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 2959728e39dee7..d52b819981d821 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -22,7 +22,7 @@ permissions: read-all jobs: PyTorch_Models_Tests: name: PyTorch Models tests - timeout-minutes: ${{ inputs.model_scope == 'precommit' && 40 || 400 }} + timeout-minutes: ${{ inputs.model_scope == 'precommit' && 45 || 400 }} runs-on: ${{ inputs.runner }} container: ${{ fromJSON(inputs.container) }} defaults: diff --git a/.github/workflows/job_tensorflow_layer_tests.yml b/.github/workflows/job_tensorflow_layer_tests.yml index 26730f9b55df7a..0de1708527739a 100644 --- a/.github/workflows/job_tensorflow_layer_tests.yml +++ b/.github/workflows/job_tensorflow_layer_tests.yml @@ -24,12 +24,13 @@ on: permissions: read-all env: - PIP_CACHE_PATH: /mount/caches/pip/linux + PIP_CACHE_PATH_LINUX: /mount/caches/pip/linux + PIP_CACHE_PATH_WIN: "C:\\mount\\caches\\pip\\win" jobs: TensorFlow_Layer_Tests: name: TensorFlow Layer Tests - timeout-minutes: 30 + timeout-minutes: 45 runs-on: ${{ inputs.runner }} container: ${{ fromJSON(inputs.container) }} defaults: @@ -94,9 +95,9 @@ jobs: uses: ./openvino/.github/actions/setup_python with: version: ${{ inputs.python-version }} - pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }} - should-setup-pip-paths: ${{ runner.os == 'Linux' }} - self-hosted-runner: ${{ runner.os == 'Linux' }} + pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH_LINUX || env.PIP_CACHE_PATH_WIN }} + should-setup-pip-paths: ${{ runner.os != 'macOS' }} + self-hosted-runner: ${{ runner.os != 'macOS' }} - name: Install OpenVINO Python wheels (Linux and macOS) if: runner.os != 'Windows' diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml index 5c5e59aa3bec97..4b84bee25c78f4 100644 --- a/.github/workflows/job_tokenizers.yml +++ b/.github/workflows/job_tokenizers.yml @@ -28,7 +28,8 @@ on: permissions: read-all env: - PIP_CACHE_PATH: /mount/caches/pip/linux + PIP_CACHE_PATH_LINUX: /mount/caches/pip/linux + PIP_CACHE_PATH_WIN: "C:\\mount\\caches\\pip\\win" TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} jobs: @@ -70,9 +71,9 @@ jobs: uses: ./.github/actions/setup_python with: version: ${{ inputs.python-version }} - pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }} - should-setup-pip-paths: ${{ runner.os == 'Linux' }} - self-hosted-runner: ${{ runner.os == 'Linux' }} + pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH_LINUX || env.PIP_CACHE_PATH_WIN }} + should-setup-pip-paths: ${{ runner.os != 'macOS' }} + self-hosted-runner: ${{ runner.os != 'macOS' }} - name: Clone OpenVINO Tokenizers uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 0af30621a2a7fd..10de6867c7d0e2 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -5,7 +5,7 @@ on: merge_group: push: branches: - # - master + - master - 'releases/**' concurrency: diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index 486e89fe8647c1..7e49d6fc11fe12 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -169,7 +169,7 @@ jobs: - name: Cmake build - CC COLLECT run: | cmake --build ${BUILD_DIR} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib + cmake --build ${BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats @@ -182,7 +182,7 @@ jobs: - name: Build C++ samples - OpenVINO build tree run: | cmake -G "${{ env.CMAKE_GENERATOR }}" -DOpenVINO_DIR=${BUILD_DIR} -S ${INSTALL_DIR}/samples/cpp -B ${BUILD_DIR}/cpp_samples - cmake --build ${BUILD_DIR}/cpp_samples --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device + cmake --build ${BUILD_DIR}/cpp_samples --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device - name: Build C samples - OpenVINO install tree run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples diff --git a/.github/workflows/linux_riscv.yml b/.github/workflows/linux_riscv.yml index c450a5d30768e4..85b0db8c36294e 100644 --- a/.github/workflows/linux_riscv.yml +++ b/.github/workflows/linux_riscv.yml @@ -213,13 +213,13 @@ jobs: source ${OPENVINO_BUILD_DIR}/dependencies/deactivate_conanbuild.sh - name: Cmake - Build - run: cmake --build ${OPENVINO_BUILD_DIR} --parallel + run: cmake --build ${OPENVINO_BUILD_DIR} --parallel $(nproc) - name: Show ccache stats run: ccache --show-stats - name: Cmake - Install - run: cmake --build ${OPENVINO_BUILD_DIR} --parallel --target install + run: cmake --build ${OPENVINO_BUILD_DIR} --parallel $(nproc) --target install - name: Build OpenVINO C++ samples run: | diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 5492ad40aa17b4..c587c5ad7323b3 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -159,7 +159,7 @@ jobs: -B ${{ env.BUILD_DIR }} - name: Cmake build - OpenVINO - run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show ccache stats run: ccache --show-stats @@ -186,7 +186,7 @@ jobs: -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}/modules/custom_operations \ -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: CMake configure, build and install - OpenVINO JS API if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API @@ -196,7 +196,7 @@ jobs: -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR_JS }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake # diff --git a/.github/workflows/mac_arm64.yml b/.github/workflows/mac_arm64.yml index 8100b74734ab17..0708a844fe6b8b 100644 --- a/.github/workflows/mac_arm64.yml +++ b/.github/workflows/mac_arm64.yml @@ -159,7 +159,7 @@ jobs: -B ${{ env.BUILD_DIR }} - name: Cmake build - OpenVINO - run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + run: cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: Show ccache stats run: ccache --show-stats @@ -187,7 +187,7 @@ jobs: -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}/modules/custom_operations \ -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} - name: CMake configure, build and install - OpenVINO JS API if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API @@ -197,7 +197,7 @@ jobs: -S ${{ env.OPENVINO_REPO }} \ -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR }} --parallel + cmake --build ${{ env.BUILD_DIR }} --parallel $(nproc) cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR_JS }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake # diff --git a/.github/workflows/manylinux_2014.yml b/.github/workflows/manylinux_2014.yml new file mode 100644 index 00000000000000..bd5da965226a50 --- /dev/null +++ b/.github/workflows/manylinux_2014.yml @@ -0,0 +1,236 @@ +name: Manylinux 2014 +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-manylinux-2014 + cancel-in-progress: true + +permissions: read-all + +env: + PIP_CACHE_PATH: /mount/caches/pip/linux + +jobs: + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + ref_name: ${{ github.ref_name }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + + - name: Show affected components + run: | + echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}" + shell: bash + + Docker: + needs: Smart_CI + if: "!needs.smart_ci.outputs.skip_workflow" + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_22_04_x64_docker + ov_build/manylinux2014_x86_64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Docker] + timeout-minutes: 120 + defaults: + run: + shell: bash + runs-on: aks-linux-16-cores-32gb-manylinux + if: ${{ github.repository_owner == 'openvinotoolkit' }} + container: + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64_docker }} + volumes: + - /mount:/mount + options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING -e DOCKER_CONFIG -v ${{ github.workspace }}:${{ github.workspace }} + env: + CMAKE_BUILD_TYPE: 'Release' + ARCH: 'x86_64' + OPENVINO_REPO: ${{ github.workspace }}/src + INSTALL_DIR: ${{ github.workspace }}/install/openvino + INSTALL_WHEELS_DIR: ${{ github.workspace }}/install/wheels + BUILD_DIR: ${{ github.workspace }}/build + DOCKER_CONFIG: "/mount/.docker" + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_C_COMPILER_LAUNCHER: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: 1 + SCCACHE_SERVER_PORT: 35555 + SCCACHE_CACHE_SIZE: 50G + SCCACHE_AZURE_KEY_PREFIX: manylinux_2014 + ARTIFACTS_SHARE: "/mount/build-artifacts" + MANIFEST_PATH: ${{ github.workspace }}/manifest.yml + PRODUCT_TYPE: public_manylinux2014_x86_64_release + + steps: + - name: Clone OpenVINO + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{ env.OPENVINO_REPO }} + submodules: 'true' + + - name: System info + uses: ./src/.github/actions/system_info + + - name: Generate product manifest and set CI_BUILD_NUMBER & CI_BUILD_DEV_TAG + id: create_manifest + uses: ./src/.github/actions/create_manifest + with: + repos: | + ${{ env.OPENVINO_REPO }} + product_type: ${{ env.PRODUCT_TYPE }} + target_arch: ${{ env.ARCH }} + build_type: ${{ env.CMAKE_BUILD_TYPE }} + save_to: ${{ env.MANIFEST_PATH }} + + - name: Create docker build cache + run: | + docker volume create ov_build_cache + + - name: Build OpenVINO + run: | + docker run --rm \ + -v ${{ env.OPENVINO_REPO }}:/work/src \ + -v ov_build_cache:/work/build \ + -v ${{ env.INSTALL_DIR }}:/work/install \ + -e SCCACHE_AZURE_BLOB_CONTAINER \ + -e SCCACHE_AZURE_CONNECTION_STRING \ + -e SCCACHE_SERVER_PORT \ + -e SCCACHE_IGNORE_SERVER_IO_ERROR \ + -e SCCACHE_CACHE_SIZE \ + -e SCCACHE_AZURE_KEY_PREFIX \ + -e CMAKE_CXX_COMPILER_LAUNCHER \ + -e CMAKE_C_COMPILER_LAUNCHER \ + -e CI_BUILD_NUMBER \ + -e CI_BUILD_DEV_TAG \ + -w /work/src \ + ${{ fromJSON(needs.docker.outputs.images).ov_build.manylinux2014_x86_64 }} \ + /bin/bash -c " + cmake -DENABLE_CPPLINT=OFF -DENABLE_NCC_STYLE=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF -S /work/src -B /work/build && + cmake --build /work/build --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} && + cmake --install /work/build --config ${{ env.CMAKE_BUILD_TYPE }} --prefix /work/install + " + + - name: Pack Artifacts + run: mkdir -p ${{ env.BUILD_DIR }} && tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/openvino_package.tar.gz + working-directory: ${{ env.INSTALL_DIR }} + + - name: Build Python API(Python 3.9-3.13) + run: | + SUPPORTED_PYTHON_VERSIONS=("39" "310" "311" "312" "313") + for PY_VER in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + python_path=/opt/python/cp${PY_VER}-cp${PY_VER}/bin + docker run --rm \ + -v ${{ env.OPENVINO_REPO }}:/work/src \ + -v ${{ env.INSTALL_WHEELS_DIR }}:/work/wheels \ + -v ${{ env.PIP_CACHE_PATH }}:/work/pip_cache \ + -v ov_build_cache:/work/build \ + -e SCCACHE_AZURE_BLOB_CONTAINER \ + -e SCCACHE_AZURE_CONNECTION_STRING \ + -e SCCACHE_SERVER_PORT \ + -e SCCACHE_IGNORE_SERVER_IO_ERROR \ + -e SCCACHE_CACHE_SIZE \ + -e SCCACHE_AZURE_KEY_PREFIX \ + -e CMAKE_CXX_COMPILER_LAUNCHER \ + -e CMAKE_C_COMPILER_LAUNCHER \ + -e CI_BUILD_NUMBER \ + -e CI_BUILD_DEV_TAG \ + -w /work/src \ + ${{ fromJSON(needs.docker.outputs.images).ov_build.manylinux2014_x86_64 }} \ + /bin/bash -c " + export PATH=${python_path}:\$PATH + PIP_VER=$(python3 -c "import pip; print(pip.__version__)") + export "PIP_CACHE_DIR=/work/pip_cache/${PIP_VER}" + python3 -m pip install -r /work/src/src/bindings/python/wheel/requirements-dev.txt && + cmake -DOpenVINODeveloperPackage_DIR=/work/build -DENABLE_PYTHON=ON -DENABLE_WHEEL=ON -S /work/src/src/bindings/python -B /work/build_py${PY_VER} && + cmake --build /work/build_py${PY_VER} --parallel $(nproc) --target ie_wheel --config ${{ env.CMAKE_BUILD_TYPE }} && + cmake --install /work/build_py${PY_VER} --config ${{ env.CMAKE_BUILD_TYPE }} --prefix /work/wheels --component python_wheels + " + done + + # + # Upload build artifacts + # + - name: Upload openvino package + if: ${{ always() }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + if-no-files-found: 'error' + + - name: Upload openvino wheels + if: ${{ always() }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: openvino_wheels + path: ${{ env.INSTALL_WHEELS_DIR }}/wheels/*.whl + if-no-files-found: 'error' + + - name: Store artifacts to a shared drive + id: store_artifacts + if: ${{ always() }} + uses: ./src/.github/actions/store_artifacts + with: + artifacts: | + ${{ env.BUILD_DIR }}/openvino_package.tar.gz + ${{ env.MANIFEST_PATH }} + ${{ env.INSTALL_WHEELS_DIR }}/wheels + storage_dir: ${{ env.PRODUCT_TYPE }} + storage_root: ${{ env.ARTIFACTS_SHARE }} + + Overall_Status: + name: ci/gha_overall_status_manylinux2014 + needs: [Smart_CI, Build] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Check status of all jobs + if: >- + ${{ + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + }} + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/mo.yml b/.github/workflows/mo.yml deleted file mode 100644 index f48986d4a0d304..00000000000000 --- a/.github/workflows/mo.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: MO -on: - push: - paths: - - 'tools/mo/**' - - '.github/workflows/mo.yml' - branches: - - 'master' - - 'releases/**' - pull_request: - paths: - - 'tools/mo/**' - - '.github/workflows/mo.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: read-all - -jobs: - Pylint-UT: - runs-on: ubuntu-22.04 - if: ${{ github.repository_owner == 'openvinotoolkit' }} - steps: - - name: Clone OpenVINO - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.10' - - - name: Cache pip - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/mo/requirements*.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - # For UT - pip install unittest-xml-reporting==3.0.2 - # MO requirements - pip install -r requirements_caffe.txt - pip install -r requirements_kaldi.txt - pip install -r requirements_onnx.txt - pip install -r requirements_tf2.txt - pip install -r requirements_dev.txt - working-directory: tools/mo - - - name: Pylint-MO - run: pylint -d C,R,W openvino/tools/mo - working-directory: tools/mo diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 92178fce7f5054..f4caec8b2458a0 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -477,7 +477,7 @@ jobs: -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ -S ${OPENVINO_CONTRIB_REPO}/modules/nvidia_plugin \ -B ${NVIDIA_BUILD_DIR} - cmake --build ${NVIDIA_BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --verbose -- ov_nvidia_func_tests ov_nvidia_unit_tests + cmake --build ${NVIDIA_BUILD_DIR} --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} --verbose -- ov_nvidia_func_tests ov_nvidia_unit_tests - name: Show ccache stats run: ${SCCACHE_PATH} --show-stats diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index 5212e862e43ac4..1a424a3c9ca2ca 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -193,7 +193,7 @@ jobs: - name: Build C++ samples - OpenVINO build tree run: | cmake -G "${{ env.CMAKE_GENERATOR }}" -DOpenVINO_DIR=${{ env.BUILD_DIR }} -S ${{ env.INSTALL_DIR }}/samples/cpp -B ${{ env.BUILD_DIR }}/cpp_samples - cmake --build ${{ env.BUILD_DIR }}/cpp_samples --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device + cmake --build ${{ env.BUILD_DIR }}/cpp_samples --parallel $ENV:NUMBER_OF_PROCESSORS --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device - name: Build C samples - OpenVINO install tree run: | @@ -331,7 +331,7 @@ jobs: -B ${{ env.BUILD_DIR }} - name: Cmake build - CC ON - run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target benchmark_app --verbose + run: cmake --build ${{ env.BUILD_DIR }} --parallel $ENV:NUMBER_OF_PROCESSORS --config ${{ env.CMAKE_BUILD_TYPE }} --target benchmark_app --verbose - name: List bin files shell: cmd diff --git a/.github/workflows/windows_vs2019_release.yml b/.github/workflows/windows_vs2019_release.yml index b9b8fa76d37c34..1b218cdf7d430b 100644 --- a/.github/workflows/windows_vs2019_release.yml +++ b/.github/workflows/windows_vs2019_release.yml @@ -14,7 +14,7 @@ concurrency: env: TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} - PIP_CACHE_PATH: /mount/caches/pip/win + PIP_CACHE_PATH: "C:\\mount\\caches\\pip\\win" PYTHON_VERSION: '3.11' permissions: read-all @@ -287,7 +287,7 @@ jobs: with: version: ${{ env.PYTHON_VERSION }} pip-cache-path: ${{ env.PIP_CACHE_PATH }} - should-setup-pip-paths: 'false' + should-setup-pip-paths: 'true' self-hosted-runner: 'true' - name: Install OpenVINO Python wheels @@ -296,9 +296,6 @@ jobs: $ovCoreWheelPath=Get-ChildItem -Path . -Filter openvino-*.whl | % { $_.FullName } python3 -m pip install "$ovCoreWheelPath" - # Find and install the dev OV wheel - $ovDevWheelPath=Get-ChildItem -Path . -Filter openvino_dev*.whl | % { $_.FullName } - python3 -m pip install "$ovDevWheelPath[caffe,kaldi,onnx,tensorflow2,pytorch]" working-directory: ${{ env.INSTALL_WHEELS_DIR }} - name: Install Python API tests dependencies @@ -309,8 +306,11 @@ jobs: # For torchvision to OpenVINO preprocessing converter python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/python/preprocess/torchvision/requirements.txt - # TODO: replace with Python API tests requirements - python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/mo/requirements_dev.txt + # For validation of Python API + python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/bindings/python/requirements_test.txt + + # ONNX tests requirements + python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/requirements_onnx # For getting rid of SSL issues during model downloading for unit tests python3 -m pip install certifi @@ -318,34 +318,17 @@ jobs: - name: Set SSL_CERT_FILE for model downloading for unit tests run: echo SSL_CERT_FILE=$(python3 -m certifi) >> $env:GITHUB_ENV - - name: Python API Tests - #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test # Ticket: 127101 - shell: cmd - run: | - set PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% - python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py - - - name: Model Optimizer UT - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - shell: cmd - run: | - python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml - - name: Install Python Layer tests dependencies run: | # layer test requirements python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - name: ONNX Layer Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test + - name: Python API Tests + #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test # Ticket: 127101 shell: cmd run: | - :: requires 'unit_tests' from 'tools/mo' - set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\mo;%PYTHONPATH% - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/onnx_tests -n logical -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 + set PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH% + python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py - name: TensorFlow Lite Layer Tests - TFL FE if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test @@ -366,18 +349,6 @@ jobs: --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-onnx_frontend.xml ^ --ignore=${{ env.INSTALL_TEST_DIR }}/onnx/test_python/test_zoo_models.py - - name: MO Python API Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - shell: cmd - run: | - :: Used for 'test_utils' installed in '\python\openvino\test_utils' - set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\python\openvino\test_utils;${{ env.INSTALL_TEST_DIR }}\python;%PYTHONPATH% - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/mo_python_api_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - name: OVC Python API Tests if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test shell: cmd diff --git a/CMakeLists.txt b/CMakeLists.txt index e9e8d3724d9ac5..65a72ef8f4936e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -138,7 +138,7 @@ function(ov_developer_package_export_targets) endforeach() endif() else() - message(FATAL_ERROR "Internal error: ${target_name} does not represent a cmake target") + message(FATAL_ERROR "Internal error: '${EXPORT_TARGET}' does not represent a cmake target") endif() list(REMOVE_DUPLICATES _OPENVINO_DEVELOPER_PACKAGE_TARGETS) diff --git a/README.md b/README.md index c37f2ef42b9785..695f84f1628118 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,11 @@ Check [system requirements](https://docs.openvino.ai/2024/about-openvino/system- [OpenVINO Quickstart example](https://docs.openvino.ai/2024/get-started.html) will walk you through the basics of deploying your first model. Learn how to optimize and deploy popular models with the [OpenVINO Notebooks](https://github.com/openvinotoolkit/openvino_notebooks)šŸ“š: -- [Create an LLM-powered Chatbot using OpenVINO](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/llm-chatbot/llm-chatbot.ipynb) -- [YOLOv8 Optimization](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/quantizing-model-with-accuracy-control/yolov8-quantization-with-accuracy-control.ipynb) -- [Text-to-Image Generation](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/controlnet-stable-diffusion/controlnet-stable-diffusion.ipynb) +- [Create an LLM-powered Chatbot using OpenVINO](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/llm-chatbot/llm-chatbot-generate-api.ipynb) +- [YOLOv11 Optimization](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/yolov11-optimization/yolov11-object-detection.ipynb) +- [Text-to-Image Generation](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/text-to-image-genai/text-to-image-genai.ipynb) +- [Multimodal assistant with LLaVa and OpenVINO](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/llava-multimodal-chatbot/llava-multimodal-chatbot-genai.ipynb) +- [Automatic speech recognition using Whisper and OpenVINO](https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/notebooks/whisper-asr-genai/whisper-asr-genai.ipynb) Here are easy-to-follow code examples demonstrating how to run PyTorch and TensorFlow model inference using OpenVINO: diff --git a/cmake/developer_package/compile_flags/sdl.cmake b/cmake/developer_package/compile_flags/sdl.cmake index 34ad5904519e7f..35f59cb8970573 100644 --- a/cmake/developer_package/compile_flags/sdl.cmake +++ b/cmake/developer_package/compile_flags/sdl.cmake @@ -55,11 +55,20 @@ if(ENABLE_INTEGRITYCHECK) set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /INTEGRITYCHECK") endif() -set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") -set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") -set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") -set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR (OV_COMPILER_IS_INTEL_LLVM AND WIN32)) + # add sdl required flags to both Debug and Release on Windows + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OV_C_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OV_C_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OV_LINKER_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${OV_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OV_LINKER_FLAGS}") +else() + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${OV_C_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${OV_LINKER_FLAGS}") +endif() unset(OV_C_CXX_FLAGS) unset(OV_LINKER_FLAGS) diff --git a/cmake/features.cmake b/cmake/features.cmake index e5c36e1730731a..e46eff156c2048 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -206,6 +206,9 @@ ov_dependent_option (ENABLE_SYSTEM_PROTOBUF "Enables use of system Protobuf" OFF # the option is turned off by default, because we don't want to have a dependency on libsnappy.so ov_dependent_option (ENABLE_SYSTEM_SNAPPY "Enables use of system version of Snappy" OFF "ENABLE_SNAPPY_COMPRESSION" OFF) +# the option is turned off by default, because we are not sure that system version of ZE loader is fresh enough +ov_dependent_option (ENABLE_SYSTEM_LEVEL_ZERO "Enables use of system version of Level Zero" OFF + "ENABLE_INTEL_NPU" OFF) ov_dependent_option(ENABLE_JS "Enables JS API building" ${ENABLE_JS_DEFAULT} "NOT ANDROID;NOT EMSCRIPTEN" OFF) diff --git a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst index 3f3d0064e8a4c6..c80dc388568004 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst @@ -76,14 +76,14 @@ Feature Support and API Coverage | HETERO | 61.22 % | 99.24 % | 86.05 % | +-------------------------+-----------+------------------+-------------------+ | || Percentage of API supported by the device, | -| || as of OpenVINO 2024.4, 25 Oct, 2024. | +| || as of OpenVINO 2024.5, 20 Nov. 2024. | +-------------------------+-----------+------------------+-------------------+ For setting up a relevant configuration, refer to the :doc:`Integrate with Customer Application <../../openvino-workflow/running-inference/integrate-openvino-with-your-application>` topic (step 3 "Configure input and output"). -.. dropdown:: Device support across OpenVINO 2024.4 distributions +.. dropdown:: Device support across OpenVINO 2024.5 distributions =============== ========== ====== =============== ======== ============ ========== ========== ========== Device Archives PyPI APT/YUM/ZYPPER Conda Homebrew vcpkg Conan npm diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index ed9d39aaf8b9e6..8a58dc27df1f83 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -59,7 +59,7 @@ implemented in your solutions. Click the buttons below to see the chosen benchma .. grid-item:: .. button-link:: # - :class: ovms-toolkit-benchmark-llm + :class: ovms-toolkit-benchmark-llm-result :color: primary :outline: :expand: @@ -132,21 +132,21 @@ For a listing of all platforms and configurations used for testing, refer to the .. grid-item:: - .. button-link:: ../_static/benchmarks_files/OV-2024.4-platform_list.pdf + .. button-link:: ../_static/benchmarks_files/OV-2024.5-platform_list.pdf :color: primary :outline: :expand: :material-regular:`download;1.5em` Click for Hardware Platforms [PDF] - .. button-link:: ../_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx + .. button-link:: ../_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx :color: primary :outline: :expand: :material-regular:`download;1.5em` Click for Configuration Details [XLSX] - .. button-link:: ../_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx + .. button-link:: ../_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx :color: primary :outline: :expand: @@ -160,10 +160,10 @@ For a listing of all platforms and configurations used for testing, refer to the **Disclaimers** * IntelĀ® Distribution of OpenVINOā„¢ toolkit performance results are based on release - 2024.3, as of July 31, 2024. + 2024.5, as of November 20, 2024. * OpenVINO Model Server performance results are based on release - 2024.3, as of Aug. 19, 2024. + 2024.4, as of Sept. 30, 2024. The results may not reflect all publicly available updates. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software, or service diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst index b8256af650e2f8..5697fcbf6e4d74 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -5,9 +5,7 @@ This page is regularly updated to help you identify the best-performing LLMs on IntelĀ® Coreā„¢ Ultra processor family and AI PCs. The current data is as of OpenVINO 2024.4, 24 Oct. 2024 -The tables below list the key performance indicators for a selection of Large Language Models, -running on an IntelĀ® Coreā„¢ Ultra 7-165H, IntelĀ® Coreā„¢ Ultra 7-265V, and IntelĀ® Coreā„¢ Ultra -7-288V based system, on built-in GPUs. +The tables below list the key performance indicators for inference on built-in GPUs. @@ -16,14 +14,32 @@ running on an IntelĀ® Coreā„¢ Ultra 7-165H, IntelĀ® Coreā„¢ Ultra 7-265V, and In +.. tab-set:: -.. csv-table:: - :class: modeldata stripe - :name: supportedModelsTableOv - :header-rows: 1 - :file: ../../_static/benchmarks_files/llm_models.csv + .. tab-item:: 9-288V + + .. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models_9-288V.csv + + .. tab-item:: 7-268V + + .. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models_7-258V.csv + + .. tab-item:: 7-155H + + .. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models_7-155H.csv -| .. grid:: 1 1 2 2 :gutter: 4 diff --git a/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst b/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst index 3162bae7254704..e87733a1445356 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst @@ -1,9 +1,6 @@ Model Accuracy ============== - - -The following two tables present the absolute accuracy drop calculated as the accuracy difference between OV-accuracy and the original framework accuracy for FP32, and the same for INT8, BF16, and FP16 representations of a model on three platform architectures. The third table presents the GenAI model accuracies as absolute accuracy values. Refer to notes below the table for more @@ -11,7 +8,7 @@ information. * A - IntelĀ® Coreā„¢ i9-9000K (AVX2), INT8 and FP32 * B - IntelĀ® XeonĀ® 6338, (VNNI), INT8 and FP32 -* C - IntelĀ® Xeon 8480+ (VNNI, AMX), INT8, BF16, FP32 +* C - IntelĀ® Xeon 8580 (VNNI, AMX), INT8, BF16, FP32 * D - IntelĀ® Flex-170, INT8 and FP16 @@ -28,73 +25,52 @@ information. * - bert-base-cased - SST-2_bert_cased_padded - spearman@cosine - - 3.33% - - 3.22% - - 3.05% - - 2.88% - * - bert-large-uncased-whole-word-masking-squad-0001 - - SQUAD_v1_1_bert_msl384_mql64_ds128_lowercase - - F1 - - 0.12% - - 0.03% - - 0.03% - - 0.28% + - 3.06% + - 2.89% + - 2.71% + - 2.71% * - efficientdet-d0 - COCO2017_detection_91cl - coco_precision - - 0.00% - - -0.52% - - -0.54% - - -0.60% + - -0.84% + - -0.59% + - -0.59% + - -0.55% * - mask_rcnn_resnet50_atrous_coco - COCO2017_detection_91cl_bkgr - coco_orig_precision - - 0.05% - - 0.03% - - 0.08% - - -0.09% + - -0.10% + - -0.04% + - 0.07% + - -0.01% * - mobilenet-v2 - ImageNet2012 - accuracy @ top1 - - - -0.87% - - -0.88% - - -0.88% + - -0.97% + - -0.98% + - -0.95% * - resnet-50 - ImageNet2012 - accuracy @ top1 - - -0.17% - - -0.18% - - -0.18% - - -0.16% + - 0.74% + - 0.76% + - 0.74% + - 0.82% * - ssd-resnet34-1200 - COCO2017_detection_80cl_bkgr - map - - -0.03% - - -0.02% - - -0.03% - - 0.02% + - -0.06% + - -0.08% + - -0.07% + - -0.06% * - ssd-mobilenet-v1-coco - COCO2017_detection_80cl_bkgr - coco-precision - - -2.74% - - -0.11% - - -0.13% - - -0.12% - * - unet-camvid-onnx-0001 - - CamVid_12cl - - mean_iou @ mean - - -6.28% - - 6.45% - - 6.46% - - 6.43% - * - yolo_v5m - - COCO2017_detection_80cl - - map - - -0.40% - - -0.32% - - -0.32% - - -0.31% + - -2.94% + - -0.28% + - -0.28% + - -0.26% * - yolo_v8n - COCO2017_detection_80cl - map @@ -121,30 +97,22 @@ information. - 0.00% - 0.00% - -0.01% - - 0.01% - * - bert-large-uncased-whole-word-masking-squad-0001 - - SQUAD_v1_1_bert_msl384_mql64_ds128_lowercase - - F1 - - 0.04% - - 0.04% - - 0.06% - - 0.06% - - 0.04% + - 0.02% * - efficientdet-d0 - COCO2017_detection_91cl - coco_precision - 0.01% - - -0.02% - 0.01% + - 0.01% + - 0.00% - 0.00% - - -0.02% * - mask_rcnn_resnet50_atrous_coco - COCO2017_detection_91cl_bkgr - coco_orig_precision - -0.01% - -0.01% - -0.01% - - -0.05% + - 0.05% - 0.00% * - mobilenet-v2 - ImageNet2012 @@ -160,40 +128,24 @@ information. - 0.00% - 0.00% - 0.00% - - -0.01% - - -0.01% + - 0.01% + - 0.01% * - ssd-resnet34-1200 - COCO2017_detection_80cl_bkgr - map - 0.02% - - 0.00% - - 0.00% - - -0.02% - - 0.04% + - 0.02% + - 0.02% + - -0.01% + - 0.02% * - ssd-mobilenet-v1-coco - COCO2017_detection_80cl_bkgr - coco-precision - - -0.08% - - 0.01% + - 0.04% - 0.01% + - 0.04% - 0.08% - 0.01% - * - unet-camvid-onnx-0001 - - CamVid_12cl - - mean_iou @ mean - - 0.00% - - 0.00% - - 0.00% - - -0.03% - - -0.03% - * - yolo_v5m - - COCO2017_detection_80cl - - map - - 0.00% - - 0.05% - - 0.05% - - 0.07% - - 0.07% * - yolo_v8n - COCO2017_detection_80cl - map @@ -213,46 +165,60 @@ information. - B, VNNI-INT4 - C, FAMX-FP16 - D, MTL-INT4 - * - chatGLM2-6b + * - chatGLM4 - Wikiset - ppl - - 5.24 - - 6.03 - - 5.24 - - 6.03 - * - Falcon-7b-instruct + - + - + - + - + * - Gemma-2-9B - Wikitext - ppl - - 1.65 - - 1.76 - - 1.65 - - 1.76 + - + - 1.57 + - 1.57 + - * - Llama-2-7b-chat - Wikiset - ppl - - 1.58 - - 1.59 - - 1.91 + - + - - 1.59 + - * - Llama-3-8b - Wikiset - ppl - - 1.54 - - 1.56 + - 1.45 + - 1.48 + - 1.45 + - + * - Llama-3.2-3b-instruct + - Wikiset + - ppl + - 1.60 + - 1.62 - 1.17 - - 1.57 + - * - Mistral-7b - Wikitext - ppl - 1.48 - 1.49 - - 1.39 - - 1.49 + - 1.48 + - * - Phi3-mini-4k-instruct - Wikitext - ppl - 1.52 + - 1.55 + - 1.52 - 1.56 + * - Qwen-2-7B + - Wikitext + - ppl + - 1.52 + - 1.53 - 1.52 - 1.56 diff --git a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst index c55d3f44451f1c..0f70c93e9c8b96 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst @@ -31,10 +31,13 @@ Performance Information F.A.Q. .. dropdown:: How can I run the benchmark results on my own? - All of the performance benchmarks are generated using the + All of the performance benchmarks on traditional network models are generated using the open-source tool within the IntelĀ® Distribution of OpenVINOā„¢ toolkit called :doc:`benchmark_app <../../learn-openvino/openvino-samples/benchmark-tool>`. + For diffusers (Stable-Diffusion) and foundational models (aka LLMs) please use the OpenVINO GenAI + opensource repo `OpenVINO GenAI tools/llm_bench `__ + For a simple instruction on testing performance, see the :doc:`Getting Performance Numbers Guide `. .. dropdown:: Where can I find a more detailed description of the workloads used for benchmarking? @@ -50,23 +53,27 @@ Performance Information F.A.Q. - Public Network - Task - Input Size - * - `chatGLM2-6B `__ + * - `chatGLM4-9B `__ - THUDM - Transformer - - 32K - * - `Falcon-7b-instruct `__ + - 128K + * - `Gemma-2-9B `__ - Hugginface - - Causal Decoder-only - - 2048 - * - `Llama-2-7b-chat `__ + - Text-To-Text Decoder-only + - 8K + * - `Llama-2-7b-chat `__ - Meta AI - Auto regressive language - - 4096 - * - `Llama-3-8b `__ + - 4K + * - `Llama-3-8b `__ + - Meta AI + - Auto regressive language + - 8K + * - `Llama-3.2-3B `__ - Meta AI - Auto regressive language - - 8192 - * - `Mistral-7b `__ + - 128K + * - `Mistral-7b-V0.1 `__ - Mistral AI - Auto regressive language - 4096 @@ -74,7 +81,11 @@ Performance Information F.A.Q. - Huggingface - Auto regressive language - 4096 - * - `Stable-Diffusion-V1-5 `__ + * - `Qwen-2-7B `__ + - Huggingface + - Auto regressive language + - 128K + * - `Stable-Diffusion-V1-5 `__ - Hugginface - Latent Diffusion Model - 77 @@ -82,10 +93,6 @@ Performance Information F.A.Q. - BERT - question / answer - 128 - * - `bert-large-uncased-whole-word-masking-squad-int8-0001 `__ - - BERT-large - - question / answer - - 384 * - `efficientdet-d0 `__ - Efficientdet - classification @@ -110,15 +117,7 @@ Performance Information F.A.Q. - ssd-resnet34 onnx model - object detection - 1200x1200 - * - `unet-camvid-onnx-0001 `__ - - U-Net - - semantic segmentation - - 368x480 - * - `yolo-v5m `__ - - YOLO V5 Medium - - object detection - - 640x640 - * - `yolov8n `__ + * - `yolov8n `__ - Yolov8nano - object detection - 608x608 diff --git a/docs/articles_en/about-openvino/release-notes-openvino.rst b/docs/articles_en/about-openvino/release-notes-openvino.rst index 4bd0b5d32c0f0e..343c9e780f05dc 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino.rst @@ -1,3 +1,4 @@ +============================= OpenVINO Release Notes ============================= @@ -15,115 +16,462 @@ OpenVINO Release Notes -2024.4 - 19 September 2024 +2024.5 - 20 November 2024 ############################# :doc:`System Requirements <./release-notes-openvino/system-requirements>` | :doc:`Release policy <./release-notes-openvino/release-policy>` | :doc:`Installation Guides <./../get-started/install-openvino>` + + What's new +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* More Gen AI coverage and framework integrations to minimize code changes. +* More GenAI coverage and framework integrations to minimize code changes. - * Support for GLM-4-9B Chat, MiniCPM-1B, Llama 3 and 3.1, Phi-3-Mini, Phi-3-Medium and - YOLOX-s models. - * Noteworthy notebooks added: Florence-2, NuExtract-tiny Structure Extraction, Flux.1 Image - Generation, PixArt-Ī±: Photorealistic Text-to-Image Synthesis, and Phi-3-Vision Visual - Language Assistant. + * New models supported: Llama 3.2 (1B & 3B), Gemma 2 (2B & 9B), and YOLO11. + * LLM support on NPU: Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, Qwen2-7B-Instruct and Phi-3 + Mini-Instruct. + * Noteworthy notebooks added: Sam2, Llama3.2, Llama3.2 - Vision, Wav2Lip, Whisper, and Llava. + * Preview: support for Flax, a high-performance Python neural network library based on JAX. + Its modular design allows for easy customization and accelerated inference on GPUs. * Broader Large Language Model (LLM) support and more model compression techniques. - * OpenVINOā„¢ runtime optimized for IntelĀ® Xe Matrix Extensions (IntelĀ® XMX) systolic arrays on - built-in GPUs for efficient matrix multiplication resulting in significant LLM performance - boost with improved 1st and 2nd token latency, as well as a smaller memory footprint on - IntelĀ® Coreā„¢ Ultra Processors (Series 2). - * Memory sharing enabled for NPUs on IntelĀ® Coreā„¢ Ultra Processors (Series 2) for efficient - pipeline integration without memory copy overhead. - * Addition of the PagedAttention feature for discrete GPUs* enables a significant boost in - throughput for parallel inferencing when serving LLMs on IntelĀ® Arcā„¢ Graphics or IntelĀ® - Data Center GPU Flex Series. + * Optimizations for built-in GPUs on IntelĀ® Coreā„¢ Ultra Processors (Series 1) and IntelĀ® Arcā„¢ + Graphics include KV Cache compression for memory reduction along with improved usability, + and model load time optimizations to improve first token latency for LLMs. + * Dynamic quantization was enabled to improve first token latency for LLMs on built-in + IntelĀ® GPUs without impacting accuracy on IntelĀ® Coreā„¢ Ultra Processors (Series 1). Second + token latency will also improve for large batch inference. + * A new method to generate synthetic text data is implemented in the Neural Network + Compression Framework (NNCF). This will allow LLMs to be compressed more accurately using + data-aware methods without datasets. Coming soon: This feature will soon be accessible via + Optimum Intel on Hugging Face. * More portability and performance to run AI at the edge, in the cloud, or locally. - * Support for IntelĀ® Core Ultra Processors Series 2 (formerly codenamed Lunar Lake) on Windows. - * OpenVINOā„¢ Model Server now comes with production-quality support for OpenAI-compatible API - which enables significantly higher throughput for parallel inferencing on IntelĀ® XeonĀ® - processors when serving LLMs to many concurrent users. - * Improved performance and memory consumption with prefix caching, KV cache compression, and - other optimizations for serving LLMs using OpenVINOā„¢ Model Server. - * Support for Python 3.12. - * Support for Red Hat Enterprise Linux (RHEL) version 9.3 - 9.4. + * Support for + `IntelĀ® XeonĀ® 6 Processors with P-cores `__ + (formerly codenamed Granite Rapids) and + `IntelĀ® Coreā„¢ Ultra 200V series processors `__ + (formerly codenamed Arrow Lake-S). + * Preview: GenAI API enables multimodal AI deployment with support for multimodal pipelines + for improved contextual awareness, transcription pipelines for easy audio-to-text + conversions, and image generation pipelines for streamlined text-to-visual conversions. + * Speculative decoding feature added to the GenAI API for improved performance and efficient + text generation using a small draft model that is periodically corrected by the full-size + model. + * Preview: LoRA adapters are now supported in the GenAI API for developers to quickly and + efficiently customize image and text generation models for specialized tasks. + * The GenAI API now also supports LLMs on NPU allowing developers to specify NPU as the + target device, specifically for WhisperPipeline (for whisper-base, whisper-medium, and + whisper-small) and LLMPipeline (for Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, + Qwen2-7B-Instruct and Phi-3 Mini-instruct). Use driver version 32.0.100.3104 or later for + best performance. Now deprecated +----------------------------------------------------------------------------------------------- + +* Python 3.8 is no longer supported: + + +OpenVINOā„¢ Runtime +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* The following will not be available beyond the 2024.4 OpenVINO version: +Common +----------------------------------------------------------------------------------------------- - * The macOS x86_64 debug bins - * Python 3.8 - * Discrete Keem Bay support +* Numpy 2.x has been adopted for all currently supported components, including NNCF. +* A new constant constructor has been added, enabling constants to be created from data pointer + as shared memory. Additionally, it can take ownership of a shared, or other, object, avoiding + a two-step process to wrap memory into ``ov::Tensor``. +* Files are now read via the async ReadFile API, reducing the bottleneck for LLM model load + times on GPU. +* CPU implementation of SliceScatter operator is now available, used for models such as Gemma, + supporting increased LLM performance. -* IntelĀ® Streaming SIMD Extensions (IntelĀ® SSE) will be supported in source code form, but not - enabled in the binary package by default, starting with OpenVINO 2025.0. -| Check the `deprecation section <#deprecation-and-support>`__ for more information. +CPU Device Plugin +----------------------------------------------------------------------------------------------- +* Gold support of the IntelĀ® XeonĀ® 6 platform with P-cores (formerly code name Granite Rapids) + has been reached. +* Support of IntelĀ® Coreā„¢ Ultra 200V series processors (formerly codenamed Arrow Lake-S) has + been implemented. +* LLM performance has been further improved with Rotary Position Embedding optimization; Query, + Key, and Value; and multi-layer perceptron fusion optimization. +* FP16 support has been extended with SDPA and PagedAttention, improving performance of LLM via + both native APIs and the vLLM integration. +* Models with LoRA adapters are now supported. -Common +GPU Device Plugin +----------------------------------------------------------------------------------------------- + +* The KV cache INT8 compression mechanism is now available for all supported GPUs. It enables a + significant reduction in memory consumption, increasing performance with a minimal impact to + accuracy (it affects systolic devices slightly more than non-systolic ones). The feature is + activated by default for non-systolic devices. +* LoRA adapters are now functionally supported on GPU. +* A new feature of GPU weightless blob caching enables caching model structure only and reusing + the weights from the original model file. Use the new OPTIMIZE_SIZE property to activate. +* Dynamic quantization with INT4 and INT8 precisions has been implemented and enabled by + default on IntelĀ® Coreā„¢ Ultra platforms, improving LLM first token latency. + + +NPU Device Plugin +----------------------------------------------------------------------------------------------- + +* Models retrieved from the OpenVINO cache have a smaller memory footprint now. The plugin + releases the cached model (blob) after weights are loaded in NPU regions. Model export is not + available in this scenario. Memory consumption is reduced during inference execution with one + blob size. This optimization requires the latest NPU driver: 32.0.100.3104. +* A driver bug for ``ov::intel_npu::device_total_mem_size`` has been fixed. The plugin will now + report 2GB as the maximum allocatable memory for any driver that does not support graph + extension 1.8. Even if older drivers report a larger amount of memory to be available, memory + allocation would fail when 2GB are exceeded. Plugin reports the number that driver exposes + for any driver that supports graph extension 1.8 (or newer). +* A new API is used to initialize the model (available in graph extension 1.8). +* Inference request set_tensors is now supported. +* ``ov::device::LUID`` is now exposed on Windows. +* LLM-related improvements have been implemented in terms of both memory usage and performance. +* AvgPool and MaxPool operator support has been extended, adding support for more PyTorch models. + +* NOTE: for systems based on IntelĀ® Coreā„¢ Ultra Processors Series 2, more than 16GB of RAM may + be required to use larger models, such as Llama-2-7B, Mistral-0.2-7B, and Qwen-2-7B + (exceeding 4B parameters) with prompt sizes over 1024 tokens. + + +OpenVINO Python API +----------------------------------------------------------------------------------------------- + +* Constant now can be created from openvino.Tensor. +* The ā€œrelease_memoryā€ method has been added for a compiled model, improving control over + memory consumption. + + + +OpenVINO Node.js API +----------------------------------------------------------------------------------------------- + +* Querying the best device to perform inference of a model with specific operations + is now available in JavaScript API. +* Contribution guidelines have been improved to make it easier for developers to contribute. +* Testing scope has been extended by inference in end-to-end tests. +* JavaScript API samples have been improved for readability and ease of running. + + + +TensorFlow Framework Support +----------------------------------------------------------------------------------------------- + +* TensorFlow 2.18.0, Keras 3.6.0, NumPy 2.0.2 in Python 3.12, and NumPy 1.26.4 in other Python + versions have been added to validation. +* Out-of-the-box conversion with static ranks has been improved by devising a new shape for + Switch-Merge condition sub-graphs. +* Complex type for the following operations is now supported: ExpandDims, Pack, Prod, Rsqrt, + ScatterNd, Sub. +* The following issues have been fixed: + + * the corner case with one element in LinSpace to avoid division by zero, + * support FP16 and FP64 input types for LeakyRelu, + * support non-i32/i64 output index type for ArgMin/Max operations. + + + +PyTorch Framework Support +----------------------------------------------------------------------------------------------- + +* PyTorch version 2.5 is now supported. +* OpenVINO Model Converter (OVC) now supports TorchScript and ExportedProgram saved on a drive. +* The issue of aten.index.Tensor conversion for indices with ā€œNoneā€ values has been fixed, + helping to support the HF Stable Diffusion model in ExportedProgram format. + + + +ONNX Framework Support +----------------------------------------------------------------------------------------------- + +* ONNX version 1.17.0 is now used. +* Customers' models with DequantizeLinear-21, com.microsoft.MatMulNBits, and + com.microsoft.QuickGelu operations are now supported. + +JAX/Flax Framework Support +----------------------------------------------------------------------------------------------- + +* JAX 0.4.35 and Flax 0.10.0 has been added to validation. +* jax._src.core.ClosedJaxpr object conversion is now supported. +* Vision Transformer from google-research/vision_transformer is now supported + (with support for 37 new operations). + + +OpenVINO Model Server +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* Encryption and decryption of topology in model cache is now supported with callback functions - provided by the user (CPU only for now; ov::cache_encryption_callbacks). -* The Ubuntu20 and Ubuntu22 Docker images now include the tokenizers and GenAI CPP modules, - including pre-installed Python modules, in development versions of these images. -* Python 3.12 is now supported. +* The OpenAI API text embedding endpoint has been added, enabling OVMS to be used as a building + block for AI applications like RAG. + `(read more) `__ +* The rerank endpoint has been added based on Cohere API, enabling easy similarity detection + between a query and a set of documents. It is one of the building blocks for AI applications + like RAG and makes integration with frameworks such as langchain easy. + `(read more) `__ +* The following improvements have been done to LLM text generation: -CPU Device Plugin + * The ``echo`` sampling parameter together with ``logprobs`` in the ``completions`` endpoint + is now supported. + * Performance has been increased on both CPU and GPU. + * Throughput in high-concurrency scenarios has been increased with dynamic_split_fuse for GPU. + * Testing coverage and stability has been improved. + * The procedure for service deployment and model repository preparation has been simplified. + +* An experimental version of a Windows binary package - native model server for Windows OS - is + available. This release includes a set of limitations and has limited tests coverage. It is + intended for testing, while the production-ready release is expected with 2025.0. All feedback + is welcome. + + +Neural Network Compression Framework +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* The following is now supported: +* A new nncf.data.generate_text_data() method has been added for generating a synthetic dataset + for LLM compression. This approach helps to compress LLMs more accurately in situations when + the dataset is not available or not sufficient. + `See our example `__ + for more information about the usage. +* Support of data-free and data-aware weight compression methods - nncf.compress_weights() - + has been extended with NF4 per-channel quantization, making compressed LLMs more accurate and + faster on NPU. +* Caching of computed statistics in nncf.compress_weights() is now available, significantly + reducing compression time when performing compression of the same LLM multiple times, with + different compression parameters. To enable it, set the advanced ``statistics_path`` parameter + of nncf.compress_weights() to the desired file path location. +* The ``backup_mode`` optional parameter has been added to nncf.compress_weights(), for + specifying the data type for embeddings, convolutions, and last linear layers during 4-bit + weight compression. Available options are INT8_ASYM (default), INT8_SYM, and NONE (retains + the original floating-point precision of the model weights). In certain situations, + non-default value might give better accuracy of compressed LLMs. +* Preview support is now available for optimizing models in Torch + `FX format `__, nncf.quantize(), and + nncf.compress_weights() methods. After optimization such models can be directly executed + via torch.compile(compressed_model, backend="openvino"). For more details, see + `INT8 quantization example `__. +* Memory consumption of data-aware weight compression methods - nncf.compress_weights() ā€“ has + been reduced significantly, with some variation depending on the model and method. +* Support for the following has changed: + + * NumPy 2 added + * PyTorch upgraded to 2.5.1 + * ONNX upgraded to 1.17 + * Python 3.8 discontinued - * Tensor parallel feature for multi-socket CPU inference, with performance improvement for - LLMs with 6B+ parameters (enabled through model_distribution_policy hint configurations). - * RMSNorm operator, optimized with JIT kernel to improve both the 1st and 2nd token - performance of LLMs. -* The following has been improved: - * vLLM support, with PagedAttention exposing attention score as the second output. It can now - be used in the cache eviction algorithm to improve LLM serving performance. - * 1st token performance with Llama series of models, with additional CPU operator optimization - (such as MLP, SDPA) on BF16 precision. - * Default oneTBB version on Linux is now 2021.13.0, improving overall performance on latest - Intel XEON platforms. - * MXFP4 weight compression models (compressing weights to 4-bit with the e2m1 data type - without a zero point and with 8-bit e8m0 scales) have been optimized for Xeon platforms - thanks to fullyconnected compressed weight LLM support. +OpenVINO Tokenizers ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* The following has been fixed: +* Several operations have been introduced and optimized. +* Conversion parameters and environment info have been added to ``rt_info``, improving + reproducibility and debugging. - * Memory leak when ov::num_streams value is 0. - * CPU affinity mask is changed after OpenVINO execution when OpenVINO is compiled - with -DTHREADING=SEQ. -GPU Device Plugin +OpenVINO.GenAI +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -* Dynamic quantization for LLMs is now supported on discrete GPU platforms. -* Stable Diffusion 3 is now supported with good accuracy on Intel GPU platforms. -* Both first and second token latency for LLMs have been improved on Intel GPU platforms. -* The issue of model cache not regenerating with the value changes of - ``ov::hint::performance_mode`` or ``ov::hint::dynamic_quantization_group_size`` has been - fixed. +* The following has been added: + * LoRA adapter for the LLMPipeline. + * Text2ImagePipeline with LoRA adapter and text2image samples. + * VLMPipeline and visual_language_chat sample for text generation models with text and image + inputs. + * WhisperPipeline and whisper_speech_recognition sample. -NPU Device Plugin +* speculative_decoding_lm has been moved to LLMPipeline based implementation and is now + installed as part of the package. +* On NPU, a set of pipelines has been enabled: WhisperPipeline (for whisper-base, + whisper-medium, and whisper-small), LLMPipeline (for Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, + Qwen2-7B-Instruct, and Phi-3 Mini-instruct). Use driver version 32.0.100.3104 or later for + best performance. + + + + + +Other Changes and Known Issues +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +Jupyter Notebooks +----------------------------- + +* `Text-to-Image generation using OpenVINO GenAI `__ +* `Multi LoRA Image Generation `__ +* `Virtual Try-on using OpenVINO and CatVTON `__ +* `Visual Language Assistant using OpenVINO GenAI `__ +* `Speech recognition using OpenVINO GenAI `__ +* `YoloV11 `__ +* `Llama-3.2-vision `__ +* `Pixtral `__ +* `Segment Anything 2 `__ +* `Video Lips-sync using Wav2Lip `__ +* `Convert JAX to OpenVINO tutorial `__ + + +Known Issues +----------------------------- + +| **Component: CPU Plugin** +| ID: 155898 +| Description: +| Description: When using new version of Transformer version to convert some of LLMs + (GPT-J/GPT-NeoX or falcon-7b), the inference accuracy may be impacted on 4th or 5th + generation of IntelĀ® XeonĀ® processors, due to model structure update triggering inference + precision difference in part of the model. The workaround is to use transformer version of + 4.44.2 or lower. + +| **Component: GPU Plugin** +| ID: 154583 +| Description: +| LLM accuracy can be low especially on non-systolic platforms like IntelĀ® Coreā„¢ Ultra. When + facing the low accuracy issue, user needs to manually set a config ACTIVATION_SCALING_FACOTR + with a value of 8.0 in the compile_model() function. From the next release, scaling factor + value will be automatically applied through updated IR. + +| **Component: GenAI** +| ID: 156437, 148933 +| Description: +| When using Python GenAI APIs, if ONNX 17.0 and later is installed, it may encounter the + error ā€œDLL load failed while importing onnx_cpp2py_export: A dynamic link library (DLL) + initialization routine failed.ā€ It is due to the ONNX dependency issue + `onnx/onnx#6267 `__, + Install + `Microsoft Visual C++ Redistributable `__ + latest supported downloads to fix the issue. + +| **Component: GenAI** +| ID: 156944 +| Description: +| There were backward incompatible changes resulting in different text generated by LLMs like + Mistralai/Mistral-7B-Instruct-v0.2 and TinyLlama/TinyLlama-1.1B-Chat-v1.0 when using a + tokenizer converted by older openvino_tolenizers. A way to resolve the issue is to convert + tokenizer and detokenizer models using the latest openvino_tokenizers. + + + + + + + + +Previous 2024 releases ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +.. dropdown:: 2024.4 - 19 September 2024 + :animate: fade-in-slide-down + :color: secondary + + **What's new** + + * More Gen AI coverage and framework integrations to minimize code changes. + + * Support for GLM-4-9B Chat, MiniCPM-1B, Llama 3 and 3.1, Phi-3-Mini, Phi-3-Medium and + YOLOX-s models. + * Noteworthy notebooks added: Florence-2, NuExtract-tiny Structure Extraction, Flux.1 Image + Generation, PixArt-Ī±: Photorealistic Text-to-Image Synthesis, and Phi-3-Vision Visual + Language Assistant. + + * Broader Large Language Model (LLM) support and more model compression techniques. + + * OpenVINOā„¢ runtime optimized for IntelĀ® Xe Matrix Extensions (IntelĀ® XMX) systolic arrays on + built-in GPUs for efficient matrix multiplication resulting in significant LLM performance + boost with improved 1st and 2nd token latency, as well as a smaller memory footprint on + IntelĀ® Coreā„¢ Ultra Processors (Series 2). + * Memory sharing enabled for NPUs on IntelĀ® Coreā„¢ Ultra Processors (Series 2) for efficient + pipeline integration without memory copy overhead. + * Addition of the PagedAttention feature for discrete GPUs* enables a significant boost in + throughput for parallel inferencing when serving LLMs on IntelĀ® Arcā„¢ Graphics or IntelĀ® + Data Center GPU Flex Series. + + * More portability and performance to run AI at the edge, in the cloud, or locally. + + * Support for IntelĀ® Coreā„¢ Ultra Processors Series 2 (formerly codenamed Lunar Lake) on Windows. + * OpenVINOā„¢ Model Server now comes with production-quality support for OpenAI-compatible API + which enables significantly higher throughput for parallel inferencing on IntelĀ® XeonĀ® + processors when serving LLMs to many concurrent users. + * Improved performance and memory consumption with prefix caching, KV cache compression, and + other optimizations for serving LLMs using OpenVINOā„¢ Model Server. + * Support for Python 3.12. + * Support for Red Hat Enterprise Linux (RHEL) version 9.3 - 9.4. + + *Now deprecated* + + * The following will not be available beyond the 2024.4 OpenVINO version: + + * The macOS x86_64 debug bins + * Python 3.8 + * Discrete Keem Bay support + + * IntelĀ® Streaming SIMD Extensions (IntelĀ® SSE) will be supported in source code form, but not + enabled in the binary package by default, starting with OpenVINO 2025.0. + + Check the `deprecation section <#deprecation-and-support>`__ for more information. + + **OpenVINOā„¢ Runtime** + + *Common* + + * Encryption and decryption of topology in model cache is now supported with callback functions + provided by the user (CPU only for now; ov::cache_encryption_callbacks). + * The Ubuntu20 and Ubuntu22 Docker images now include the tokenizers and GenAI CPP modules, + including pre-installed Python modules, in development versions of these images. + * Python 3.12 is now supported. + + *CPU Device Plugin* + + * The following is now supported: + + * Tensor parallel feature for multi-socket CPU inference, with performance improvement for + LLMs with 6B+ parameters (enabled through model_distribution_policy hint configurations). + * RMSNorm operator, optimized with JIT kernel to improve both the 1st and 2nd token + performance of LLMs. + + * The following has been improved: + + * vLLM support, with PagedAttention exposing attention score as the second output. It can now + be used in the cache eviction algorithm to improve LLM serving performance. + * 1st token performance with Llama series of models, with additional CPU operator optimization + (such as MLP, SDPA) on BF16 precision. + * Default oneTBB version on Linux is now 2021.13.0, improving overall performance on latest + IntelĀ® XeonĀ® platforms. + * MXFP4 weight compression models (compressing weights to 4-bit with the e2m1 data type + without a zero point and with 8-bit e8m0 scales) have been optimized for IntelĀ® XeonĀ® + platforms thanks to fullyconnected compressed weight LLM support. + + * The following has been fixed: + + * Memory leak when ov::num_streams value is 0. + * CPU affinity mask is changed after OpenVINO execution when OpenVINO is compiled + with -DTHREADING=SEQ. + + + *GPU Device Plugin* + + * Dynamic quantization for LLMs is now supported on discrete GPU platforms. + * Stable Diffusion 3 is now supported with good accuracy on Intel GPU platforms. + * Both first and second token latency for LLMs have been improved on Intel GPU platforms. + * The issue of model cache not regenerating with the value changes of + ``ov::hint::performance_mode`` or ``ov::hint::dynamic_quantization_group_size`` has been + fixed. + + + *NPU Device Plugin* + * `Remote Tensor API `__ is now supported. * You can now query the available number of tiles (ov::intel_npu::max_tiles) and force a @@ -140,193 +488,178 @@ NPU Device Plugin only during the export method. -OpenVINO Python API -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *OpenVINO Python API* -* Openvino.Tensor, when created in the shared memory mode, now prevents ā€œgarbage collectionā€ of - numpy memory. -* The ``openvino.experimental`` submodule is now available, providing access to experimental - functionalities under development. -* New python-exclusive openvino.Model constructors have been added. -* Image padding in PreProcessor is now available. -* OpenVINO Runtime is now compatible with numpy 2.0. + * Openvino.Tensor, when created in the shared memory mode, now prevents ā€œgarbage collectionā€ of + numpy memory. + * The ``openvino.experimental`` submodule is now available, providing access to experimental + functionalities under development. + * New python-exclusive openvino.Model constructors have been added. + * Image padding in PreProcessor is now available. + * OpenVINO Runtime is now compatible with numpy 2.0. -OpenVINO Node.js API -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *OpenVINO Node.js API* -* The following has been improved + * The following has been improved - * Unit tests for increased efficiency and stability - * Security updates applied to dependencies + * Unit tests for increased efficiency and stability + * Security updates applied to dependencies -* `Electron `__ - compatibility is now confirmed with new end-to-end tests. -* `New API methods `__ added. + * `Electron `__ + compatibility is now confirmed with new end-to-end tests. + * `New API methods `__ added. -TensorFlow Framework Support -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *TensorFlow Framework Support* -* TensorFlow 2.17.0 is now supported. -* JAX 0.4.31 is now supported via a path of jax2tf with native_serialization=False -* `8 NEW* operations `__ - have been added. -* Tensor lists with multiple undefined dimensions in element_shape are now supported, enabling - support for TF Hub lite0-detection/versions/1 model. + * TensorFlow 2.17.0 is now supported. + * JAX 0.4.31 is now supported via a path of jax2tf with native_serialization=False + * `8 NEW* operations `__ + have been added. + * Tensor lists with multiple undefined dimensions in element_shape are now supported, enabling + support for TF Hub lite0-detection/versions/1 model. -PyTorch Framework Support -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *PyTorch Framework Support* -* Torch 2.4 is now supported. -* Inplace ops are now supported automatically if the regular version is supported. -* Symmetric GPTQ model from Hugging Face will now be automatically converted to the signed type - (INT4) and zero-points will be removed. + * Torch 2.4 is now supported. + * Inplace ops are now supported automatically if the regular version is supported. + * Symmetric GPTQ model from Hugging Face will now be automatically converted to the signed type + (INT4) and zero-points will be removed. -ONNX Framework Support -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + *ONNX Framework Support* -* ONNX 1.16.0 is now supported -* models with constants/inputs of uint4/int4 types are now supported. -* 4 NEW operations have been added. + * ONNX 1.16.0 is now supported + * models with constants/inputs of uINT4/INT4 types are now supported. + * 4 NEW operations have been added. -OpenVINO Model Server -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **OpenVINO Model Server** -* OpenAI API for text generation is now officially supported and recommended for production - usage. It comes with the following new features: + * OpenAI API for text generation is now officially supported and recommended for production + usage. It comes with the following new features: - * Prefix caching feature, caching the prompt evaluation to speed up text generation. - * Ability to compress the KV Cache to a lower precision, reducing memory consumption without - a significant loss of accuracy. - * ``stop`` sampling parameters, to define a sequence that stops text generation. - * ``logprobs`` sampling parameter, returning the probabilities to returned tokens. - * Generic metrics related to execution of the MediaPipe graph that can be used for autoscaling - based on the current load and the level of concurrency. - * `Demo of text generation horizontal scalability `__ - using basic docker containers and Kubernetes. - * Automatic cancelling of text generation for disconnected clients. - * Non-UTF-8 responses from the model can be now automatically changed to Unicode replacement - characters, due to their configurable handling. - * Intel GPU with paged attention is now supported. - * Support for Llama3.1 models. + * Prefix caching feature, caching the prompt evaluation to speed up text generation. + * Ability to compress the KV Cache to a lower precision, reducing memory consumption without + a significant loss of accuracy. + * ``stop`` sampling parameters, to define a sequence that stops text generation. + * ``logprobs`` sampling parameter, returning the probabilities to returned tokens. + * Generic metrics related to execution of the MediaPipe graph that can be used for autoscaling + based on the current load and the level of concurrency. + * `Demo of text generation horizontal scalability `__ + using basic docker containers and Kubernetes. + * Automatic cancelling of text generation for disconnected clients. + * Non-UTF-8 responses from the model can be now automatically changed to Unicode replacement + characters, due to their configurable handling. + * Intel GPU with paged attention is now supported. + * Support for Llama3.1 models. -* The following has been improved: + * The following has been improved: - * Handling of model templates without bos_token is now fixed. - * Performance of the multinomial sampling algorithm. - * ``finish_reason`` in the response correctly determines reaching max_tokens (length) and - completing the sequence (stop). - * Security and stability. + * Handling of model templates without bos_token is now fixed. + * Performance of the multinomial sampling algorithm. + * ``finish_reason`` in the response correctly determines reaching max_tokens (length) and + completing the sequence (stop). + * Security and stability. -Neural Network Compression Framework -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **Neural Network Compression Framework** -* The LoRA Correction algorithm is now included in the Weight Compression method, improving the - accuracy of INT4-compressed models on top of other data-aware algorithms, such as AWQ and - Scale Estimation. To enable it, set the lora_correction option to True in - nncf.compress_weights(). -* The GPTQ compression algorithm can now be combined with the Scale Estimation algorithm, - making it possible to run GPTQ, AWQ, and Scale Estimation together, for the optimum-accuracy - INT4-compressed models. -* INT8 quantization of LSTMSequence and Convolution operations for constant inputs is now - enabled, resulting in better performance and reduced model size. + * The LoRA Correction algorithm is now included in the Weight Compression method, improving the + accuracy of INT4-compressed models on top of other data-aware algorithms, such as AWQ and + Scale Estimation. To enable it, set the lora_correction option to True in + nncf.compress_weights(). + * The GPTQ compression algorithm can now be combined with the Scale Estimation algorithm, + making it possible to run GPTQ, AWQ, and Scale Estimation together, for the optimum-accuracy + INT4-compressed models. + * INT8 quantization of LSTMSequence and Convolution operations for constant inputs is now + enabled, resulting in better performance and reduced model size. -OpenVINO Tokenizers -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **OpenVINO Tokenizers** -* Split and BPE tokenization operations have been reimplemented, resulting in improved - tokenization accuracy and performance. -* New building options are now available, offering up to a 12x reduction in binary size. -* An operation is now available to validate and skip/replace model-generated non-Unicode - bytecode sequences during detokenization. + * Split and BPE tokenization operations have been reimplemented, resulting in improved + tokenization accuracy and performance. + * New building options are now available, offering up to a 12x reduction in binary size. + * An operation is now available to validate and skip/replace model-generated non-Unicode + bytecode sequences during detokenization. -OpenVINO.GenAI -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **OpenVINO.GenAI** -* New samples and pipelines are now available: + * New samples and pipelines are now available: - * An example IterableStreamer implementation in - `multinomial_causal_lm/python sample `__ + * An example IterableStreamer implementation in + `multinomial_causal_lm/python sample `__ -* GenAI compilation is now available as part of OpenVINO via the ā€“DOPENVINO_EXTRA_MODULES CMake - option. + * GenAI compilation is now available as part of OpenVINO via the ā€“DOPENVINO_EXTRA_MODULES CMake + option. -Other Changes and Known Issues -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + **Other Changes and Known Issues** -Jupyter Notebooks ------------------------------ + *Jupyter Notebooks* -* `Florence-2 `__ -* `NuExtract: Structure Extraction `__ -* `Flux.1 Image Generation `__ -* `PixArt-Ī±: Photorealistic Text-to-Image Synthesis `__ -* `Phi-3-Vision Visual Language Assistant `__ -* `MiniCPMV2.6 `__ -* `InternVL2 `__ -* The list of supported models in - `LLM chatbot `__ - now includes Phi3.5, Gemma2 support + * `Florence-2 `__ + * `NuExtract: Structure Extraction `__ + * `Flux.1 Image Generation `__ + * `PixArt-Ī±: Photorealistic Text-to-Image Synthesis `__ + * `Phi-3-Vision Visual Language Assistant `__ + * `MiniCPMV2.6 `__ + * `InternVL2 `__ + * The list of supported models in + `LLM chatbot `__ + now includes Phi3.5, Gemma2 support -Known Issues ------------------------------ + *Known Issues* -| **Component: CPU** -| ID: CVS-150542, CVS-145996 -| Description: -| The upgrade of default oneTBB on Linux platforms to 2021.13.0 improves overall - performance on latest Intel XEON platform but causes regression in some cases. Limit the - threads usage of postprocessing done by Torch can mitigate the regression (For example: - torch.set_num_threads(n), n can be 1, beam search number, prompt batch size or other - numbers). - -| **Component: OpenVINO.Genai** -| ID: 149694 -| Description: -| Passing openvino.Tensor instance to LLMPipleine triggers incompatible arguments error if - OpenVINO and GenAI are installed from PyPI on Windows. + | **Component: CPU** + | ID: CVS-150542, CVS-145996 + | Description: + | The upgrade of default oneTBB on Linux platforms to 2021.13.0 improves overall + performance on latest IntelĀ® XeonĀ® platform but causes regression in some cases. Limit the + threads usage of postprocessing done by Torch can mitigate the regression (For example: + torch.set_num_threads(n), n can be 1, beam search number, prompt batch size or other + numbers). + + | **Component: OpenVINO.Genai** + | ID: 149694 + | Description: + | Passing openvino.Tensor instance to LLMPipleine triggers incompatible arguments error if + OpenVINO and GenAI are installed from PyPI on Windows. -| **Component: OpenVINO.Genai** -| ID: 148308 -| Description: -| OpenVINO.GenAI archive doesn't have debug libraries for OpenVINO Tokenizers and - OpenVINO.GenAI. + | **Component: OpenVINO.Genai** + | ID: 148308 + | Description: + | OpenVINO.GenAI archive doesn't have debug libraries for OpenVINO Tokenizers and + OpenVINO.GenAI. + + | **Component: ONNX for ARM** + | ID: n/a + | Description: + | For ARM binaries, the `1.16 ONNX library `__ + is not yet available. The ONNX library for ARM, version 1.15, does not include the latest + functional and security updates. Users should update to the latest version as it becomes + available. + | Currently, if an unverified AI model is supplied to the ONNX frontend, it could lead to a + directory traversal issue. Ensure that the file name and file path that a model contains + are verified and correct. To learn more about the vulnerability, see: + `CVE-2024-27318 `__ and + `CVE-2024-27319 `__. + + | **Component: Kaldi** + | ID: n/a + | Description: + | There is a known issue with the Kaldi DL framework support on the Python version 3.12 due + to the numpy version incompatibilities. As Kaldi support in OpenVINO is currently deprecated + and will be discontinued with version 2025.0, the issue will not be addressed. -| **Component: ONNX for ARM** -| ID: n/a -| Description: -| For ARM binaries, the `1.16 ONNX library `__ - is not yet available. The ONNX library for ARM, version 1.15, does not include the latest - functional and security updates. Users should update to the latest version as it becomes - available. -| Currently, if an unverified AI model is supplied to the ONNX frontend, it could lead to a - directory traversal issue. Ensure that the file name and file path that a model contains - are verified and correct. To learn more about the vulnerability, see: - `CVE-2024-27318 `__ and - `CVE-2024-27319 `__. - -| **Component: Kaldi** -| ID: n/a -| Description: -| There is a known issue with the Kaldi DL framework support on the Python version 3.12 due - to the numpy version incompatibilities. As Kaldi support in OpenVINO is currently deprecated - and will be discontinued with version 2025.0, the issue will not be addressed. -Previous 2024 releases -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. dropdown:: 2024.3 - 31 July 2024 :animate: fade-in-slide-down @@ -614,7 +947,7 @@ Previous 2024 releases * Preview: addition of the :doc:`Generate API <../learn-openvino/llm_inference_guide/genai-guide>`, a simplified API for text generation using large language models with only a few lines of code. The API is available through the newly launched OpenVINO GenAI package. - * Support for Intel AtomĀ® Processor X Series. For more details, see :doc:`System Requirements <./release-notes-openvino/system-requirements>`. + * Support for IntelĀ® AtomĀ® Processor X Series. For more details, see :doc:`System Requirements <./release-notes-openvino/system-requirements>`. * Preview: Support for IntelĀ® XeonĀ® 6 processor. **OpenVINOā„¢ Runtime** @@ -638,8 +971,8 @@ Previous 2024 releases *CPU Device Plugin* * Performance when using latency mode in FP32 precision has been improved on Intel client - platforms, including Core Ultra (codename Meteor Lake) and 13th Gen Core processors - (codename Raptor Lake). + platforms, including IntelĀ® Coreā„¢ Ultra (formerly codenamed Meteor Lake) and 13th Gen Core + processors (formerly codenamed Raptor Lake). * 2nd token latency and memory footprint for FP16 LLMs have been improved significantly on AVX2 and AVX512 based CPU platforms, particularly for small batch sizes. * PagedAttention has been optimized on AVX2, AVX512 and AMX platforms together with INT8 KV cache @@ -653,9 +986,9 @@ Previous 2024 releases * Both first token and average token latency of LLMs is improved on all GPU platforms, most significantly on discrete GPUs. Memory usage of LLMs has been reduced as well. - * Stable Diffusion FP16 performance improved on Core Ultra platforms, with significant pipeline - improvement for models with dynamic-shaped input. Memory usage of the pipeline has been reduced, - as well. + * Stable Diffusion FP16 performance improved on IntelĀ® Coreā„¢ Ultra platforms, with significant + pipeline improvement for models with dynamic-shaped input. Memory usage of the pipeline + has been reduced, as well. * Optimized permute_f_y kernel performance has been improved. *NPU Device Plugin* @@ -710,7 +1043,7 @@ Previous 2024 releases * OpenVINO Model server can be now used for text generation use cases using OpenAI compatible API. * Added support for continuous batching and PagedAttention algorithms for text generation with - fast and efficient in high concurrency load especially on Intel Xeon processors. + fast and efficient in high concurrency load especially on IntelĀ® XeonĀ® processors. `Learn more about it `__. **Neural Network Compression Framework** @@ -753,8 +1086,9 @@ Previous 2024 releases | Description: | In 2024.2, oneTBB 2021.2.x is used for Intel Distribution of OpenVINO Ubuntu and Red Hat archives, instead of system TBB/oneTBB. This improves performance on the new generation of - Xeon platforms but may increase latency of some models on the previous generation. You can - build OpenVINO with **-DSYSTEM_TBB=ON** to get better latency performance for these models. + IntelĀ® XeonĀ® platforms but may increase latency of some models on the previous generation. + You can build OpenVINO with **-DSYSTEM_TBB=ON** to get better latency performance for + these models. | **Component: python API** | ID: CVS-141744 @@ -943,7 +1277,7 @@ Previous 2024 releases deployed in an arbitrary path without any code changes. * KServe REST API support has been extended to properly handle the string format in JSON body, just like the binary format compatible with NVIDIA Tritonā„¢. - * `A demo showcasing a full RAG algorithm `__ + * `A demo showcasing a full RAG algorithm `__ fully delegated to the model server has been added. **Neural Network Compression Framework** @@ -1000,7 +1334,7 @@ Previous 2024 releases * `RMBG background removal `__ * `AnimateAnyone: pose guided image to video generation `__ * `LLaVA-Next visual-language assistant `__ - * `TripoSR: single image 3d reconstruction `__ + * `TripoSR: single image 3d reconstruction `__ * `RAG system with OpenVINO and LangChain `__ *Known Issues* @@ -1069,8 +1403,8 @@ Previous 2024 releases * More portability and performance to run AI at the edge, in the cloud, or locally. * A preview plugin architecture of the integrated Neural Processor Unit (NPU) as part of - IntelĀ® Coreā„¢ Ultra processor (codename Meteor Lake) is now included in the main OpenVINOā„¢ - package on PyPI. + IntelĀ® Coreā„¢ Ultra processor (formerly codenamed Meteor Lake) is now included in the + main OpenVINOā„¢ package on PyPI. * Improved performance on ARM by enabling the ARM threading library. In addition, we now support multi-core ARM processors and enabled FP16 precision by default on MacOS. * New and improved LLM serving samples from OpenVINO Model Server for multi-batch inputs and @@ -1263,10 +1597,11 @@ Previous 2024 releases | **Component: CPU runtime** | *ID:* N/A | *Description:* - | Performance results (first token latency) may vary from those offered by the previous OpenVINO version, for - ā€œlatencyā€ hint inference of LLMs with long prompts on Xeon platforms with 2 or more - sockets. The reason is that all CPU cores of just the single socket running the application - are employed, lowering the memory overhead for LLMs when numa control is not used. + | Performance results (first token latency) may vary from those offered by the previous + OpenVINO version, for ā€œlatencyā€ hint inference of LLMs with long prompts on IntelĀ® XeonĀ® + platforms with 2 or more sockets. The reason is that all CPU cores of just the single + socket running the application are employed, lowering the memory overhead for LLMs when + numa control is not used. | *Workaround:* | The behavior is expected but stream and thread configuration may be used to include cores from all sockets. @@ -1309,28 +1644,32 @@ Discontinued in 2024 * `Accuracy Checker `__. * `Post-Training Optimization Tool `__ (POT). Neural Network Compression Framework (NNCF) should be used instead. - * A `Git patch `__ + * A `Git patch `__ for NNCF integration with `huggingface/transformers `__. The recommended approach is to use `huggingface/optimum-intel `__ for applying NNCF optimization on top of models from Hugging Face. * Support for Apache MXNet, Caffe, and Kaldi model formats. Conversion to ONNX may be used as a solution. + * The macOS x86_64 debug bins are no longer provided with the OpenVINO toolkit, starting + with OpenVINO 2024.5. + * Python 3.8 is no longer supported, starting with OpenVINO 2024.5. -Deprecated and to be removed in the future --------------------------------------------- + * As MxNet doesn't support Python version higher than 3.8, according to the + `MxNet PyPI project `__, + it is no longer supported by OpenVINO, either. -* The macOS x86_64 debug bins will no longer be provided with the OpenVINO toolkit, starting - with OpenVINO 2024.5. -* Python 3.8 is now considered deprecated, and it will not be available beyond the 2024.4 - OpenVINO version. + * Discrete Keem Bay support is no longer supported, starting with OpenVINO 2024.5. + * Support for discrete devices (formerly codenamed Raptor Lake) is no longer available for + NPU. - * As MxNet doesn't support Python version higher than 3.8, according to the - `MxNet PyPI project `__, - it will no longer be supported in future versions, either. -* Discrete Keem Bay support is now considered deprecated and will be fully removed with OpenVINO 2024.5 +Deprecated and to be removed in the future +-------------------------------------------- + * IntelĀ® Streaming SIMD Extensions (IntelĀ® SSE) will be supported in source code form, but not - enabled in the binary package by default, starting with OpenVINO 2025.0 + enabled in the binary package by default, starting with OpenVINO 2025.0. +* Ubuntu 20.04 support will be deprecated in future OpenVINO releases due to the end of + standard support. * The openvino-nightly PyPI module will soon be discontinued. End-users should proceed with the Simple PyPI nightly repo instead. More information in `Release Policy `__. @@ -1360,25 +1699,25 @@ Deprecated and to be removed in the future * See alternative: `PaddleOCR with OpenVINOā„¢ `__, * See alternative: `Handwritten Text Recognition Demo `__ - * `Image In-painting with OpenVINOā„¢ `__ + * `Image In-painting with OpenVINOā„¢ `__ * See alternative: `Image Inpainting Python Demo `__ - * `Interactive Machine Translation with OpenVINO `__ + * `Interactive Machine Translation with OpenVINO `__ * See alternative: `Machine Translation Python* Demo `__ - * `Open Model Zoo Tools Tutorial `__ + * `Open Model Zoo Tools Tutorial `__ * No alternatives, demonstrates deprecated tools. - * `Super Resolution with OpenVINOā„¢ `__ + * `Super Resolution with OpenVINOā„¢ `__ * See alternative: `Super Resolution with PaddleGAN and OpenVINO `__ * See alternative: `Image Processing C++ Demo `__ - * `Image Colorization with OpenVINO Tutorial `__ - * `Interactive Question Answering with OpenVINOā„¢ `__ + * `Image Colorization with OpenVINO Tutorial `__ + * `Interactive Question Answering with OpenVINOā„¢ `__ * See alternative: `BERT Question Answering Embedding Python* Demo `__ * See alternative: `BERT Question Answering Python* Demo `__ @@ -1387,37 +1726,37 @@ Deprecated and to be removed in the future * See alternative: `Security Barrier Camera C++ Demo `__ - * `The attention center model with OpenVINOā„¢ `_ - * `Image Generation with DeciDiffusion `_ - * `Image generation with DeepFloyd IF and OpenVINOā„¢ `_ - * `Depth estimation using VI-depth with OpenVINOā„¢ `_ + * `The attention center model with OpenVINOā„¢ `_ + * `Image Generation with DeciDiffusion `_ + * `Image generation with DeepFloyd IF and OpenVINOā„¢ `_ + * `Depth estimation using VI-depth with OpenVINOā„¢ `_ * `Instruction following using Databricks Dolly 2.0 and OpenVINOā„¢ `_ * See alternative: `LLM Instruction-following pipeline with OpenVINO `__ - * `Image generation with FastComposer and OpenVINOā„¢ `__ + * `Image generation with FastComposer and OpenVINOā„¢ `__ * `Video Subtitle Generation with OpenAI Whisper `__ * See alternative: `Automatic speech recognition using Distil-Whisper and OpenVINO `__ - * `Introduction to Performance Tricks in OpenVINOā„¢ `__ - * `Speaker Diarization with OpenVINOā„¢ `__ - * `Subject-driven image generation and editing using BLIP Diffusion and OpenVINO `__ - * `Text Prediction with OpenVINOā„¢ `__ - * `Training to Deployment with TensorFlow and OpenVINOā„¢ `__ - * `Speech to Text with OpenVINOā„¢ `__ - * `Convert and Optimize YOLOv7 with OpenVINOā„¢ `__ - * `Quantize Data2Vec Speech Recognition Model using NNCF PTQ API `__ + * `Introduction to Performance Tricks in OpenVINOā„¢ `__ + * `Speaker Diarization with OpenVINOā„¢ `__ + * `Subject-driven image generation and editing using BLIP Diffusion and OpenVINO `__ + * `Text Prediction with OpenVINOā„¢ `__ + * `Training to Deployment with TensorFlow and OpenVINOā„¢ `__ + * `Speech to Text with OpenVINOā„¢ `__ + * `Convert and Optimize YOLOv7 with OpenVINOā„¢ `__ + * `Quantize Data2Vec Speech Recognition Model using NNCF PTQ API `__ * See alternative: `Quantize Speech Recognition Models with accuracy control using NNCF PTQ API `__ - * `Semantic segmentation with LRASPP MobileNet v3 and OpenVINO `__ - * `Video Recognition using SlowFast and OpenVINOā„¢ `__ + * `Semantic segmentation with LRASPP MobileNet v3 and OpenVINO `__ + * `Video Recognition using SlowFast and OpenVINOā„¢ `__ * See alternative: `Live Action Recognition with OpenVINOā„¢ `__ - * `Semantic Segmentation with OpenVINOā„¢ using Segmenter `__ - * `Programming Language Classification with OpenVINO `__ + * `Semantic Segmentation with OpenVINOā„¢ using Segmenter `__ + * `Programming Language Classification with OpenVINO `__ * `Stable Diffusion Text-to-Image Demo `__ * See alternative: `Stable Diffusion v2.1 using Optimum-Intel OpenVINO and multiple Intel Hardware `__ @@ -1426,10 +1765,10 @@ Deprecated and to be removed in the future * See alternative: `Stable Diffusion v2.1 using Optimum-Intel OpenVINO and multiple Intel Hardware `__ - * `Image generation with Segmind Stable Diffusion 1B (SSD-1B) model and OpenVINO `__ - * `Data Preparation for 2D Medical Imaging `__ - * `Train a Kidney Segmentation Model with MONAI and PyTorch Lightning `__ - * `Live Inference and Benchmark CT-scan Data with OpenVINOā„¢ `__ + * `Image generation with Segmind Stable Diffusion 1B (SSD-1B) model and OpenVINO `__ + * `Data Preparation for 2D Medical Imaging `__ + * `Train a Kidney Segmentation Model with MONAI and PyTorch Lightning `__ + * `Live Inference and Benchmark CT-scan Data with OpenVINOā„¢ `__ * See alternative: `Quantize a Segmentation Model and Show Live Inference `__ @@ -1458,7 +1797,7 @@ are available on request. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Learn more at -`www.intel.com `__ +`www.intel.com `__ or from the OEM or retailer. No computer system can be absolutely secure. diff --git a/docs/articles_en/documentation/legacy-features.rst b/docs/articles_en/documentation/legacy-features.rst index f859a3a4572f88..2457d28cf24c15 100644 --- a/docs/articles_en/documentation/legacy-features.rst +++ b/docs/articles_en/documentation/legacy-features.rst @@ -96,7 +96,7 @@ Discontinued: | *New solution:* API 2.0 launched in OpenVINO 2022.1 | *Old solution:* discontinued with OpenVINO 2024.0 - | `The last version supporting API 1.0 `__ + | `2023.2 is the last version supporting API 1.0 `__ .. dropdown:: Compile tool diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst index b5d3c08b39f480..fb9f41c755d4fb 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats.rst @@ -120,7 +120,7 @@ Here are code examples of how to use these methods with different model formats: For more details on conversion, refer to the :doc:`guide <[legacy]-supported-model-formats/[legacy]-convert-tensorflow>` - and an example `tutorial `__ + and an example `tutorial `__ on this topic. * The ``read_model()`` and ``compile_model()`` methods: @@ -592,7 +592,7 @@ to OpenVINO IR or ONNX before running inference should be considered the default OpenVINO versions of 2023 are mostly compatible with the old instructions, through a deprecated MO tool, installed with the deprecated OpenVINO Developer Tools package. - `OpenVINO 2023.0 `__ is the last + `OpenVINO 2023.0 `__ is the last release officially supporting the MO conversion process for the legacy formats. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-faster-r-cnn.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-faster-r-cnn.rst index 711a060b7467b8..7880b261c80b81 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-faster-r-cnn.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-faster-r-cnn.rst @@ -14,7 +14,7 @@ Converting an ONNX Faster R-CNN Model The instructions below are applicable **only** to the Faster R-CNN model converted to the ONNX file format from the `maskrcnn-benchmark model `__: -1. Download the pretrained model file from `onnx/models `__ (commit-SHA: 8883e49e68de7b43e263d56b9ed156dfa1e03117). +1. Download the pretrained model file from `onnx/models `__ (commit-SHA: 8883e49e68de7b43e263d56b9ed156dfa1e03117). 2. Generate the Intermediate Representation of the model, by changing your current working directory to the model conversion API installation directory, and running model conversion with the following parameters: diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-gpt-2.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-gpt-2.rst index 84392e92e620d2..4c10c941c7fb47 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-gpt-2.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-onnx-gpt-2.rst @@ -12,7 +12,7 @@ Converting an ONNX GPT-2 Model This guide describes a deprecated conversion method. The guide on the new and recommended method can be found in the :doc:`Python tutorials <../../../../../../learn-openvino/interactive-tutorials-python>`. -`Public pre-trained GPT-2 model `__ is a large +`Public pre-trained GPT-2 model `__ is a large transformer-based language model with a simple objective: predict the next word, given all of the previous words within some text. Downloading the Pre-Trained Base GPT-2 Model diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-quartz-net.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-quartz-net.rst index de3af8ce5175f0..f1ee885dae0b26 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-quartz-net.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-quartz-net.rst @@ -20,7 +20,7 @@ Downloading the Pre-trained QuartzNet Model To download the pre-trained model, refer to the `NeMo Speech Models Catalog `__. Here are the instructions on how to obtain QuartzNet in ONNX format. -1. Install the NeMo toolkit, using the `instructions `__. +1. Install the NeMo toolkit, using the `instructions `__. 2. Run the following code: diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-rnn-t.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-rnn-t.rst index 4f33e510a40267..ad646568aed598 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-rnn-t.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-pytorch-rnn-t.rst @@ -44,7 +44,7 @@ For UNIX-like systems, you can use ``wget``: The link was taken from ``setup.sh`` in the ``speech_recoginitin/rnnt`` subfolder. You will get exactly the same weights as -if you were following the `guide `__. +if you were following the `guide `__. **Step 4**. Install required Python packages: diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-convert-tensorflow.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-convert-tensorflow.rst index 955d5418d37270..2bcb6fde9b833b 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-convert-tensorflow.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-convert-tensorflow.rst @@ -59,7 +59,7 @@ To convert such TensorFlow model, run the `mo` script with a path to the MetaGra 3. **SavedModel format**. In this case, a model consists of a special directory with a ``.pb`` file -and several subfolders: ``variables``, ``assets``, and ``assets.extra``. For more information about the SavedModel directory, refer to the `README `__ file in the TensorFlow repository. +and several subfolders: ``variables``, ``assets``, and ``assets.extra``. For more information about the SavedModel directory, refer to the `README `__ file in the TensorFlow repository. To convert such TensorFlow model, run the ``mo`` script with a path to the SavedModel directory: .. code-block:: sh diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility.rst index fc78b12640771a..3d2365f45ffe3b 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility.rst @@ -160,7 +160,7 @@ It is important to mention that sometimes it seems like transformation cannot be because the actual values of inputs or shapes are needed. In fact, manipulations of shapes or values can be implemented using operations that are added to the graph. Consider the ``extensions/front/onnx/flattenONNX_to_reshape.py`` transformation, which replaces an ONNX -`Flatten `__ operation with a sub-graph of operations performing +`Flatten `__ operation with a sub-graph of operations performing the following (when ``axis`` is not equal to 0 and 1): 1. Calculate a shape of the ``Flatten`` input tensor, using the :doc:`ShapeOf <../../openvino-ir-format/operation-sets/operation-specs/shape/shape-of-3>` operation. diff --git a/docs/articles_en/documentation/openvino-ecosystem/openvino-security-add-on.rst b/docs/articles_en/documentation/openvino-ecosystem/openvino-security-add-on.rst index 2d5598a5eb8e9d..3959ebefb09a4a 100644 --- a/docs/articles_en/documentation/openvino-ecosystem/openvino-security-add-on.rst +++ b/docs/articles_en/documentation/openvino-ecosystem/openvino-security-add-on.rst @@ -580,7 +580,7 @@ Building OpenVINOā„¢ Security Add-on depends on OpenVINOā„¢ Model Server docker 1. Download the `OpenVINOā„¢ Model Server software `__ -2. Build the `OpenVINOā„¢ Model Server Docker images `__ +2. Build the `OpenVINOā„¢ Model Server Docker images `__ .. code-block:: sh diff --git a/docs/articles_en/documentation/openvino-ecosystem/openvino-training-extensions.rst b/docs/articles_en/documentation/openvino-ecosystem/openvino-training-extensions.rst index a7a81acd9ba3a7..8a5bd91f9c1b7b 100644 --- a/docs/articles_en/documentation/openvino-ecosystem/openvino-training-extensions.rst +++ b/docs/articles_en/documentation/openvino-ecosystem/openvino-training-extensions.rst @@ -32,9 +32,9 @@ If the results are unsatisfactory, add datasets and perform the same steps, star OpenVINO Training Extensions Components ####################################### -* `OpenVINO Training Extensions API `__ +* `OpenVINO Training Extensions API `__ * `OpenVINO Training Extensions CLI `__ -* `OpenVINO Training Extensions Algorithms `__ +* `OpenVINO Training Extensions Algorithms `__ Tutorials ######### diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations.rst index 6ba9e0a9b60f52..9451fabd6219d8 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations.rst @@ -35,7 +35,7 @@ The goal of Low Precision Transformations (LPT) is to transform a quantized mode As result, operation input tensor precisions will be changed from original to low precision and operations can be inferred by OpenVINOā„¢ plugin in low precision. -For a more detailed description on how to quantize a model, see the `Low precision tools <#low-precision-tools>`__ section below. For more information about model quantization, refer to **Brief History of Lower Precision in Deep Learning** section in `this whitepaper `__. +For a more detailed description on how to quantize a model, see the `Low precision tools <#low-precision-tools>`__ section below. For more information about model quantization, refer to **Brief History of Lower Precision in Deep Learning** section in `this whitepaper `__. Input model requirements ######################## diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/activation/identity-16.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/activation/identity-16.rst new file mode 100644 index 00000000000000..19a40ecfb86ba9 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/activation/identity-16.rst @@ -0,0 +1,48 @@ +Identity +======== + + +.. meta:: + :description: Learn about Identity-16 - a simple operation that forwards the input to the output. + +**Versioned name**: *Identity-16* + +**Category**: *Activation* + +**Short description**: The *Identity* operation forwards the input to the output. + +**Detailed description**: The *Identity* operation generates a new tensor that mirrors the input tensor in shape, data type, and content, effectively implementing the linear activation function f(x) = x. +If the input and output tensor data address is the same, input is returned as output instead. + +**Input**: + +* **1**: `input` - A tensor of any shape and type `T`. **Required.** + +**Output**: + +* **1**: `output` - A tensor with the same shape and type `T` as the input, containing the same data as the input. + +**Types** + +* **T**: any supported data type. + +*Example 1: 2D input matrix.* + +.. code-block:: xml + :force: + + + + + + 3 + 3 + + + + + 3 + 3 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst index 5cc1b024f158b1..f02c5414ac4369 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst @@ -11,7 +11,7 @@ Loop **Category**: *Infrastructure* **Short description**: *Loop* operation performs recurrent execution of the network, which is described in the ``body``, iterating through the data. -The operation has similar semantic to the ONNX Loop `operation `__. +The operation has similar semantic to the ONNX Loop `operation `__. **Detailed description** @@ -73,7 +73,7 @@ Loop operation description in the IR also has several special sections: ``body`` 1. The body operation getting an input from the main graph should have an entry in the ``port_map`` section of the Loop operation. These edges connect input ports of the Loop with the body ``Parameter``\ s. 2. Input tensors to the Loop can be sliced along a specified axis, the Loop can iterates over all sliced parts. The corresponding ``input`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to slice. Therefore, inputs to the Loop operation corresponding to ``input`` entries in the ``port_map`` without ``axis`` attribute are used "as is" (without slicing). 3. The body operation producing tensor to be used in the subsequent iterations (like in RNN models) should have a back edge described in the ``back_edges`` section of the operation. The back edge connects the respective body ``Parameter`` and ``Result`` operations. For such a case the Loop operation node provides input for the first iteration, while corresponding Loop operation output produces the tensor computed during the last iteration. -4. Output tensors produced by a particular body operation across all iterations can be concatenated and returned as a Loop operation output (this is a "scan output" according to the ONNX* Loop operation `specification `__ ). The corresponding ``output`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to concatenate. Therefore, outputs from operations corresponding to ``output`` entries in the ``port_map`` without ``axis`` attribute are returned "as is" (without concatenation). +4. Output tensors produced by a particular body operation across all iterations can be concatenated and returned as a Loop operation output (this is a "scan output" according to the ONNX* Loop operation `specification `__ ). The corresponding ``output`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to concatenate. Therefore, outputs from operations corresponding to ``output`` entries in the ``port_map`` without ``axis`` attribute are returned "as is" (without concatenation). 5. There is one body ``Parameter`` operation not connected through the ``port_map``. This is a "current iteration" input. The Loop operation is responsible for providing the appropriate value for each iteration. 6. Connection of nodes inside the Loop body with the main graph should be done through ``Parameter`` and ``Result`` body operations. No other ways to connect graphs are allowed. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-cell-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-cell-3.rst index 28dbec46289f89..f58418ee923a8b 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-cell-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-cell-3.rst @@ -64,7 +64,7 @@ GRUCell * *linear_before_reset* * **Description**: *linear_before_reset* flag denotes if the layer behaves according to the modification - of *GRUCell* described in the formula in the `ONNX documentation `__. + of *GRUCell* described in the formula in the `ONNX documentation `__. * **Range of values**: true or false * **Type**: ``boolean`` * **Default value**: false diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-sequence-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-sequence-5.rst index 37c70087e121ea..f9b9a5ece850ec 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-sequence-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/gru-sequence-5.rst @@ -19,7 +19,7 @@ represents a sequence of GRU cells. The sequence can be connected differently de ``direction`` attribute that specifies the direction of traversing of input data along sequence dimension or specifies whether it should be a bidirectional sequence. The most of the attributes are in sync with the specification of ONNX GRU operator defined -`GRUCell `__ +`GRUCell `__ **Attributes** @@ -69,7 +69,7 @@ are in sync with the specification of ONNX GRU operator defined * *linear_before_reset* * **Description**: *linear_before_reset* flag denotes if the layer behaves according to the modification - of *GRUCell* described in the formula in the `ONNX documentation `__. + of *GRUCell* described in the formula in the `ONNX documentation `__. * **Range of values**: True or False * **Type**: ``boolean`` * **Default value**: False diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst index c00b4c819cc66a..abad632e5ae86c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst @@ -14,7 +14,7 @@ LSTMSequence **Detailed description** -A single cell in the sequence is implemented in the same way as in :doc:`LSTM Cell ` operation. *LSTMSequence* represents a sequence of LSTM cells. The sequence can be connected differently depending on ``direction`` attribute that specifies the direction of traversing of input data along sequence dimension or specifies whether it should be a bidirectional sequence. The most of the attributes are in sync with the specification of ONNX LSTM operator defined `LSTMCell `__ . +A single cell in the sequence is implemented in the same way as in :doc:`LSTM Cell ` operation. *LSTMSequence* represents a sequence of LSTM cells. The sequence can be connected differently depending on ``direction`` attribute that specifies the direction of traversing of input data along sequence dimension or specifies whether it should be a bidirectional sequence. The most of the attributes are in sync with the specification of ONNX LSTM operator defined `LSTMCell `__ . **Attributes** @@ -26,13 +26,12 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce * **Type**: ``int`` * **Required**: *yes* -* *activations* +* *direction* - * **Description**: *activations* specifies activation functions for gates, there are three gates, so three activation functions should be specified as a value for this attributes - * **Range of values**: any combination of *relu*, *sigmoid*, *tanh* - * **Type**: a list of strings - * **Default value**: *sigmoid,tanh,tanh* - * **Required**: *no* + * **Description**: Specify if the RNN is forward, reverse, or bidirectional. If it is one of *forward* or *reverse* then ``num_directions = 1``, if it is *bidirectional*, then ``num_directions = 2``. This ``num_directions`` value specifies input/output shape requirements. + * **Range of values**: *forward*, *reverse*, *bidirectional* + * **Type**: ``string`` + * **Required**: *yes* * *activations_alpha, activations_beta* @@ -42,6 +41,14 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce * **Default value**: None * **Required**: *no* +* *activations* + + * **Description**: *activations* specifies activation functions for gates, there are three gates, so three activation functions should be specified as a value for this attributes + * **Range of values**: any combination of *relu*, *sigmoid*, *tanh* + * **Type**: a list of strings + * **Default value**: *sigmoid,tanh,tanh* + * **Required**: *no* + * *clip* * **Description**: *clip* specifies bound values *[-C, C]* for tensor clipping. Clipping is performed before activations. @@ -50,12 +57,6 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce * **Default value**: *infinity* that means that the clipping is not applied * **Required**: *no* -* *direction* - - * **Description**: Specify if the RNN is forward, reverse, or bidirectional. If it is one of *forward* or *reverse* then ``num_directions = 1``, if it is *bidirectional*, then ``num_directions = 2``. This ``num_directions`` value specifies input/output shape requirements. - * **Range of values**: *forward*, *reverse*, *bidirectional* - * **Type**: ``string`` - * **Required**: *yes* **Inputs** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/rnn-sequence-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/rnn-sequence-5.rst index fc9829dd999bda..a3dfc062de2dcd 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/rnn-sequence-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/rnn-sequence-5.rst @@ -14,7 +14,7 @@ RNNSequence **Detailed description** -A single cell in the sequence is implemented in the same way as in :doc:`RNNCell ` operation. *RNNSequence* represents a sequence of RNN cells. The sequence can be connected differently depending on `direction` attribute that specifies the direction of traversing of input data along sequence dimension or specifies whether it should be a bidirectional sequence. The most of the attributes are in sync with the specification of ONNX RNN operator defined `RNNCell `__. +A single cell in the sequence is implemented in the same way as in :doc:`RNNCell ` operation. *RNNSequence* represents a sequence of RNN cells. The sequence can be connected differently depending on `direction` attribute that specifies the direction of traversing of input data along sequence dimension or specifies whether it should be a bidirectional sequence. The most of the attributes are in sync with the specification of ONNX RNN operator defined `RNNCell `__. **Attributes** diff --git a/docs/articles_en/get-started/configurations/configurations-intel-gpu.rst b/docs/articles_en/get-started/configurations/configurations-intel-gpu.rst index dc43881780b1e6..e10a67fddadb53 100644 --- a/docs/articles_en/get-started/configurations/configurations-intel-gpu.rst +++ b/docs/articles_en/get-started/configurations/configurations-intel-gpu.rst @@ -37,7 +37,7 @@ Below are the instructions on how to install the OpenCL packages on supported Li and install the apt package `ocl-icd-libopencl1` with the OpenCl ICD loader. Alternatively, you can add the apt repository by following the - `installation guide `__. + `installation guide `__. Then install the `ocl-icd-libopencl1`, `intel-opencl-icd`, `intel-level-zero-gpu` and `level-zero` apt packages: diff --git a/docs/articles_en/get-started/configurations/genai-dependencies.rst b/docs/articles_en/get-started/configurations/genai-dependencies.rst index e347c11953fbc2..59d29ef3108da0 100644 --- a/docs/articles_en/get-started/configurations/genai-dependencies.rst +++ b/docs/articles_en/get-started/configurations/genai-dependencies.rst @@ -4,8 +4,8 @@ OpenVINOā„¢ GenAI Dependencies OpenVINOā„¢ GenAI depends on both `OpenVINO `__ and `OpenVINO Tokenizers `__. During OpenVINOā„¢ GenAI installation from PyPi, the same versions of OpenVINO and OpenVINO Tokenizers -are used (e.g. ``openvino==2024.4.0`` and ``openvino-tokenizers==2024.4.0.0`` are installed for -``openvino-genai==2024.4.0``). +are used (e.g. ``openvino==2024.5.0`` and ``openvino-tokenizers==2024.5.0.0`` are installed for +``openvino-genai==2024.5.0``). Trying to update any of the dependency packages might result in a version incompatiblibty due to different Application Binary Interfaces (ABIs), which will result in errors while running diff --git a/docs/articles_en/get-started/install-openvino.rst b/docs/articles_en/get-started/install-openvino.rst index 7f26ab9ec72c9f..be00804faa01d2 100644 --- a/docs/articles_en/get-started/install-openvino.rst +++ b/docs/articles_en/get-started/install-openvino.rst @@ -1,4 +1,4 @@ -Install OpenVINOā„¢ 2024.4 +Install OpenVINOā„¢ 2024.5 ========================== @@ -19,16 +19,16 @@ Install OpenVINOā„¢ 2024.4 .. raw:: html - + - + -OpenVINO 2024.4, described here, is not a Long-Term-Support version! +OpenVINO 2024.5, described here, is not a Long-Term-Support version! All currently supported versions are: -* 2024.4 (development) +* 2024.5 (development) * 2023.3 (LTS) -* 2022.3 (LTS) + .. dropdown:: Effortless GenAI integration with OpenVINO GenAI Flavor diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst index 21a35a14047dca..d03fa8fe8b0dc5 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst @@ -37,7 +37,7 @@ Step 1: Set Up the OpenVINO Toolkit APT Repository .. code-block:: sh - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo gpg --output /etc/apt/trusted.gpg.d/intel.gpg --dearmor GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst index e777c06253a37a..20965f2f22d095 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst @@ -58,7 +58,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -73,9 +73,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu24_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu24_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu24_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu24_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 @@ -83,9 +83,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu22_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu22_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu22_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu22_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: Ubuntu 20.04 :sync: ubuntu-20 @@ -93,9 +93,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: RHEL 8 :sync: rhel-8 @@ -103,18 +103,18 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_rhel8_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_rhel8_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_rhel8_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_rhel8_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: CentOS 7 :sync: centos-7 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_centos7_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_centos7_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_centos7_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_centos7_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: ARM 64-bit @@ -122,25 +122,25 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_arm64.tgz -O openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.4.0.16579.c3152d32c9c_arm64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_arm64.tgz -O openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.5.0.17288.7975fa5da0c_arm64 /opt/intel/openvino_2024.5.0 .. tab-item:: ARM 32-bit :sync: arm-32 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/linux/l_openvino_toolkit_debian10_2024.4.0.16579.c3152d32c9c_armhf.tgz -O openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv l_openvino_toolkit_debian10_2024.4.0.16579.c3152d32c9c_armhf /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux/l_openvino_toolkit_debian10_2024.5.0.17288.7975fa5da0c_armhf.tgz -O openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv l_openvino_toolkit_debian10_2024.5.0.17288.7975fa5da0c_armhf /opt/intel/openvino_2024.5.0 5. Install required system dependencies on Linux. To do this, OpenVINO provides a script in the extracted installation directory. Run the following command: .. code-block:: sh - cd /opt/intel/openvino_2024.4.0 + cd /opt/intel/openvino_2024.5.0 sudo -E ./install_dependencies/install_openvino_dependencies.sh 6. (Optional) Install *numpy* Python Library: @@ -149,11 +149,11 @@ Step 1: Download and Install the OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.4.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.5.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.4.0 + cd /opt/intel/openvino_2024.5.0 python3 -m pip install -r ./python/requirements.txt 7. For simplicity, it is useful to create a symbolic link as below: @@ -162,7 +162,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /opt/intel - sudo ln -s openvino_2024.4.0 openvino_2024 + sudo ln -s openvino_2024.5.0 openvino_2024 .. note:: If you have already installed a previous release of OpenVINO 2024, a symbolic link to the ``openvino_2024`` folder may already exist. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst index e9157a99e1c882..e4bff378106122 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst @@ -47,7 +47,7 @@ Step 1: Install OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -57,9 +57,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/macos/m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_x86_64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_x86_64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/macos/m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_x86_64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_x86_64 /opt/intel/openvino_2024.5.0 .. tab-item:: ARM, 64-bit :sync: arm-64 @@ -67,9 +67,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/macos/m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_arm64.tgz --output openvino_2024.4.0.tgz - tar -xf openvino_2024.4.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.4.0.16579.c3152d32c9c_arm64 /opt/intel/openvino_2024.4.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/macos/m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_arm64.tgz --output openvino_2024.5.0.tgz + tar -xf openvino_2024.5.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.5.0.17288.7975fa5da0c_arm64 /opt/intel/openvino_2024.5.0 5. (Optional) Install *numpy* Python Library: @@ -78,11 +78,11 @@ Step 1: Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.4.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.5.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.4.0 + cd /opt/intel/openvino_2024.5.0 python3 -m pip install -r ./python/requirements.txt 6. For simplicity, it is useful to create a symbolic link as below: @@ -90,7 +90,7 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - sudo ln -s /opt/intel/openvino_2024.4.0 /opt/intel/openvino_2024 + sudo ln -s /opt/intel/openvino_2024.5.0 /opt/intel/openvino_2024 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst index 8f3efeeb720dc9..9db280ec81472e 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst @@ -41,18 +41,18 @@ Step 1: Download and Install OpenVINO Core Components ``C:\Program Files (x86)\Intel`` is the recommended folder. You may also use a different path if desired or if you don't have administrator privileges on your computer. -2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. +2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. If you prefer using command-lines, run the following commands in the command prompt window you opened: .. code-block:: sh cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.4/windows/w_openvino_toolkit_windows_2024.4.0.16579.c3152d32c9c_x86_64.zip --output openvino_2024.4.0.zip + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/windows/w_openvino_toolkit_windows_2024.5.0.17288.7975fa5da0c_x86_64.zip --output openvino_2024.5.0.zip .. note:: - A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.4.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. + A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.5.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. 3. Use your favorite tool to extract the archive file, rename the extracted folder, and move it to the ``C:\Program Files (x86)\Intel`` directory. @@ -61,9 +61,9 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh - tar -xf openvino_2024.4.0.zip - ren w_openvino_toolkit_windows_2024.4.0.16579.c3152d32c9c_x86_64 openvino_2024.4.0 - move openvino_2024.4.0 "C:\Program Files (x86)\Intel" + tar -xf openvino_2024.5.0.zip + ren w_openvino_toolkit_windows_2024.5.0.17288.7975fa5da0c_x86_64 openvino_2024.5.0 + move openvino_2024.5.0 "C:\Program Files (x86)\Intel" 4. (Optional) Install *numpy* Python Library: @@ -72,11 +72,11 @@ Step 1: Download and Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.4.0\python`` folder: + You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.5.0\python`` folder: .. code-block:: sh - cd "C:\Program Files (x86)\Intel\openvino_2024.4.0" + cd "C:\Program Files (x86)\Intel\openvino_2024.5.0" python -m pip install -r .\python\requirements.txt @@ -85,7 +85,7 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh cd C:\Program Files (x86)\Intel - mklink /D openvino_2024 openvino_2024.4.0 + mklink /D openvino_2024 openvino_2024.5.0 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst index 34f04669e50d63..a10b0d0c7bbce4 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst @@ -47,24 +47,24 @@ Linux .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu24_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu24_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu22_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu22_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: Ubuntu 20.04 :sync: ubuntu-20 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu20_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu20_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: ARM 64-bit @@ -72,8 +72,8 @@ Linux .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/linux/openvino_genai_ubuntu20_2024.4.0.0_arm64.tar.gz -O openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu20_2024.5.0.0_arm64.tar.gz -O openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz Windows @@ -82,7 +82,7 @@ Windows .. code-block:: sh cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/windows/openvino_genai_windows_2024.4.0.0_x86_64.zip --output openvino_genai_2024.4.0.0.zip + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/windows/openvino_genai_windows_2024.5.0.0_x86_64.zip --output openvino_genai_2024.5.0.0.zip macOS ++++++++++++++++++++++++++ @@ -94,16 +94,16 @@ macOS .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/macos/openvino_genai_macos_12_6_2024.4.0.0_x86_64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/macos/openvino_genai_macos_12_6_2024.5.0.0_x86_64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz .. tab-item:: ARM, 64-bit :sync: arm-64 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.4/macos/openvino_genai_macos_12_6_2024.4.0.0_arm64.tar.gz --output openvino_genai_2024.4.0.0.tgz - tar -xf openvino_genai_2024.4.0.0.tgz + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/macos/openvino_genai_macos_12_6_2024.5.0.0_arm64.tar.gz --output openvino_genai_2024.5.0.0.tgz + tar -xf openvino_genai_2024.5.0.0.tgz Here are the full guides: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst b/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst index 6326513fa3cea1..cd3fd41fed03e0 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst @@ -119,7 +119,7 @@ to see if your case needs any of them. .. code-block:: python - from openvino import get_cmake_path + from openvino.utils import get_cmake_path cmake_path = get_cmake_path() For detailed instructions on how to use these configurations in your build setup, check out the diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst b/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst index 0ff1b95c8eb212..475f623ef86598 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst @@ -108,6 +108,6 @@ Additional Resources - `Official Yocto Project documentation `__ - `BitBake Tool `__ - `Poky `__ -- `Meta-intel `__ +- `Meta-intel `__ - `Meta-openembedded `__ - `Meta-clang `__ \ No newline at end of file diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst index 4585ca97488023..5a641300a68edb 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst @@ -9,7 +9,7 @@ This guide will give you extra details on how to utilize NPU with the GenAI flav for information on how to start. Prerequisites -############# +##################### Install required dependencies: @@ -17,35 +17,96 @@ Install required dependencies: python -m venv npu-env npu-env\Scripts\activate - pip install optimum-intel nncf==2.11 onnx==1.16.1 + pip install nncf==2.12 onnx==1.16.1 optimum-intel==1.19.0 pip install --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly +NOTE that for systems based on IntelĀ® Core Ultra Processors Series 2 and 16 GB of RAM, +prompts longer then 1024 characters will not work with a model of 7B or more parameters, +such as Llama-2-7B, Mistral-0.2-7B, and Qwen-2-7B. + Export an LLM model via Hugging Face Optimum-Intel ################################################## -A chat-tuned TinyLlama model is used in this example. The following conversion & optimization -settings are recommended when using the NPU: +Since **symmetrically-quantized 4-bit (INT4) models are preffered for inference on NPU**, make sure to export +the model with the proper conversion and optimization settings. + +| You may export LLMs via Optimum-Intel, using one of two compression methods: +| **group quantization** - for both smaller and larger models, +| **channel-wise quantization** - remarkably effective but for models exceeding 1 billion parameters. + +You select one of the methods by setting the ``--group-size`` parameter to either ``128`` or ``-1``, respectively. See the following examples: + +.. tab-set:: + + .. tab-item:: Group quantization + + .. code-block:: console + :name: group-quant + + optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-Chat-v1.0 --weight-format int4 --sym --ratio 1.0 --group_size 128 TinyLlama-1.1B-Chat-v1.0 + + .. tab-item:: Channel-wise quantization + + .. tab-set:: -.. code-block:: python + .. tab-item:: Data-free quantization - optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-Chat-v1.0 --weight-format int4 --sym --group-size 128 --ratio 1.0 TinyLlama -**For models exceeding 1 billion parameters**, it is recommended to use **channel-wise -quantization** that is remarkably effective. For example, you can try the approach with the -llama-2-7b-chat-hf model: + .. code-block:: console + :name: channel-wise-data-free-quant -.. code-block:: python + optimum-cli export openvino -m meta-llama/Llama-2-7b-chat-hf --weight-format int4 --sym --ratio 1.0 --group-size -1 Llama-2-7b-chat-hf - optimum-cli export openvino -m meta-llama/Llama-2-7b-chat-hf --weight-format int4 --sym --group-size -1 --ratio 1.0 Llama-2-7b-chat-hf + .. tab-item:: Data-aware quantization + + If you want to improve accuracy, make sure you: + + 1. Update NNCF: ``pip install nncf==2.13`` + 2. Use ``--scale_estimation --dataset=`` and accuracy aware quantization ``--awq``: + + .. code-block:: console + :name: channel-wise-data-aware-quant + + optimum-cli export openvino -m meta-llama/Llama-2-7b-chat-hf --weight-format int4 --sym --group-size -1 --ratio 1.0 --awq --scale-estimation --dataset=wikitext2 Llama-2-7b-chat-hf + + + .. important:: + + Remember that the negative value of ``-1`` is required here, not ``1``. + + + +You can also try using 4-bit (INT4) +`GPTQ models `__, +which do not require specifying quantization parameters: + +.. code-block:: console + + optimum-cli export openvino -m TheBloke/Llama-2-7B-Chat-GPTQ + + +| Remember, NPU supports GenAI models quantized symmetrically to INT4. +| Below is a list of such models: + +* meta-llama/Meta-Llama-3-8B-Instruct +* microsoft/Phi-3-mini-4k-instruct +* Qwen/Qwen2-7B +* mistralai/Mistral-7B-Instruct-v0.2 +* openbmb/MiniCPM-1B-sft-bf16 +* TinyLlama/TinyLlama-1.1B-Chat-v1.0 +* TheBloke/Llama-2-7B-Chat-GPTQ +* Qwen/Qwen2-7B-Instruct-GPTQ-Int4 Run generation using OpenVINO GenAI ################################### -It is recommended to install the latest available +It is typically recommended to install the latest available `driver `__. -Use the following code snippet to perform generation with OpenVINO GenAI API: +Use the following code snippet to perform generation with OpenVINO GenAI API. +Note that **currently, the NPU pipeline supports greedy decoding only**. This means that +you need to add ``do_sample=False`` **to the** ``generate()`` **method:** .. tab-set:: @@ -53,26 +114,31 @@ Use the following code snippet to perform generation with OpenVINO GenAI API: :sync: py .. code-block:: python + :emphasize-lines: 4 import openvino_genai as ov_genai model_path = "TinyLlama" pipe = ov_genai.LLMPipeline(model_path, "NPU") - print(pipe.generate("The Sun is yellow because", max_new_tokens=100)) + print(pipe.generate("The Sun is yellow because", max_new_tokens=100, do_sample=False)) .. tab-item:: C++ :sync: cpp .. code-block:: cpp + :emphasize-lines: 7, 9 #include "openvino/genai/llm_pipeline.hpp" #include int main(int argc, char* argv[]) { std::string model_path = "TinyLlama"; - ov::genai::LLMPipeline pipe(model_path, "NPU"); - std::cout << pipe.generate("The Sun is yellow because", ov::genai::max_new_tokens(100)); + ov::genai::GenerationConfig config; + config.do_sample=false; + config.max_new_tokens=100; + std::cout << pipe.generate("The Sun is yellow because", config); } + Additional configuration options ################################ @@ -88,9 +154,9 @@ user explicitly sets a lower length limit for the response. You may configure both the 'maximum input prompt length' and 'minimum response length' using the following parameters: -* ``MAX_PROMPT_LEN``: Defines the maximum number of tokens that the LLM pipeline can process - for the input prompt (default: 1024). -* ``MIN_RESPONSE_LEN``: Defines the minimum number of tokens that the LLM pipeline will generate +* ``MAX_PROMPT_LEN`` - defines the maximum number of tokens that the LLM pipeline can process + for the input prompt (default: 1024), +* ``MIN_RESPONSE_LEN`` - defines the minimum number of tokens that the LLM pipeline will generate in its response (default: 150). Use the following code snippet to change the default settings: @@ -113,10 +179,93 @@ Use the following code snippet to change the default settings: ov::AnyMap pipeline_config = { { "MAX_PROMPT_LEN", 1024 }, { "MIN_RESPONSE_LEN", 512 } }; ov::genai::LLMPipeline pipe(model_path, "NPU", pipeline_config); +Cache compiled models ++++++++++++++++++++++ + +Specify the ``NPUW_CACHE_DIR`` option in ``pipeline_config`` for NPU pipeline to +cache the compiled models. Using the code snippet below shortens the initialization time +of the pipeline runs coming next: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: python + + pipeline_config = { "NPUW_CACHE_DIR": ".npucache" } + pipe = ov_genai.LLMPipeline(model_path, "NPU", pipeline_config) + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + ov::AnyMap pipeline_config = { { "NPUW_CACHE_DIR", ".npucache" } }; + ov::genai::LLMPipeline pipe(model_path, "NPU", pipeline_config); + + +Disable memory allocation ++++++++++++++++++++++++++ + +In case of execution failures, either silent or with errors, try to update the NPU driver to +`32.0.100.3104 or newer `__. +If the update is not possible, set the ``DISABLE_OPENVINO_GENAI_NPU_L0`` +environment variable to disable NPU memory allocation, which might be supported +only on newer drivers for Intel Core Ultra 200V processors. + +Set the environment variable in a terminal: + +.. tab-set:: + + .. tab-item:: Linux + :sync: linux + + .. code-block:: console + + export DISABLE_OPENVINO_GENAI_NPU_L0=1 + + .. tab-item:: Windows + :sync: win + + .. code-block:: console + + set DISABLE_OPENVINO_GENAI_NPU_L0=1 + + +Performance modes ++++++++++++++++++++++ + +You can configure the NPU pipeline with the ``GENERATE_HINT`` option to switch +between two different performance modes: + +* ``FAST_COMPILE`` (default) - enables fast compilation at the expense of performance, +* ``BEST_PERF`` - ensures best possible performance at lower compilation speed. + +Use the following code snippet: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: python + + pipeline_config = { "GENERATE_HINT": "BEST_PERF" } + pipe = ov_genai.LLMPipeline(model_path, "NPU", pipeline_config) + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + ov::AnyMap pipeline_config = { { "GENERATE_HINT", "BEST_PERF" } }; + ov::genai::LLMPipeline pipe(model_path, "NPU", pipeline_config); + Additional Resources #################### * :doc:`NPU Device <../../openvino-workflow/running-inference/inference-devices-and-modes/npu-device>` * `OpenVINO GenAI Repo `__ -* `Neural Network Compression Framework `__ \ No newline at end of file +* `Neural Network Compression Framework `__ diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst index 953784c03fdef0..6033bd8ed96106 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst @@ -249,23 +249,21 @@ and use audio files in WAV format at a sampling rate of 16 kHz as input. def infer(model_dir: str, wav_file_path: str): - raw_speech = read_wav(wav_file_path) - pipe = openvino_genai.WhisperPipeline(model_dir) - - def streamer(word: str) -> bool: - print(word, end="") - return False + device = "CPU" # GPU or NPU can be used as well. + pipe = openvino_genai.WhisperPipeline(model_dir, device) + # The pipeline expects normalized audio with a sampling rate of 16kHz. + raw_speech = read_wav(wav_file_path) result = pipe.generate( raw_speech, max_new_tokens=100, language="<|en|>", task="transcribe", return_timestamps=True, - streamer=streamer, ) - print() + print(result) + for chunk in result.chunks: print(f"timestamps: [{chunk.start_ts}, {chunk.end_ts}] text: {chunk.text}") @@ -288,26 +286,21 @@ and use audio files in WAV format at a sampling rate of 16 kHz as input. std::filesystem::path models_path = argv[1]; std::string wav_file_path = argv[2]; - std::string device = "CPU"; // GPU can be used as well + std::string device = "CPU"; // GPU or NPU can be used as well. ov::genai::WhisperPipeline pipeline(models_path, device); - ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); - ov::genai::WhisperGenerationConfig config(models_path / "generation_config.json"); config.max_new_tokens = 100; config.language = "<|en|>"; config.task = "transcribe"; config.return_timestamps = true; - auto streamer = [](std::string word) { - std::cout << word; - return false; - }; - - auto result = pipeline.generate(raw_speech, config, streamer); + // The pipeline expects normalized audio with a sampling rate of 16kHz. + ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); + auto result = pipeline.generate(raw_speech, config); - std::cout << "\n"; + std::cout << result << "\n"; for (auto& chunk : *result.chunks) { std::cout << "timestamps: [" << chunk.start_ts << ", " << chunk.end_ts << "] text: " << chunk.text << "\n"; diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst index 7f220111f64b98..2476a0423e30e1 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst @@ -31,8 +31,8 @@ some examples of popular Generative AI scenarios: To write such pipelines, you can follow the examples provided as part of OpenVINO: -* `OpenVINO Latent Consistency Model C++ image generation pipeline `__ -* `OpenVINO Stable Diffusion (with LoRA) C++ image generation pipeline `__ +* `OpenVINO Latent Consistency Model C++ image generation pipeline `__ +* `OpenVINO Stable Diffusion (with LoRA) C++ image generation pipeline `__ To perform inference, models must be first converted to OpenVINO IR format using Hugging Face Optimum-Intel API. diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst b/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst index d6e23b3791d001..2064aa843a93d8 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst @@ -336,7 +336,7 @@ Additional Resources * `OpenVINO Tokenizers repo `__ * `OpenVINO Tokenizers Notebook `__ -* `Text generation C++ samples that support most popular models like LLaMA 2 `__ +* `Text generation C++ samples that support most popular models like LLaMA 3 `__ * `OpenVINO GenAI Repo `__ diff --git a/docs/articles_en/learn-openvino/openvino-samples/benchmark-tool.rst b/docs/articles_en/learn-openvino/openvino-samples/benchmark-tool.rst index 19c4a013c54aae..390fe00605f2c6 100644 --- a/docs/articles_en/learn-openvino/openvino-samples/benchmark-tool.rst +++ b/docs/articles_en/learn-openvino/openvino-samples/benchmark-tool.rst @@ -245,6 +245,13 @@ There are several options for setting the number of inference iterations: The more iterations a model runs, the better the statistics will be for determining average latency and throughput. +Maximum inference rate +++++++++++++++++++++++ + +By default, the benchmarking app will run inference at maximum rate based on device capabilities. +The maximum inferance rate can be configured by ``-max_irate `` option. +Tweaking this value allow better accuracy in power usage measurement by limiting the number of executions. + Inputs ++++++++++++++++++++ @@ -337,7 +344,7 @@ following usage message: [Step 1/11] Parsing and validating input arguments [ INFO ] Parsing input parameters usage: benchmark_app.py [-h [HELP]] [-i PATHS_TO_INPUT [PATHS_TO_INPUT ...]] -m PATH_TO_MODEL [-d TARGET_DEVICE] - [-hint {throughput,cumulative_throughput,latency,none}] [-niter NUMBER_ITERATIONS] [-t TIME] [-b BATCH_SIZE] [-shape SHAPE] + [-hint {throughput,cumulative_throughput,latency,none}] [-niter NUMBER_ITERATIONS] [-max_irate MAXIMUM_INFERENCE_RATE] [-t TIME] [-b BATCH_SIZE] [-shape SHAPE] [-data_shape DATA_SHAPE] [-layout LAYOUT] [-extensions EXTENSIONS] [-c PATH_TO_CLDNN_CONFIG] [-cdir CACHE_DIR] [-lfile [LOAD_FROM_FILE]] [-api {sync,async}] [-nireq NUMBER_INFER_REQUESTS] [-nstreams NUMBER_STREAMS] [-inference_only [INFERENCE_ONLY]] [-infer_precision INFER_PRECISION] [-ip {bool,f16,f32,f64,i8,i16,i32,i64,u8,u16,u32,u64}] @@ -536,6 +543,9 @@ following usage message: 'none': no device performance mode will be set. Using explicit 'nstreams' or other device-specific options, please set hint to 'none' -niter Optional. Number of iterations. If not specified, the number of iterations is calculated depending on a device. + -max_irate Optional. Maximum inference rate by frame per second. + If not specified, default value is 0, the inference will run at maximium rate depending on a device capabilities. + Tweaking this value allow better accuracy in power usage measurement by limiting the execution. -t Optional. Time in seconds to execute topology. Input shapes diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst index 5033d24ba3785a..2a551d7aa44eb5 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst @@ -76,7 +76,7 @@ of optimization methods (`"compression"` section). :fragment: [nncf_congig] Here is a brief description of the required parameters of the Filter Pruning method. For a full description refer to the -`GitHub `__ page. +`GitHub `__ page. * ``pruning_init`` - initial pruning rate target. For example, value ``0.1`` means that at the begging of training, convolutions that can be pruned will have 10% of their filters set to zero. diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst index 47cfed977dc3df..046dde9661c3bb 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst @@ -6,38 +6,36 @@ LLM Weight Compression :hidden: weight-compression/microscaling-quantization + weight-compression/4-bit-weight-quantization -Weight compression is a technique for enhancing the efficiency of models, -especially those with large memory requirements. This method reduces the model's -memory footprint, a crucial factor for Large Language Models (LLMs). +Weight compression enhances the efficiency of models by reducing their memory footprint, +a crucial factor for Large Language Models (LLMs). It is especially effective for networks with high memory requirements. -Unlike full model quantization, where weights and activations are quantized, -weight compression in `Neural Network Compression Framework (NNCF) `__ -only targets the model's weights. This approach allows the activations to remain as -floating-point numbers, preserving most of the model's accuracy while improving its -speed and reducing its size. +Unlike full model quantization, where both weights and activations are quantized, it +only targets weights, keeping activations as floating-point numbers. This means preserving most +of the model's accuracy while improving its +speed and reducing its size. The reduction in size is especially noticeable with larger models. +For instance the 7 billion parameter Llama 2 model can be reduced +from about 25GB to 4GB using 4-bit weight compression. -The reduction in size is especially noticeable with larger models, -for instance the 7 billion parameter Llama 2 model can be reduced -from about 25GB to 4GB using 4-bit weight compression. With smaller models (i.e. less -than 1B parameters), weight compression may result in more accuracy reduction than -with larger models. +.. note:: + + With smaller language models (i.e. less than 1B parameters), weight + compression may result in more accuracy reduction than with larger models. + Therefore, weight compression is recommended for use with LLMs only. -LLMs and other models that require +LLMs and other GenAI models that require extensive memory to store the weights during inference can benefit from weight compression as it: * enables inference of exceptionally large models that cannot be accommodated in the device memory; - * reduces storage and memory overhead, making models more lightweight and less resource intensive for deployment; - * improves inference speed by reducing the latency of memory access when computing the operations with weights, for example, Linear layers. The weights are smaller and thus faster to load from memory; - * unlike quantization, does not require sample data to calibrate the range of activation values. @@ -46,197 +44,228 @@ provides weight quantization to 8 and 4-bit integer data types as a compression method primarily designed to optimize LLMs. +Compression Methods (8-bit vs. 4-bit) +##################################### -Compress Model Weights -###################### +For models that come from `Hugging Face `__ and are supported +by Optimum, it is recommended to use the **Optimum Intel API**, which employs NNCF weight +compression capabilities to optimize various large Transformer models. -**8-bit weight quantization** method offers a balance between model size reduction and -maintaining accuracy, which usually leads to significant performance improvements for -Transformer-based models. Models with 8-bit compressed weights are performant on the -vast majority of supported CPU and GPU platforms. By default, weights are compressed -asymmetrically to "INT8_ASYM" mode. +The NNCF ``nncf.compress_weights()`` API, with most of its options, is exposed in the +``.from_pretrained()`` method of Optimum Intel classes. Optimum also has several datasets +for data-aware quantization available out-of-the-box. +You can use the examples below to perform data-free 8-bit or 4-bit weight quantization. +Before you start, make sure Optimum Intel is installed in your environment +by running the following command: -The code snippet below shows how to do asymmetrical 8-bit quantization of the model weights -represented in OpenVINO IR using NNCF: +.. code-block:: python + + pip install optimum[openvino] + +**8-bit weight quantization** offers a good balance between reducing the size and lowering the +accuracy of a model. It usually results in significant improvements for transformer-based models +and guarantees good model performance for a vast majority of supported CPU and GPU platforms. +By default, weights are compressed asymmetrically to "INT8_ASYM" mode. .. tab-set:: - .. tab-item:: OpenVINO - :sync: openvino + .. tab-item:: Compression with Optimum-Intel + :sync: optimum - .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py - :language: python - :fragment: [compression_8bit] + Load a pre-trained Hugging Face model, compress it to INT8_ASYM, using the + Optimum Intel API, and then execute inference with a text phrase: + Simply use the optimum-cli command line tool: -Now, the model is ready for compilation and inference. -It can be also saved into a compressed format, resulting in a smaller binary file. + .. code-block:: console -**4-bit weight quantization** method stands for an INT4-INT8 mixed-precision weight quantization, -where INT4 is considered as the primary precision and asymmetric INT8 is the backup one. -It usually results in a smaller model size and lower inference latency, although the accuracy -degradation could be higher, depending on the model. + optimum-cli export openvino --model microsoft/Phi-3.5-mini-instruct --weight-format int8 ov_phi-3.5-mini-instruct -The code snippet below shows how to do 4-bit quantization of the model weights represented -in OpenVINO IR using NNCF: + You can also use the code sample to the same effect: -.. tab-set:: + .. code-block:: python - .. tab-item:: OpenVINO - :sync: openvino + from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig + from transformers import AutoTokenizer, pipeline - .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py - :language: python - :fragment: [compression_4bit] + # Load and compress a model from Hugging Face. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained( + model_id, + export=True, + quantization_config=OVWeightQuantizationConfig(bits=8) + ) + # Inference + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) -The table below summarizes the benefits and trade-offs for each compression type in terms of -memory reduction, speed gain, and accuracy loss. + For more details, refer to the article on how to + :doc:`infer LLMs using Optimum Intel <../../learn-openvino/llm_inference_guide/llm-inference-hf>`. -.. list-table:: - :widths: 25 20 20 20 - :header-rows: 1 + .. tab-item:: Compression with NNCF + :sync: nncf - * - - - Memory Reduction - - Latency Improvement - - Accuracy Loss - * - INT8 Asymmetric - - Low - - Medium - - Low - * - INT4 Symmetric - - High - - High - - High - * - INT4 Asymmetric - - High - - Medium - - Medium + Load a pre-trained Hugging Face model, using the Optimum Intel API, + compress it to INT8_ASYM, using NNCF, and then execute inference with a text phrase: + .. code-block:: python + from nncf import compress_weights, CompressWeightsMode + from optimum.intel.openvino import OVModelForCausalLM + from transformers import AutoTokenizer, pipeline -The INT4 method has several parameters that can provide different performance-accuracy -trade-offs after optimization: + # Load a model and compress it with NNCF. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained(model_id, export=True, load_in_8bit=False, compile=False) + model.model = compress_weights(model.model, mode=CompressWeightsMode.INT8_ASYM) -* ``mode`` - there are two optimization modes: symmetric and asymmetric. + # Inference + model.compile() + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) - **Symmetric Compression** - ``INT4_SYM`` - INT4 Symmetric mode involves quantizing weights to a signed 4-bit integer - symmetrically without zero point. This mode is faster than the INT8_ASYM, making - it ideal for situations where **speed and size reduction are prioritized over accuracy**. +Here is an example of code using NNCF to perform asymmetrical 8-bit weight quantization of +a model in the OpenVINO IR format: - .. code-block:: python +.. tab-set:: - from nncf import compress_weights - from nncf import CompressWeightsMode + .. tab-item:: OpenVINO + :sync: openvino - compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_SYM) + .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py + :language: python + :fragment: [compression_8bit] - **Asymmetric Compression** - ``INT4_ASYM`` - INT4 Asymmetric mode also uses an unsigned 4-bit integer but quantizes weights - asymmetrically with a non-fixed zero point. This mode slightly compromises speed in - favor of better accuracy compared to the symmetric mode. This mode is useful when - **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. +**4-bit weight quantization** is actually a mixed-precision compression, +primarily INT4 and a backup asymmetric INT8 precisions. It produces a smaller model, +offering lower inference latency but potentially noticeable accuracy degradation, +depending on the model. - .. code-block:: python +.. tab-set:: - from nncf import compress_weights - from nncf import CompressWeightsMode + .. tab-item:: Compression with Optimum-Intel + :sync: optimum - compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM) + Load a pre-trained Hugging Face model, compress it to INT4, using the + Optimum Intel API, and then execute inference with a text phrase: -* ``group_size`` controls the size of the group of weights that share the same - quantization parameters. Shared quantization parameters help to speed up the - calculation of activation values as they are dequantized and quantized between - layers. However, they can reduce accuracy. The following group sizes are - recommended: ``128``, ``64``, ``32`` (``128`` is default value). + Simply use the optimum-cli command line tool: - `Smaller Group Size`: Leads to a more accurate model but increases the model's - footprint and reduces inference speed. + .. code-block:: console - `Larger Group Size`: Results in faster inference and a smaller model, but might - compromise accuracy. + optimum-cli export openvino --model microsoft/Phi-3.5-mini-instruct --weight-format int4 --awq --scale-estimation --dataset wikitext2 --group-size 64 --ratio 1.0 ov_phi-3.5-mini-instruct -* ``ratio`` controls the ratio between the layers compressed to the precision defined - by ``mode`` and the rest of the layers that will be kept in the ``backup_mode`` in the optimized model. - Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be - compressed to the precision defined by ``mode``, while the rest will be compressed to - ``backup_mode`` precision. The default value for ratio is 1. + You can also use the code sample to the same effect: - `Higher Ratio (more layers set to mode precision)`: Reduces the model size and increase inference speed but - might lead to higher accuracy degradation. + .. code-block:: python - `Lower Ratio (more layers set to backup_mode precision)`: Maintains better accuracy but results in a larger model size - and potentially slower inference. + from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig + from transformers import AutoTokenizer, pipeline - In this example, 90% of the model's layers are quantized to INT4 asymmetrically with - a group size of 64: + # Load and compress a model from Hugging Face. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained( + model_id, + export=True, + quantization_config=OVWeightQuantizationConfig( + bits=4, + quant_method="awq", + scale_estimation=True, + dataset="wikitext2", + group_size=64, + ratio=1.0 + ) + ) - .. code-block:: python + # Inference + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) - from nncf import compress_weights, CompressWeightsMode + .. tab-item:: Compression with NNCF + :sync: nncf - # Example: Compressing weights with INT4_ASYM mode, group size of 64, and 90% INT4 ratio - compressed_model = compress_weights( - model, - mode=CompressWeightsMode.INT4_ASYM, - group_size=64, - ratio=0.9, - ) + Load a pre-trained Hugging Face model, using the Optimum Intel API, + compress it to INT4 using NNCF, and then execute inference with a text phrase: -* ``scale_estimation`` - boolean parameter that enables more accurate estimation of - quantization scales. Especially helpful when the weights of all layers are quantized to - 4 bits. Requires dataset. + .. code-block:: python -* ``awq`` - boolean parameter that enables the AWQ method for more accurate INT4 weight - quantization. Especially helpful when the weights of all the layers are quantized to - 4 bits. The method can sometimes result in reduced accuracy when used with - Dynamic Quantization of activations. Requires dataset. + from nncf import compress_weights, CompressWeightsMode + from optimum.intel.openvino import OVModelForCausalLM + from transformers import AutoTokenizer, pipeline -* ``gptq`` - boolean parameter that enables the GPTQ method for more accurate INT4 weight - quantization. Requires dataset. + # Load a model and compress it with NNCF. + model_id = "microsoft/Phi-3.5-mini-instruct" + model = OVModelForCausalLM.from_pretrained(model_id, export=True, load_in_8bit=False, compile=False) + model.model = compress_weights(model.model, mode=CompressWeightsMode.INT4_SYM) -* ``dataset`` - calibration dataset for data-aware weight compression. It is required - for some compression options, for example, ``scale_estimation``, ``gptq`` or ``awq``. Some types - of ``sensitivity_metric`` can use data for precision selection. + # Inference + model.compile() + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) -* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing - layers in the bit-width selection algorithm. Some of the metrics require dataset to be - provided. The following types are supported: - * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - data-free metric computed as - the inverted 8-bit quantization noise. Weights with highest value of this metric can - be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in - 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more - accurate than per-channel, accuracy should not degrade. + For more details, refer to the article on how to + :doc:`infer LLMs using Optimum Intel <../../../learn-openvino/llm_inference_guide/llm-inference-hf>`. - * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires dataset. The average - Hessian trace of weights with respect to the layer-wise quantization error multiplied - by L2 norm of 8-bit quantization noise. +The code snippet below shows how to do 4-bit quantization of the model weights represented +in OpenVINO IR using NNCF: - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires dataset. The mean - variance of the layers' inputs multiplied by inverted 8-bit quantization noise. +.. tab-set:: - * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires dataset. The maximum - variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + .. tab-item:: OpenVINO + :sync: openvino - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires dataset. The mean - magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. + .. doxygensnippet:: docs/optimization_guide/nncf/code/weight_compression_openvino.py + :language: python + :fragment: [compression_4bit] -* ``all_layers`` - boolean parameter that enables INT4 weight quantization of all - Fully-Connected and Embedding layers, including the first and last layers in the model. +Refer to the article about +:doc:`4-bit weight quantization <./weight-compression/4-bit-weight-quantization>` +for more details. -* ``lora_correction`` - boolean parameter that enables the LoRA Correction Algorithm - to further improve the accuracy of INT4 compressed models on top of other - algorithms - AWQ and Scale Estimation. +Once the model has been optimized, it is ready for compilation and inference. The model can +also be :ref:`saved into a compressed format `, resulting in a +smaller binary file. -* ``backup_mode`` - defines a backup precision for mixed-precision weight compression. - There are three modes: INT8_ASYM, INT8_SYM, and NONE, which retains - the original floating-point precision of the model weights (``INT8_ASYM`` is default value). +The table below summarizes the benefits and trade-offs for each compression type in terms of +memory reduction, speed gain, and accuracy loss. + +.. list-table:: + :widths: 25 20 20 20 + :header-rows: 1 + + * - + - Memory Reduction + - Latency Improvement + - Accuracy Loss + * - INT8 Asymmetric + - Low + - Medium + - Low + * - INT4 Symmetric + - High + - High + - High + * - INT4 Asymmetric + - High + - Medium + - Medium **Use synthetic data for LLM weight compression** @@ -255,80 +284,41 @@ for details of the usage. from nncf import Dataset from nncf.data import generate_text_data + from functools import partial + + from transformers import AutoTokenizer, AutoModelForCausalLM # Example: Generating synthetic dataset - synthetic_data = generate_text_data(model, tokenizer) - nncf_dataset = nncf.Dataset(synthetic_data, transform_fn) + tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + hf_model = AutoModelForCausalLM.from_pretrained( + MODEL_ID, export=True, load_in_8bit=False + ) + + # Synthetic-based compression + synthetic_dataset = nncf.data.generate_text_data(hf_model, tokenizer, dataset_size=100) + quantization_dataset = nncf.Dataset( + synthetic_dataset, + transform_fn # See the example in NNCF repo to learn how to make transform_fn. + ) + + model = compress_weights( + model, + mode=CompressWeightsMode.INT4_ASYM, + group_size=64, + ratio=1.0, + dataset=quantization_dataset, + awq=True, + scale_estimation=True + ) # The model is openvino.Model. For data-aware weight compression refer to the following `example `__. .. note:: - Some methods can be stacked on top of one another to achieve a better - accuracy-performance trade-off after weight quantization. For example, the Scale Estimation - method can be applied along with AWQ and mixed-precision quantization (the ``ratio`` parameter). - -The example below shows data-free 4-bit weight quantization -applied on top of OpenVINO IR. Before trying the example, make sure Optimum Intel -is installed in your environment by running the following command: - -.. code-block:: python - - pip install optimum[openvino] - -The first example loads a pre-trained Hugging Face model using the Optimum Intel API, -compresses it to INT4 using NNCF, and then executes inference with a text phrase. - -If the model comes from `Hugging Face `__ and is supported -by Optimum, it may be easier to use the Optimum Intel API to perform weight compression. -The compression type is specified when the model is loaded using the ``load_in_8bit=True`` -or ``load_in_4bit=True`` parameter. The second example uses the Weight Compression API -from Optimum Intel instead of NNCF to compress the model to INT8_ASYM. - -.. tab-set:: - - .. tab-item:: OpenVINO - :sync: openvino - - .. code-block:: python - - from nncf import compress_weights, CompressWeightsMode - from optimum.intel.openvino import OVModelForCausalLM - from transformers import AutoTokenizer, pipeline - - # Load model from Hugging Face - model_id = "HuggingFaceH4/zephyr-7b-beta" - model = OVModelForCausalLM.from_pretrained(model_id, export=True, load_in_8bit=False, compile=False) - - # Compress to INT4 Symmetric - model.model = compress_weights(model.model, mode=CompressWeightsMode.INT4_SYM) - - # Inference - model.compile() - tokenizer = AutoTokenizer.from_pretrained(model_id) - pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) - phrase = "The weather is" - results = pipe(phrase) - print(results) - - .. tab-item:: Optimum-Intel - - .. code-block:: python - - from optimum.intel.openvino import OVModelForCausalLM - from transformers import AutoTokenizer, pipeline - - # Load and compress model from Hugging Face - model_id = "HuggingFaceH4/zephyr-7b-beta" - model = OVModelForCausalLM.from_pretrained(model_id, export=True, load_in_8bit=True) - - # Inference - tokenizer = AutoTokenizer.from_pretrained(model_id) - pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) - phrase = "The weather is" - results = pipe(phrase) - print(results) + Some methods can be stacked on top of one another to achieve a better + accuracy-performance trade-off after weight quantization. For example, the **Scale Estimation** + method can be applied along with **AWQ** and mixed-precision quantization (the ``ratio`` parameter). Exporting and Loading Compressed Models @@ -341,179 +331,157 @@ so it is preferable to compress the model once, save it, and then load the compressed model later for faster time to first inference. .. code-block:: python + :name: save_pretrained - # Save compressed model for faster loading later - model.save_pretrained("zephyr-7b-beta-int4-sym-ov") - tokenizer.save_pretrained("zephyr-7b-beta-int4-sym-ov") + # Save compressed model for faster loading later + model.save_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") + tokenizer.save_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") - # Load a saved model - model = OVModelForCausalLM.from_pretrained("zephyr-7b-beta-int4-sym-ov") - tokenizer = AutoTokenizer.from_pretrained("zephyr-7b-beta-int4-sym-ov") + # Load a saved model + model = OVModelForCausalLM.from_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") + tokenizer = AutoTokenizer.from_pretrained("Phi-3.5-mini-instruct-int4-sym-ov") -GPTQ Models -############ +.. tip:: -OpenVINO also supports 4-bit models from Hugging Face -`Transformers `__ library optimized -with `GPTQ `__. In this case, there is no -need for an additional model optimization step because model conversion will -automatically preserve the INT4 optimization results, allowing model inference to benefit from it. + Models optimized with with NNCF or Optimum Intel can be used with + :doc:`OpenVINO GenAI <../../learn-openvino/llm_inference_guide/genai-guide>`. -A compression example using a GPTQ model is shown below. -Make sure to install GPTQ dependencies by running the following command: -.. code-block:: python - - pip install optimum[openvino] auto-gptq - -.. code-block:: python - - from optimum.intel.openvino import OVModelForCausalLM - from transformers import AutoTokenizer, pipeline - - # Load model from Hugging Face already optimized with GPTQ - model_id = "TheBloke/Llama-2-7B-Chat-GPTQ" - model = OVModelForCausalLM.from_pretrained(model_id, export=True) +Auto-tuning of Weight Compression Parameters +############################################ - # Inference - tokenizer = AutoTokenizer.from_pretrained(model_id) - pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) - phrase = "The weather is" - results = pipe(phrase) - print(results) +To find the optimal weight compression parameters for a particular model, refer to the +`example `__ , +where weight compression parameters are being searched from the subset of values. +To speed up the search, a self-designed validation pipeline called +`WhoWhatBench `__ +is used. The pipeline can quickly evaluate the changes in the accuracy of the optimized +model compared to the baseline. -An `example of a model `__ -that has been optimized using GPTQ. Compression Metrics Examples -######################################## +############################ -The table below shows examples of text-generation Language Models with different +Below you will find examples of text-generation Language Models with different optimization settings in a data-free setup, where no dataset is used at the optimization step. The Perplexity metric is a measurement of response accuracy, where a higher complexity score indicates a lower accuracy. It is measured on the `Lambada OpenAI dataset `__. -.. list-table:: - :widths: 40 55 25 25 - :header-rows: 1 - - * - Model - - Optimization - - Perplexity\* - - Model Size (Gb) - * - databricks/dolly-v2-3b - - FP32 - - 5.01 - - 10.3 - * - databricks/dolly-v2-3b - - INT8_ASYM - - 5.07 - - 2.6 - * - databricks/dolly-v2-3b - - INT4_ASYM,group_size=32,ratio=0.5 - - 5.28 - - 2.2 - * - facebook/opt-6.7b - - FP32 - - 4.25 - - 24.8 - * - facebook/opt-6.7b - - INT8_ASYM - - 4.27 - - 6.2 - * - facebook/opt-6.7b - - INT4_ASYM,group_size=64,ratio=0.8 - - 4.32 - - 4.1 - * - meta-llama/Llama-2-7b-chat-hf - - FP32 - - 3.28 - - 25.1 - * - meta-llama/Llama-2-7b-chat-hf - - INT8_ASYM - - 3.29 - - 6.3 - * - meta-llama/Llama-2-7b-chat-hf - - INT4_ASYM,group_size=128,ratio=0.8 - - 3.41 - - 4.0 - * - togethercomputer/RedPajama-INCITE-7B-Instruct - - FP32 - - 4.15 - - 25.6 - * - togethercomputer/RedPajama-INCITE-7B-Instruct - - INT8_ASYM - - 4.17 - - 6.4 - * - togethercomputer/RedPajama-INCITE-7B-Instruct - - INT4_ASYM,group_size=128,ratio=1.0 - - 4.17 - - 3.6 - * - meta-llama/Llama-2-13b-chat-hf - - FP32 - - 2.92 - - 48.5 - * - meta-llama/Llama-2-13b-chat-hf - - INT8_ASYM - - 2.91 - - 12.1 - * - meta-llama/Llama-2-13b-chat-hf - - INT4_SYM,group_size=64,ratio=0.8 - - 2.98 - - 8.0 - - -The following table shows accuracy metric in a data-aware 4-bit weight quantization -setup measured on the `Wikitext dataset `__. - -.. list-table:: - :widths: 40 55 25 25 - :header-rows: 1 - - * - Model - - Optimization - - Word perplexity\* - - Model Size (Gb) - * - meta-llama/llama-7b-chat-hf - - FP32 - - 11.57 - - 12.61 - * - meta-llama/llama-7b-chat-hf - - INT4_SYM,group_size=128,ratio=1.0,awq=True - - 12.34 - - 2.6 - * - stabilityai_stablelm-3b-4e1t - - FP32 - - 10.17 - - 10.41 - * - stabilityai_stablelm-3b-4e1t - - INT4_SYM,group_size=64,ratio=1.0,awq=True - - 10.89 - - 2.6 - * - HuggingFaceH4/zephyr-7b-beta - - FP32 - - 9.82 - - 13.99 - * - HuggingFaceH4/zephyr-7b-beta - - INT4_SYM,group_size=128,ratio=1.0 - - 10.32 - - 2.6 +.. dropdown:: Perplexity\* in data-free optimization + + .. list-table:: + :widths: 40 55 25 25 + :header-rows: 1 + + * - Model + - Optimization + - Perplexity\* + - Model Size (Gb) + * - databricks/dolly-v2-3b + - FP32 + - 5.01 + - 10.3 + * - databricks/dolly-v2-3b + - INT8_ASYM + - 5.07 + - 2.6 + * - databricks/dolly-v2-3b + - INT4_ASYM,group_size=32,ratio=0.5 + - 5.28 + - 2.2 + * - facebook/opt-6.7b + - FP32 + - 4.25 + - 24.8 + * - facebook/opt-6.7b + - INT8_ASYM + - 4.27 + - 6.2 + * - facebook/opt-6.7b + - INT4_ASYM,group_size=64,ratio=0.8 + - 4.32 + - 4.1 + * - meta-llama/Llama-2-7b-chat-hf + - FP32 + - 3.28 + - 25.1 + * - meta-llama/Llama-2-7b-chat-hf + - INT8_ASYM + - 3.29 + - 6.3 + * - meta-llama/Llama-2-7b-chat-hf + - INT4_ASYM,group_size=128,ratio=0.8 + - 3.41 + - 4.0 + * - togethercomputer/RedPajama-INCITE-7B-Instruct + - FP32 + - 4.15 + - 25.6 + * - togethercomputer/RedPajama-INCITE-7B-Instruct + - INT8_ASYM + - 4.17 + - 6.4 + * - togethercomputer/RedPajama-INCITE-7B-Instruct + - INT4_ASYM,group_size=128,ratio=1.0 + - 4.17 + - 3.6 + * - meta-llama/Llama-2-13b-chat-hf + - FP32 + - 2.92 + - 48.5 + * - meta-llama/Llama-2-13b-chat-hf + - INT8_ASYM + - 2.91 + - 12.1 + * - meta-llama/Llama-2-13b-chat-hf + - INT4_SYM,group_size=64,ratio=0.8 + - 2.98 + - 8.0 + + +.. dropdown:: Perplexity\* in data-aware optimization + + The following table shows accuracy metric in a data-aware 4-bit weight quantization + setup measured on the `Wikitext dataset `__. + + .. list-table:: + :widths: 40 55 25 25 + :header-rows: 1 + + * - Model + - Optimization + - Word perplexity\* + - Model Size (Gb) + * - meta-llama/llama-7b-chat-hf + - FP32 + - 11.57 + - 12.61 + * - meta-llama/llama-7b-chat-hf + - INT4_SYM,group_size=128,ratio=1.0,awq=True + - 12.34 + - 2.6 + * - stabilityai_stablelm-3b-4e1t + - FP32 + - 10.17 + - 10.41 + * - stabilityai_stablelm-3b-4e1t + - INT4_SYM,group_size=64,ratio=1.0,awq=True + - 10.89 + - 2.6 + * - HuggingFaceH4/zephyr-7b-beta + - FP32 + - 9.82 + - 13.99 + * - HuggingFaceH4/zephyr-7b-beta + - INT4_SYM,group_size=128,ratio=1.0 + - 10.32 + - 2.6 \*Perplexity metric in both tables was measured without the Dynamic Quantization feature enabled in the OpenVINO runtime. -Auto-tuning of Weight Compression Parameters -############################################ - -To find the optimal weight compression parameters for a particular model, refer to the -`example `__ , -where weight compression parameters are being searched from the subset of values. -To speed up the search, a self-designed validation pipeline called -`WhoWhatBench `__ -is used. The pipeline can quickly evaluate the changes in the accuracy of the optimized -model compared to the baseline. Additional Resources #################### diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression/4-bit-weight-quantization.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression/4-bit-weight-quantization.rst new file mode 100644 index 00000000000000..ae9bc7d7b8b4a3 --- /dev/null +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression/4-bit-weight-quantization.rst @@ -0,0 +1,175 @@ +4-bit Weight Quantization +========================= + +The 4-bit weight quantization method results in significant reduction in model size and +memory usage, making LLMs more accessible to less performant devices. +It also usually offers lower inference latency, however, depending on specific models, +it may potentially impact the accuracy. + +Nevertheless, the INT4 method has several parameters that can provide different performance-accuracy +trade-offs after optimization: + +* ``mode`` - there are two optimization modes: symmetric and asymmetric. + + .. tab-set:: + + .. tab-item:: Symmetric Compression + :sync: int4-sym + + INT4 Symmetric mode (``INT4_SYM``) involves quantizing weights to a signed 4-bit integer + symmetrically without zero point. This mode is faster than the INT8_ASYM, making + it ideal for situations where **speed and size reduction are prioritized over accuracy**. + + .. code-block:: python + + from nncf import compress_weights + from nncf import CompressWeightsMode + + compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_SYM) + + .. tab-item:: Asymmetric Compression + :sync: int4-asym + + INT4 Asymmetric mode (``INT4_ASYM``) also uses an unsigned 4-bit integer but quantizes weights + asymmetrically with a non-fixed zero point. This mode slightly compromises speed in + favor of better accuracy compared to the symmetric mode. This mode is useful when + **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. + + .. code-block:: python + + from nncf import compress_weights + from nncf import CompressWeightsMode + + compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM) + +* ``group_size`` controls the size of the group of weights that share the same + quantization parameters. Shared quantization parameters help to speed up the + calculation of activation values as they are dequantized and quantized between + layers. However, they can reduce accuracy. The following group sizes are + recommended: ``128``, ``64``, ``32`` (``128`` is default value). + + `Smaller Group Size`: Leads to a more accurate model but increases the model's + footprint and reduces inference speed. + + `Larger Group Size`: Results in faster inference and a smaller model, but might + compromise accuracy. + +* ``ratio`` controls the ratio between the layers compressed to the precision defined + by ``mode`` and the rest of the layers that will be kept in the ``backup_mode`` in the optimized model. + Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be + compressed to the precision defined by ``mode``, while the rest will be compressed to + ``backup_mode`` precision. The default value for ratio is 1. + + | **Higher Ratio (more layers set to mode precision)**: + | Reduces the model size and increase inference speed but + might lead to higher accuracy degradation. + + | **Lower Ratio (more layers set to backup_mode precision)**: + | Maintains better accuracy but results in a larger model size + and potentially slower inference. + + In the example below, 90% of the model's layers are quantized to INT4 asymmetrically with + a group size of 64: + + .. code-block:: python + + from nncf import compress_weights, CompressWeightsMode + + # Example: Compressing weights with INT4_ASYM mode, group size of 64, and 90% INT4 ratio + compressed_model = compress_weights( + model, + mode=CompressWeightsMode.INT4_ASYM, + group_size=64, + ratio=0.9, + ) + +* ``scale_estimation`` - a boolean parameter that enables more accurate estimation of + quantization scales. Especially helpful when the weights of all layers are quantized to + 4 bits. Requires dataset. + +* ``awq`` - a boolean parameter that enables the AWQ method for more accurate INT4 weight + quantization. Especially helpful when the weights of all the layers are quantized to + 4 bits. The method can sometimes result in reduced accuracy when used with + Dynamic Quantization of activations. Requires dataset. + +* ``gptq`` - a boolean parameter that enables the GPTQ method for more accurate INT4 weight + quantization. Requires dataset. + +* ``dataset`` - a calibration dataset for data-aware weight compression. It is required + for some compression options, for example, ``scale_estimation``, ``gptq`` or ``awq``. Some types + of ``sensitivity_metric`` can use data for precision selection. + +* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing + layers in the bit-width selection algorithm. Some of the metrics require dataset to be + provided. The following types are supported: + + * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - a data-free metric computed as + the inverted 8-bit quantization noise. Weights with highest value of this metric can + be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in + 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more + accurate than per-channel, accuracy should not degrade. + + * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires a dataset. The average + Hessian trace of weights with respect to the layer-wise quantization error multiplied + by L2 norm of 8-bit quantization noise. + + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires a dataset. The mean + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + + * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires a dataset. The maximum + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires a dataset. The mean + magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. + +* ``all_layers`` - a boolean parameter that enables INT4 weight quantization of all + Fully-Connected and Embedding layers, including the first and last layers in the model. + +* ``lora_correction`` - a boolean parameter that enables the LoRA Correction Algorithm + to further improve the accuracy of INT4 compressed models on top of other + algorithms - AWQ and Scale Estimation. + +* ``backup_mode`` - defines a backup precision for mixed-precision weight compression. + There are three modes: INT8_ASYM, INT8_SYM, and NONE, which retains + the original floating-point precision of the model weights (``INT8_ASYM`` is default value). + +| + +4-bit Weight Quantization with GPTQ +################################### + +You can use models from Hugging Face +`Transformers `__ library, which are quantized +with `GPTQ `__ algorithm. Such models do not require +additional optimization step because the conversion will automatically preserve +the INT4 optimization results, and model inference will eventually benefit from it. + +See the `example of a model `__ +that has been optimized with GPTQ. + +You can also refer to the code sample below which shows how to load a 4-bit +GPTQ model and run inference. + +.. dropdown:: Using a GPTQ model. + + Make sure to install GPTQ dependencies by running the following command: + + .. code-block:: python + + pip install optimum[openvino] auto-gptq + + .. code-block:: python + + from optimum.intel.openvino import OVModelForCausalLM + from transformers import AutoTokenizer, pipeline + + # Load model from Hugging Face already optimized with GPTQ + model_id = "TheBloke/Llama-2-7B-Chat-GPTQ" + model = OVModelForCausalLM.from_pretrained(model_id, export=True) + + # Inference + tokenizer = AutoTokenizer.from_pretrained(model_id) + pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) + phrase = "The weather is" + results = pipe(phrase) + print(results) diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst index 41d43f7eea37d6..aa8e9cdabfda64 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst @@ -83,7 +83,7 @@ Accordingly, the code that loops over all available devices of the "GPU" type on Additional Resources #################### -* `OpenVINOā„¢ Runtime API Tutorial <./../../notebooks/openvino-api-with-output.html>`__ -* `AUTO Device Tutorial <./../../notebooks/auto-device-with-output.html>`__ -* `GPU Device Tutorial <./../../notebooks/gpu-device-with-output.html>`__ -* `NPU Device Tutorial <./../../notebooks/hello-npu-with-output.html>`__ \ No newline at end of file +* `OpenVINOā„¢ Runtime API Tutorial <../../notebooks/openvino-api-with-output.html>`__ +* `AUTO Device Tutorial <../../notebooks/auto-device-with-output.html>`__ +* `GPU Device Tutorial <../../notebooks/gpu-device-with-output.html>`__ +* `NPU Device Tutorial <../../notebooks/hello-npu-with-output.html>`__ \ No newline at end of file diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst index a440f77bc79984..46b541d84d4035 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst @@ -63,19 +63,19 @@ the model precision and the ratio of P-cores and E-cores. Then the default settings for low-level performance properties on Windows and Linux are as follows: -+--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| Property | Windows | Linux | -+======================================+========================================================================+====================================================================+ -| ``ov::num_streams`` | 1 | 1 | -+--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| ``ov::inference_num_threads`` | is equal to the number of P-cores or P-cores+E-cores on one socket | is equal to the number of P-cores or P-cores+E-cores on one socket | -+--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| ``ov::hint::scheduling_core_type`` | :ref:`Core Type Table of Latency Hint ` | :ref:`Core Type Table of Latency Hint ` | -+--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| ``ov::hint::enable_hyper_threading`` | No | No | -+--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| ``ov::hint::enable_cpu_pinning`` | No / Not Supported | Yes except using P-cores and E-cores together | -+--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ ++--------------------------------------+-----------------------------------------------------------------------+-----------------------------------------------------------------------+ +| Property | Windows | Linux | ++======================================+=======================================================================+=======================================================================+ +| ``ov::num_streams`` | 1 | 1 | ++--------------------------------------+-----------------------------------------------------------------------+-----------------------------------------------------------------------+ +| ``ov::inference_num_threads`` | is equal to the number of P-cores or P-cores+E-cores on one numa node | is equal to the number of P-cores or P-cores+E-cores on one numa node | ++--------------------------------------+-----------------------------------------------------------------------+-----------------------------------------------------------------------+ +| ``ov::hint::scheduling_core_type`` | :ref:`Core Type Table of Latency Hint ` | :ref:`Core Type Table of Latency Hint ` | ++--------------------------------------+-----------------------------------------------------------------------+-----------------------------------------------------------------------+ +| ``ov::hint::enable_hyper_threading`` | No | No | ++--------------------------------------+-----------------------------------------------------------------------+-----------------------------------------------------------------------+ +| ``ov::hint::enable_cpu_pinning`` | No / Not Supported | Yes except using P-cores and E-cores together | ++--------------------------------------+-----------------------------------------------------------------------+-----------------------------------------------------------------------+ .. note:: @@ -91,6 +91,16 @@ Then the default settings for low-level performance properties on Windows and Li enabled on Linux. Such default settings are aligned with typical workloads running in the corresponding environments to guarantee better out-of-the-box (OOB) performance. +.. note:: + + Starting from 5th Gen Intel Xeon Processors, new microarchitecture enabled new sub-NUMA clusters + feature. A sub-NUMA cluster (SNC) can create two or more localization domains (numa nodes) + within a socket by BIOS configuration. + By default OpenVINO with latency hint uses single NUMA node for inference. Although such + behavior allows to achive best performance for most of the models, there might be corner + cases which require manual tuning of ``ov::num_streams`` and ``ov::hint::enable_hyper_threading parameters``. + Please find more detail about `Sub-NUMA Clustering `__ + Throughput Hint ##################### diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst index 78cf0632f61b2b..b4e1c7ac15afcc 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst @@ -19,7 +19,7 @@ For an in-depth description of the GPU plugin, see: - `GPU plugin developer documentation `__ - `OpenVINO Runtime GPU plugin source files `__ -- `Accelerate Deep Learning Inference with IntelĀ® Processor Graphics `__ +- `Start AI Development with Intel `__ The GPU plugin is a part of the IntelĀ® Distribution of OpenVINOā„¢ toolkit. For more information on how to configure a system to use it, see the :doc:`GPU configuration <../../../get-started/configurations/configurations-intel-gpu>`. diff --git a/docs/articles_en/openvino-workflow/running-inference/optimize-inference/high-level-performance-hints.rst b/docs/articles_en/openvino-workflow/running-inference/optimize-inference/high-level-performance-hints.rst index 26a09214ea462a..e45f51a37afa5e 100644 --- a/docs/articles_en/openvino-workflow/running-inference/optimize-inference/high-level-performance-hints.rst +++ b/docs/articles_en/openvino-workflow/running-inference/optimize-inference/high-level-performance-hints.rst @@ -119,9 +119,6 @@ The hints are used on the presumption that the application queries ``ov::optimal While an application is free to create more requests if needed (for example to support asynchronous inputs population) **it is very important to at least run the** ``ov::optimal_number_of_infer_requests`` **of the inference requests in parallel**. It is recommended for efficiency, or device utilization, reasons. -Keep in mind that ``ov::hint::PerformanceMode::LATENCY`` does not necessarily imply using single inference request. For example, multi-socket CPUs can deliver as many requests at the same minimal latency as the number of NUMA nodes in the system. -To make your application fully scalable, make sure to query the ``ov::optimal_number_of_infer_requests`` directly. - .. _prefer-async-api: Prefer Async API diff --git a/docs/articles_en/openvino-workflow/running-inference/optimize-inference/optimizing-throughput/advanced_throughput_options.rst b/docs/articles_en/openvino-workflow/running-inference/optimize-inference/optimizing-throughput/advanced_throughput_options.rst index 7466d00efe5eb7..cad5633e11f85b 100644 --- a/docs/articles_en/openvino-workflow/running-inference/optimize-inference/optimizing-throughput/advanced_throughput_options.rst +++ b/docs/articles_en/openvino-workflow/running-inference/optimize-inference/optimizing-throughput/advanced_throughput_options.rst @@ -85,12 +85,12 @@ Number of Streams Considerations * Select the number of streams that is **less or equal** to the number of requests that the application would be able to run simultaneously. * To avoid wasting resources, the number of streams should be enough to meet the *average* parallel slack rather than the peak load. -* Use the `ov::streams::AUTO `__ as a more portable option (that also respects the underlying hardware configuration). +* Use the `ov::streams::AUTO <../../../../api/c_cpp_api/group__ov__runtime__cpp__prop__api.html#_CPPv44AUTO>`__ as a more portable option (that also respects the underlying hardware configuration). * It is very important to keep these streams busy, by running as many inference requests as possible (for example, start the newly-arrived inputs immediately): - * A bare minimum of requests to saturate the device can be queried as the `ov::optimal_number_of_infer_requests `__ of the ``ov:Compiled_Model``. + * A bare minimum of requests to saturate the device can be queried as the `ov::optimal_number_of_infer_requests <../../../../api/c_cpp_api/group__ov__runtime__cpp__prop__api.html#_CPPv432optimal_number_of_infer_requests>`__ of the ``ov:Compiled_Model``. -* *The maximum number of streams* for the device (per model) can be queried as the `ov::range_for_streams `__. +* *The maximum number of streams* for the device (per model) can be queried as the `ov::range_for_streams <../../../../api/c_cpp_api/group__ov__runtime__cpp__prop__api.html#_CPPv417range_for_streams>`__. Batch Size Considerations +++++++++++++++++++++++++ @@ -99,7 +99,7 @@ Batch Size Considerations * Otherwise (or if the number of "available" requests fluctuates), you may need to keep several instances of the network (reshaped to the different batch size) and select the properly sized instance in the runtime accordingly. -* For OpenVINO devices that implement a dedicated heuristic internally, the `ov::optimal_batch_size `__ is a *device* property (that accepts the actual model as a parameter) to query the recommended batch size for the model. +* For OpenVINO devices that implement a dedicated heuristic internally, the `ov::optimal_batch_size <../../../../api/c_cpp_api/group__ov__runtime__cpp__prop__api.html#_CPPv418optimal_batch_size>`__ is a *device* property (that accepts the actual model as a parameter) to query the recommended batch size for the model. A Few Device-specific Details diff --git a/docs/articles_en/openvino-workflow/running-inference/stateful-models.rst b/docs/articles_en/openvino-workflow/running-inference/stateful-models.rst index 86788b20249a3f..d00fd19c4d636d 100644 --- a/docs/articles_en/openvino-workflow/running-inference/stateful-models.rst +++ b/docs/articles_en/openvino-workflow/running-inference/stateful-models.rst @@ -139,5 +139,5 @@ sequences. You can find more examples demonstrating how to work with states in other articles: -* `LLM Chatbot notebook <../../notebooks/stable-zephyr-3b-chatbot-with-output.html>`__ +* `LLaVA-NeXT Multimodal Chatbot notebook <../../notebooks/llava-next-multimodal-chatbot-with-output.html>`__ * :doc:`Serving Stateful Models with OpenVINO Model Server <../../openvino-workflow/model-server/ovms_docs_stateful_models>` diff --git a/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst b/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst index 438c9ea9ec0bd3..3032add547f8a8 100644 --- a/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst +++ b/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst @@ -201,6 +201,6 @@ Additional Resources * Learn about the :doc:`basic steps to integrate inference in your application `. -* Use `OpenVINO tokenizers `__ to produce models that use string tensors to work with textual information as pre- and post-processing for the large language models. +* Use `OpenVINO tokenizers `__ to produce models that use string tensors to work with textual information as pre- and post-processing for the large language models. -* Check out `GenAI Samples `__ to see how string tensors are used in real-life applications. +* Check out `GenAI Samples `__ to see how string tensors are used in real-life applications. diff --git a/docs/articles_en/openvino-workflow/torch-compile.rst b/docs/articles_en/openvino-workflow/torch-compile.rst index 5bdb51a596d5d8..e5bc0ca901a5aa 100644 --- a/docs/articles_en/openvino-workflow/torch-compile.rst +++ b/docs/articles_en/openvino-workflow/torch-compile.rst @@ -288,7 +288,7 @@ PyTorch supports ``torch.compile`` officially on Windows from version 2.3.0 onwa For PyTorch versions below 2.3.0, the ``torch.compile`` feature is not supported on Windows officially. However, it can be accessed by running the following instructions: -1. Install the PyTorch nightly wheel file - `2.1.0.dev20230713 `__ , +1. Install the PyTorch nightly wheel file - `2.1.0.dev20230713 `__ , 2. Update the file at ``/Lib/site-packages/torch/_dynamo/eval_frames.py`` 3. Find the function called ``check_if_dynamo_supported()``: @@ -374,7 +374,7 @@ The ``torch.compile`` feature is part of PyTorch 2.0, and is based on: (PEP 523) to dynamically modify Python bytecode right before it is executed (PyTorch operators that cannot be extracted to FX graph are executed in the native Python environment). It maintains the eager-mode capabilities using - `Guards `__ to ensure the + `Guards `__ to ensure the generated graphs are valid. * **AOTAutograd** - generates the backward graph corresponding to the forward graph captured by TorchDynamo. diff --git a/docs/dev/ci/github_actions/overview.md b/docs/dev/ci/github_actions/overview.md index 8daf56a3a2252f..e65c085ede30d5 100644 --- a/docs/dev/ci/github_actions/overview.md +++ b/docs/dev/ci/github_actions/overview.md @@ -11,6 +11,7 @@ detailed instructions where necessary. * [Required workflows](#required-workflows) * [Workflow structure](#structure-of-the-workflows) * [Workflow and job organisation](#workflows-and-jobs-organisation) + * [Security considerations](#security-considerations) * [Finding results, artifacts and logs](#finding-results-artifacts-and-logs) * [Custom actions overview](#custom-actions) * [Machines overview](#machines) @@ -205,6 +206,10 @@ Overview of the [Linux workflow's](../../../../.github/workflows/ubuntu_22.yml) * All the steps are executed in the shell specified by the `shell` key under `defaults: run:` unless a shell is specified directly in a step. +### Security considerations + +Please consult [workflow security guidelines](security.md) before submitting a PR with GitHub Actions workflows changes. + ## Finding Results, Artifacts, and Logs ### Results diff --git a/docs/dev/ci/github_actions/security.md b/docs/dev/ci/github_actions/security.md new file mode 100644 index 00000000000000..d46cf6fd865c41 --- /dev/null +++ b/docs/dev/ci/github_actions/security.md @@ -0,0 +1,99 @@ +# Security best practices for GitHub Actions Workflows + +There are a few simple steps that we should follow to ensure our workflows are not vulnerable to common attacks. + +## Adjust `GITHUB_TOKEN` permissions + +Use the `permissions` key to make sure the `GITHUB_TOKEN` is configured with the least privileges for each job. + +Start with relatively safe permissions: + +```yaml +permissions: read-all +``` + +If you need more permissions, declare them at the job level when possible, for example: + +```yaml +jobs: + stale: + runs-on: ubuntu-latest + + # GITHUB_TOKEN will have only these permissions for + # `stale` job + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@f7176fd3007623b69d27091f9b9d4ab7995f0a06 + +``` + +Check [GitHub documentation](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/controlling-permissions-for-github_token) on this also. + +## Reduce the scope of environment variables + +Environment variables should be declared at the step level when possible (e.g. the variable is used only in this exact step). Only put variables on the job level when they're used by a few steps, and on the workflow level when they're used by most of the steps. + +Example from [the official GitHub documentation](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables): + +```yaml +name: Greeting on variable day + +on: + workflow_dispatch + +# Workflow level variables. Avoid using these. +env: + DAY_OF_WEEK: Monday + +jobs: + greeting_job: + runs-on: ubuntu-latest + # Job level variables + env: + Greeting: Hello + steps: + - name: "Say Hello Mona it's Monday" + run: echo "$Greeting $First_Name. Today is $DAY_OF_WEEK!" + # Step level variables. Prefer this approach + env: + First_Name: Mona + +``` + +## Avoid using `pull_request_target` + +**Never** use `pull_request_target` trigger event for workflows. If you want to use `pull_request_target`, contact a member of the OpenVINO GitHub Actions task force first. Check [GitHub blog post](https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/) on this as well. + +## Handle secrets correctly + +**Never ever** use plain-text secrets hard-coded in GitHub Actions Workflow. If you need to use secrets, contact a member of the OpenVINO GitHub Actions task force first. + +## Be careful with user input. + +Most of GitHub context variables propagated from user input. That means they should be treated as an untrusted and potentially malicious. There are some tactics you can use to mitigate the risk: +- Instead of using inline scripts, create an action and pass the variable as an argument +- Put the value into an environment variable for the step, and use the variable in the script + +More details are available in [this](https://securitylab.github.com/resources/github-actions-untrusted-input/) blog post. + +## Pin versions for GitHub Actions + +When using third-party actions, pin the version with a commit hash rather than a tag to shield your workflow from potential supply-chain compromise. + +For example, instead of this: + +```yaml +uses: actions/checkout@v4.2.2 +``` + +use this: + +```yaml +uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +``` + +## Further reading +Follow general [recommendations from GitHub itself](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions) diff --git a/docs/dev/ov_dependencies.txt b/docs/dev/ov_dependencies.txt index 7cf26c58a9436d..d9c344d2c3048d 100644 --- a/docs/dev/ov_dependencies.txt +++ b/docs/dev/ov_dependencies.txt @@ -1,6 +1,6 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#This file provides a comprehensive list of all dependencies of OpenVINO 2024.4 +#This file provides a comprehensive list of all dependencies of OpenVINO 2024.5 #The file is part of the automation pipeline for posting OpenVINO IR models on the HuggingFace Hub, including OneBOM dependency checks. diff --git a/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py b/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py index c82e0a8d5995f7..dcce229308da24 100644 --- a/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py +++ b/docs/openvino_custom_sphinx_sitemap/openvino_custom_sphinx_sitemap/__init__.py @@ -120,15 +120,29 @@ def process_coveo_meta(meta, url, link): for namespace, values in meta: namespace_element = ET.SubElement(url, namespace) + loc_element = url.find("loc") for tag_name, tag_value in values.items(): if tag_name == 'ovdoctype': - processed_link = process_link(link) - ET.SubElement(namespace_element, tag_name).text = processed_link - else: + ET.SubElement(namespace_element, tag_name).text = process_link(link) + elif tag_name == 'ovcategory' and loc_element is not None: + ET.SubElement(namespace_element, tag_name).text = extract_hierarchy(loc_element.text) + elif tag_name == 'ovversion': ET.SubElement(namespace_element, tag_name).text = tag_value def process_link(link): if '/' in link: return link.split('/')[0].replace("-", " ") - return link.split('.html')[0].replace("-", " ") \ No newline at end of file + return link.split('.html')[0].replace("-", " ") + +def extract_hierarchy(link): + path = link.split("://")[-1] + segments = path.split('/')[1:] + if segments and segments[-1].endswith('.html'): + segments = segments[:-1] + + hierarchy = [] + for i in range(1, len(segments) + 1): + hierarchy.append('|'.join(segments[:i])) + + return ';'.join(hierarchy) \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx deleted file mode 100644 index 9b53d90e0862db..00000000000000 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-Performance-Data.xlsx and /dev/null differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-platform_list.pdf b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-platform_list.pdf deleted file mode 100644 index cba78e5244acf1..00000000000000 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-platform_list.pdf and /dev/null differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx deleted file mode 100644 index 4e243b8190c876..00000000000000 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.4-system-info-detailed.xlsx and /dev/null differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx new file mode 100644 index 00000000000000..0c29b3282790fa Binary files /dev/null and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-platform_list.pdf b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-platform_list.pdf new file mode 100644 index 00000000000000..9cb20570020cc8 Binary files /dev/null and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-platform_list.pdf differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx new file mode 100644 index 00000000000000..60150e1a457eaf Binary files /dev/null and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-system-info-detailed.xlsx differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json index 0bb38199b997ca..59e06ef51f812d 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json @@ -1,18 +1,22 @@ [ { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 11.75, - "fp16": "", - "fp32": 4.32, - "bf16": "" + "int8": 312.06, + "fp16": 345.49, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -22,7 +26,7 @@ "Precisions": [ { "int4": "", - "int8": 87.69, + "int8": 4.83, "fp16": "", "fp32": "", "bf16": "" @@ -34,19 +38,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.17, - "fp16": "", - "fp32": 15.03, - "bf16": "" + "int8": 328.55, + "fp16": 285.3, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -56,7 +64,7 @@ "Precisions": [ { "int4": "", - "int8": 49.24, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -68,19 +76,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 32.5, + "int8": "", "fp16": "", - "fp32": 21.9, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 20.07, + "token_int8": 17.42, + "token_fp16": "" } ], "Unit": "FPS", @@ -89,8 +101,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 35.99, + "int4": 49.81, + "int8": 57.4, "fp16": "", "fp32": "", "bf16": "" @@ -102,19 +114,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 33.02, + "int8": "", "fp16": "", - "fp32": 12.59, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 36.48, + "token_int8": 27.59, + "token_fp16": "" } ], "Unit": "FPS", @@ -123,8 +139,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 45.4, + "int4": 27.41, + "int8": 36.24, "fp16": "", "fp32": "", "bf16": "" @@ -136,19 +152,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39.93, + "int8": "", "fp16": "", - "fp32": 15.99, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 42.82, + "token_int8": 33.97, + "token_fp16": 22.23 } ], "Unit": "FPS", @@ -157,9 +177,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 40.15, - "fp16": "", + "int4": 23.35, + "int8": 29.43, + "fp16": 44.97, "fp32": "", "bf16": "" } @@ -170,19 +190,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 119.95, + "int8": "", "fp16": "", - "fp32": 47.19, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 39.6, + "token_int8": 30.59, + "token_fp16": "" } ], "Unit": "FPS", @@ -191,8 +215,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 13.31, + "int4": 25.25, + "int8": 32.69, "fp16": "", "fp32": "", "bf16": "" @@ -204,19 +228,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 33.13, + "int8": "", "fp16": "", - "fp32": 22.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 55.37, + "token_int8": 51.62, + "token_fp16": 35.82 } ], "Unit": "FPS", @@ -225,9 +253,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 31.1, - "fp16": "", + "int4": 18.06, + "int8": 19.37, + "fp16": 27.91, "fp32": "", "bf16": "" } @@ -238,19 +266,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 50.91, - "fp16": "", - "fp32": 18.37, - "bf16": "" + "int8": 34.84, + "fp16": 19.43, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -260,7 +292,7 @@ "Precisions": [ { "int4": "", - "int8": 22.52, + "int8": 48.51, "fp16": "", "fp32": "", "bf16": "" @@ -272,19 +304,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38.31, + "int8": "", "fp16": "", - "fp32": 13.71, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 43.4, + "token_int8": 32.32, + "token_fp16": 20.91 } ], "Unit": "FPS", @@ -293,9 +329,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 28.51, - "fp16": "", + "int4": 23.04, + "int8": 30.94, + "fp16": 47.82, "fp32": "", "bf16": "" } @@ -306,19 +342,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 86.13, - "fp16": "", - "fp32": 33.75, - "bf16": "" + "int8": 2348.6, + "fp16": 2074.34, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -328,7 +368,7 @@ "Precisions": [ { "int4": "", - "int8": 16.59, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -340,19 +380,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 44.68, + "int8": "", "fp16": "", - "fp32": 17.96, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 59.06, + "token_int8": 47.96, + "token_fp16": 29.29 } ], "Unit": "FPS", @@ -361,9 +405,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 37.63, - "fp16": "", + "int4": 16.93, + "int8": 20.85, + "fp16": 34.14, "fp32": "", "bf16": "" } @@ -374,19 +418,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 60.67, + "int8": "", "fp16": "", - "fp32": 23.99, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 40.48, + "token_int8": 32.79, + "token_fp16": 20.67 } ], "Unit": "FPS", @@ -395,9 +443,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 24.02, - "fp16": "", + "int4": 24.7, + "int8": 30.49, + "fp16": 48.37, "fp32": "", "bf16": "" } @@ -408,19 +456,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 27.34, - "fp16": "", - "fp32": 18.04, - "bf16": "" + "int8": 1401.85, + "fp16": 1046.9, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -430,7 +482,7 @@ "Precisions": [ { "int4": "", - "int8": 30.86, + "int8": 1.42, "fp16": "", "fp32": "", "bf16": "" @@ -442,19 +494,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 32.78, - "fp16": "", - "fp32": 21.32, - "bf16": "" + "int8": 112.21, + "fp16": 73.01, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -464,7 +520,7 @@ "Precisions": [ { "int4": "", - "int8": 38.38, + "int8": 14.86, "fp16": "", "fp32": "", "bf16": "" @@ -476,19 +532,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 168.92, - "fp16": "", - "fp32": 67.23, - "bf16": "" + "int8": 1308.1, + "fp16": 1201.69, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -498,7 +558,7 @@ "Precisions": [ { "int4": "", - "int8": 10.73, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -510,19 +570,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "stable-diffusion-v1-5", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.48, + "int8": "", "fp16": "", - "fp32": 35.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -532,7 +596,7 @@ "Precisions": [ { "int4": "", - "int8": 26.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -544,19 +608,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 20.68, - "fp16": "", - "fp32": 14.76, - "bf16": "" + "int8": 517.1, + "fp16": 550.33, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -566,7 +634,7 @@ "Precisions": [ { "int4": "", - "int8": 49.95, + "int8": 3.21, "fp16": "", "fp32": "", "bf16": "" @@ -578,19 +646,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 218.51, + "int8": 23.3, "fp16": "", - "fp32": 80.07, - "bf16": "" + "fp32": 23.72, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -600,7 +672,7 @@ "Precisions": [ { "int4": "", - "int8": 14.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -612,19 +684,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 590.15, + "int8": 228.97, "fp16": "", - "fp32": 224.94, - "bf16": "" + "fp32": 219.37, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -634,7 +710,7 @@ "Precisions": [ { "int4": "", - "int8": 9.22, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -646,19 +722,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 881.6, + "int8": 59.38, "fp16": "", - "fp32": 338.79, - "bf16": "" + "fp32": 54.24, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -668,7 +748,7 @@ "Precisions": [ { "int4": "", - "int8": 5.08, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -680,19 +760,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3032.01, + "int8": 1.26, "fp16": "", - "fp32": 488.41, - "bf16": 1975.07 + "fp32": 1.08, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -702,10 +786,10 @@ "Precisions": [ { "int4": "", - "int8": 3.74, + "int8": "", "fp16": "", "fp32": "", - "bf16": 4.78 + "bf16": "" } ], "Unit": "ms", @@ -714,19 +798,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4693.95, + "int8": 111.92, "fp16": "", - "fp32": 562.05, - "bf16": 3202.49 + "fp32": 98.44, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -736,10 +824,10 @@ "Precisions": [ { "int4": "", - "int8": 3.77, + "int8": "", "fp16": "", "fp32": "", - "bf16": 4.61 + "bf16": "" } ], "Unit": "ms", @@ -748,19 +836,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 427.23, + "int8": "", "fp16": "", - "fp32": 164.01, - "bf16": "" + "fp32": 34.99, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -770,7 +862,7 @@ "Precisions": [ { "int4": "", - "int8": 11.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -782,19 +874,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 429.84, + "int8": 36.35, "fp16": "", - "fp32": 167.86, - "bf16": "" + "fp32": 33.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -804,7 +900,7 @@ "Precisions": [ { "int4": "", - "int8": 8.0, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -816,19 +912,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 76.06, + "int8": 7.26, "fp16": "", - "fp32": 30.37, - "bf16": "" + "fp32": 5.01, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -838,7 +938,7 @@ "Precisions": [ { "int4": "", - "int8": 25.96, + "int8": 139.68, "fp16": "", "fp32": "", "bf16": "" @@ -850,19 +950,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.79, + "int8": 134.16, "fp16": "", - "fp32": 21.03, - "bf16": "" + "fp32": 80.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -872,7 +976,7 @@ "Precisions": [ { "int4": "", - "int8": 32.08, + "int8": 7.8, "fp16": "", "fp32": "", "bf16": "" @@ -884,19 +988,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 73.09, + "int8": 19.87, "fp16": "", - "fp32": 26.35, - "bf16": "" + "fp32": 8.15, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -906,7 +1014,7 @@ "Precisions": [ { "int4": "", - "int8": 19.84, + "int8": 51.33, "fp16": "", "fp32": "", "bf16": "" @@ -918,19 +1026,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 404.72, - "fp16": 444.29, - "fp32": "", - "bf16": "" + "int8": 0.33, + "fp16": "", + "fp32": 0.13, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -940,7 +1052,7 @@ "Precisions": [ { "int4": "", - "int8": 3.24, + "int8": 2995.1, "fp16": "", "fp32": "", "bf16": "" @@ -952,19 +1064,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Parameters": { + "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 45.84, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 21.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -974,7 +1090,7 @@ "Precisions": [ { "int4": "", - "int8": 4.76, + "int8": 22.72, "fp16": "", "fp32": "", "bf16": "" @@ -986,19 +1102,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 187.77, - "fp16": 149.65, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 5.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1008,7 +1128,7 @@ "Precisions": [ { "int4": "", - "int8": 5.71, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1020,19 +1140,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 89.12, - "fp16": 74.2, - "fp32": "", - "bf16": "" + "int8": 10.31, + "fp16": "", + "fp32": 5.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1042,7 +1166,7 @@ "Precisions": [ { "int4": "", - "int8": 12.51, + "int8": 99.61, "fp16": "", "fp32": "", "bf16": "" @@ -1054,19 +1178,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 89.38, - "fp16": 74.26, + "int8": 22.02, + "fp16": 25.05, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1076,7 +1204,7 @@ "Precisions": [ { "int4": "", - "int8": 12.3, + "int8": 60.1, "fp16": "", "fp32": "", "bf16": "" @@ -1088,19 +1216,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 177.17, - "fp16": 139.65, + "int8": 187.37, + "fp16": 222.58, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1110,7 +1242,7 @@ "Precisions": [ { "int4": "", - "int8": 6.02, + "int8": 7.71, "fp16": "", "fp32": "", "bf16": "" @@ -1122,19 +1254,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", + "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 42.88, - "fp16": 33.61, + "int8": 48.1, + "fp16": 51.68, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1144,7 +1280,7 @@ "Precisions": [ { "int4": "", - "int8": 26.85, + "int8": 22.89, "fp16": "", "fp32": "", "bf16": "" @@ -1156,19 +1292,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 47.07, - "fp16": 39.06, + "int8": 1.16, + "fp16": 1.16, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1178,7 +1318,7 @@ "Precisions": [ { "int4": "", - "int8": 19.89, + "int8": 870.65, "fp16": "", "fp32": "", "bf16": "" @@ -1190,19 +1330,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 47.0, - "fp16": 39.61, + "int8": 93.36, + "fp16": 95.62, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1212,7 +1356,7 @@ "Precisions": [ { "int4": "", - "int8": 18.15, + "int8": 13.54, "fp16": "", "fp32": "", "bf16": "" @@ -1224,19 +1368,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 67.79, - "fp16": 52.83, + "int8": 31.79, + "fp16": 33.13, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1246,7 +1394,7 @@ "Precisions": [ { "int4": "", - "int8": 17.21, + "int8": 35.83, "fp16": "", "fp32": "", "bf16": "" @@ -1258,19 +1406,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 47.29, - "fp16": 40.73, - "fp32": "", - "bf16": "" + "int8": 39.3, + "fp16": "", + "fp32": 28.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1280,7 +1432,7 @@ "Precisions": [ { "int4": "", - "int8": 21.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1292,19 +1444,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 88.92, - "fp16": 69.92, - "fp32": "", - "bf16": "" + "int8": 480.45, + "fp16": "", + "fp32": 302.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1314,7 +1470,7 @@ "Precisions": [ { "int4": "", - "int8": 12.82, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1326,19 +1482,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 66.18, - "fp16": 52.8, - "fp32": "", - "bf16": "" + "int8": 129.7, + "fp16": "", + "fp32": 54.69, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1348,7 +1508,7 @@ "Precisions": [ { "int4": "", - "int8": 14.46, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1360,19 +1520,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.49, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 0.86, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1382,7 +1546,7 @@ "Precisions": [ { "int4": "", - "int8": 12.42, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1394,19 +1558,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 159.37, - "fp16": 100.87, - "fp32": "", - "bf16": "" + "int8": 233.16, + "fp16": "", + "fp32": 114.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1416,7 +1584,7 @@ "Precisions": [ { "int4": "", - "int8": 6.11, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1428,19 +1596,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 152.5, - "fp16": 108.49, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 41.37, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1450,7 +1622,7 @@ "Precisions": [ { "int4": "", - "int8": 7.14, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1462,19 +1634,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 316.53, - "fp16": 268.1, - "fp32": "", - "bf16": "" + "int8": 67.73, + "fp16": "", + "fp32": 36.05, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1484,7 +1660,7 @@ "Precisions": [ { "int4": "", - "int8": 4.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1496,19 +1672,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 44.78, + "int8": 14.29, "fp16": "", - "fp32": 33.39, - "bf16": "" + "fp32": 11.18, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1518,7 +1698,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 71.84, "fp16": "", "fp32": "", "bf16": "" @@ -1530,19 +1710,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 273.98, "fp16": "", - "fp32": 16.3, - "bf16": "" + "fp32": 169.54, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1552,7 +1736,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 4.05, "fp16": "", "fp32": "", "bf16": "" @@ -1564,19 +1748,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 49.58, + "int8": 45.27, "fp16": "", - "fp32": 26.72, - "bf16": "" + "fp32": 18.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1586,7 +1774,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 23.76, "fp16": "", "fp32": "", "bf16": "" @@ -1598,19 +1786,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 85.59, + "int8": 0.76, "fp16": "", - "fp32": 51.66, - "bf16": "" + "fp32": 0.31, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1620,7 +1812,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1317.43, "fp16": "", "fp32": "", "bf16": "" @@ -1632,19 +1824,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 48.88, + "int8": 98.2, "fp16": "", - "fp32": 25.61, - "bf16": "" + "fp32": 45.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1654,7 +1850,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 10.52, "fp16": "", "fp32": "", "bf16": "" @@ -1666,10 +1862,11 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ @@ -1677,8 +1874,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 13.77, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1700,19 +1900,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 61.03, + "int8": 21.58, "fp16": "", - "fp32": 32.25, - "bf16": "" + "fp32": 11.78, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1722,7 +1926,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 47.39, "fp16": "", "fp32": "", "bf16": "" @@ -1734,19 +1938,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 46.74, - "bf16": "" + "int8": 40.0, + "fp16": 34.31, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1756,7 +1964,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 34.13, "fp16": "", "fp32": "", "bf16": "" @@ -1768,19 +1976,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 240.55, - "fp16": "", - "fp32": 157.84, - "bf16": "" + "int8": 414.66, + "fp16": 324.8, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1790,7 +2002,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.49, "fp16": "", "fp32": "", "bf16": "" @@ -1802,19 +2014,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 73.76, - "bf16": "" + "int8": 106.34, + "fp16": 64.69, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1824,7 +2040,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 10.56, "fp16": "", "fp32": "", "bf16": "" @@ -1836,19 +2052,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.18, - "fp16": "", - "fp32": 0.38, - "bf16": "" + "int8": 2.16, + "fp16": 1.32, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1858,7 +2078,7 @@ "Precisions": [ { "int4": "", - "int8": 853.85, + "int8": 472.59, "fp16": "", "fp32": "", "bf16": "" @@ -1870,19 +2090,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.0, - "fp16": "", - "fp32": 1.27, - "bf16": "" + "int8": 211.07, + "fp16": 137.13, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1892,7 +2116,7 @@ "Precisions": [ { "int4": "", - "int8": 510.0, + "int8": 6.2, "fp16": "", "fp32": "", "bf16": "" @@ -1904,19 +2128,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.05, - "fp16": "", - "fp32": 1.87, - "bf16": "" + "int8": 60.92, + "fp16": 44.64, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1926,7 +2154,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 18.51, "fp16": "", "fp32": "", "bf16": "" @@ -1938,19 +2166,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 45.34, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 33.5, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -1960,7 +2192,7 @@ "Precisions": [ { "int4": "", - "int8": 424.59, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -1972,19 +2204,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 57.78, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 48.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2006,19 +2242,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.56, "fp16": "", - "fp32": 3.91, - "bf16": "" + "fp32": 0.51, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2028,7 +2268,7 @@ "Precisions": [ { "int4": "", - "int8": 128.89, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2040,19 +2280,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.01, + "int8": 525.47, "fp16": "", - "fp32": 1.89, - "bf16": "" + "fp32": 392.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2062,7 +2306,7 @@ "Precisions": [ { "int4": "", - "int8": 324.87, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2074,19 +2318,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.03, + "int8": 197.41, "fp16": "", - "fp32": 1.64, - "bf16": "" + "fp32": 115.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2096,7 +2344,7 @@ "Precisions": [ { "int4": "", - "int8": 198.73, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2108,19 +2356,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.69, + "int8": 5.38, "fp16": "", - "fp32": 1.2, - "bf16": "" + "fp32": 2.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2130,7 +2382,7 @@ "Precisions": [ { "int4": "", - "int8": 268.0, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2142,19 +2394,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.52, + "int8": 316.13, "fp16": "", - "fp32": 2.85, - "bf16": "" + "fp32": 194.29, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2176,10 +2432,11 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ @@ -2187,8 +2444,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 80.2, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2198,7 +2458,7 @@ "Precisions": [ { "int4": "", - "int8": 360.08, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2210,19 +2470,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.26, + "int8": 114.67, "fp16": "", - "fp32": 2.05, - "bf16": "" + "fp32": 78.26, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2244,19 +2508,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.77, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 4.32, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2266,7 +2534,7 @@ "Precisions": [ { "int4": "", - "int8": 329.86, + "int8": 87.73, "fp16": "", "fp32": "", "bf16": "" @@ -2278,19 +2546,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.14, + "int8": 18.94, "fp16": "", - "fp32": 1.9, - "bf16": "" + "fp32": 11.49, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2300,7 +2572,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 55.76, "fp16": "", "fp32": "", "bf16": "" @@ -2312,19 +2584,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 16.05, + "int8": 0.17, "fp16": "", - "fp32": 5.99, - "bf16": "" + "fp32": 0.04, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2334,7 +2610,7 @@ "Precisions": [ { "int4": "", - "int8": 94.97, + "int8": 5772.15, "fp16": "", "fp32": "", "bf16": "" @@ -2346,19 +2622,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.17, + "int8": 301.05, "fp16": "", - "fp32": 3.33, - "bf16": "" + "fp32": 132.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2368,7 +2648,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.6, "fp16": "", "fp32": "", "bf16": "" @@ -2380,19 +2660,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 51.66, "fp16": "", - "fp32": 1.27, - "bf16": "" + "fp32": 14.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2402,7 +2686,7 @@ "Precisions": [ { "int4": "", - "int8": 513.28, + "int8": 19.8, "fp16": "", "fp32": "", "bf16": "" @@ -2414,19 +2698,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.82, + "int8": 0.89, "fp16": "", - "fp32": 6.98, - "bf16": "" + "fp32": 0.23, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2436,7 +2724,7 @@ "Precisions": [ { "int4": "", - "int8": 102.33, + "int8": 1118.71, "fp16": "", "fp32": "", "bf16": "" @@ -2448,19 +2736,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 51.17, + "int8": 115.03, "fp16": "", - "fp32": 18.46, - "bf16": "" + "fp32": 36.99, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2470,7 +2762,7 @@ "Precisions": [ { "int4": "", - "int8": 48.78, + "int8": 9.06, "fp16": "", "fp32": "", "bf16": "" @@ -2482,19 +2774,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 78.31, + "int8": "", "fp16": "", - "fp32": 29.72, - "bf16": "" + "fp32": 11.94, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2504,7 +2800,7 @@ "Precisions": [ { "int4": "", - "int8": 38.13, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2516,19 +2812,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 244.06, + "int8": 25.97, "fp16": "", - "fp32": 41.97, - "bf16": 211.62 + "fp32": 9.66, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2538,10 +2838,10 @@ "Precisions": [ { "int4": "", - "int8": 25.21, + "int8": 40.21, "fp16": "", "fp32": "", - "bf16": 27.63 + "bf16": "" } ], "Unit": "ms", @@ -2550,19 +2850,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 305.38, - "fp16": "", - "fp32": 55.37, - "bf16": 289.16 + "int8": 43.69, + "fp16": 33.8, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2572,10 +2876,10 @@ "Precisions": [ { "int4": "", - "int8": 19.29, + "int8": 26.56, "fp16": "", "fp32": "", - "bf16": 25.79 + "bf16": "" } ], "Unit": "ms", @@ -2584,19 +2888,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38.28, - "fp16": "", - "fp32": 13.45, - "bf16": "" + "int8": 73.58, + "fp16": 58.53, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2606,7 +2914,7 @@ "Precisions": [ { "int4": "", - "int8": 69.24, + "int8": 25.45, "fp16": "", "fp32": "", "bf16": "" @@ -2618,19 +2926,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38.81, - "fp16": "", - "fp32": 15.09, - "bf16": "" + "int8": 0.48, + "fp16": 0.52, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2640,7 +2952,7 @@ "Precisions": [ { "int4": "", - "int8": 62.64, + "int8": 2110.65, "fp16": "", "fp32": "", "bf16": "" @@ -2652,19 +2964,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.34, - "fp16": "", - "fp32": 2.49, - "bf16": "" + "int8": 671.35, + "fp16": 504.8, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2674,7 +2990,7 @@ "Precisions": [ { "int4": "", - "int8": 194.83, + "int8": 2.72, "fp16": "", "fp32": "", "bf16": "" @@ -2686,19 +3002,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.5, - "fp16": "", - "fp32": 1.72, - "bf16": "" + "int8": 203.17, + "fp16": 118.59, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2708,7 +3028,7 @@ "Precisions": [ { "int4": "", - "int8": 246.14, + "int8": 6.3, "fp16": "", "fp32": "", "bf16": "" @@ -2720,19 +3040,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.21, - "fp16": "", - "fp32": 2.31, - "bf16": "" + "int8": 5.09, + "fp16": 2.78, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2742,7 +3066,7 @@ "Precisions": [ { "int4": "", - "int8": 195.21, + "int8": 210.41, "fp16": "", "fp32": "", "bf16": "" @@ -2754,19 +3078,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 121.51, - "fp16": 110.4, + "int8": 396.07, + "fp16": 221.18, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2776,7 +3104,7 @@ "Precisions": [ { "int4": "", - "int8": 9.1, + "int8": 4.3, "fp16": "", "fp32": "", "bf16": "" @@ -2788,19 +3116,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 121.77, + "fp16": 81.6, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2810,7 +3142,7 @@ "Precisions": [ { "int4": "", - "int8": 12.35, + "int8": 10.34, "fp16": "", "fp32": "", "bf16": "" @@ -2822,19 +3154,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 30.58, - "fp16": 21.73, - "fp32": "", - "bf16": "" + "int8": 243.99, + "fp16": "", + "fp32": 157.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2844,7 +3180,7 @@ "Precisions": [ { "int4": "", - "int8": 32.8, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2856,19 +3192,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10.38, - "fp16": 6.72, - "fp32": "", - "bf16": "" + "int8": 189.52, + "fp16": "", + "fp32": 154.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2878,7 +3218,7 @@ "Precisions": [ { "int4": "", - "int8": 97.03, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2890,19 +3230,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10.38, - "fp16": 6.74, - "fp32": "", - "bf16": "" + "int8": 2.45, + "fp16": "", + "fp32": 1.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2912,7 +3256,7 @@ "Precisions": [ { "int4": "", - "int8": 97.28, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2924,19 +3268,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 32.32, + "int8": 4485.9, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 2415.8, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2946,7 +3294,7 @@ "Precisions": [ { "int4": "", - "int8": 43.5, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2958,19 +3306,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.91, + "int8": 1097.16, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 475.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -2980,7 +3332,7 @@ "Precisions": [ { "int4": "", - "int8": 329.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -2992,19 +3344,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 18.81, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 9.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3014,7 +3370,7 @@ "Precisions": [ { "int4": "", - "int8": 720.67, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3026,19 +3382,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.47, - "fp16": 3.64, - "fp32": "", - "bf16": "" + "int8": 1120.99, + "fp16": "", + "fp32": 624.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3048,7 +3408,7 @@ "Precisions": [ { "int4": "", - "int8": 192.94, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3060,19 +3420,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.41, - "fp16": 4.03, - "fp32": "", - "bf16": "" + "int8": 374.74, + "fp16": "", + "fp32": 236.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3082,7 +3446,7 @@ "Precisions": [ { "int4": "", - "int8": 173.57, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3094,19 +3458,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.17, - "fp16": 4.28, - "fp32": "", - "bf16": "" + "int8": 76.15, + "fp16": "", + "fp32": 30.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3116,7 +3484,7 @@ "Precisions": [ { "int4": "", - "int8": 137.83, + "int8": 25.21, "fp16": "", "fp32": "", "bf16": "" @@ -3128,19 +3496,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.85, - "fp16": 6.59, - "fp32": "", - "bf16": "" + "int8": 97.68, + "fp16": "", + "fp32": 66.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3150,7 +3522,7 @@ "Precisions": [ { "int4": "", - "int8": 106.93, + "int8": 22.16, "fp16": "", "fp32": "", "bf16": "" @@ -3162,19 +3534,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.41, - "fp16": 4.47, - "fp32": "", - "bf16": "" + "int8": 1.2, + "fp16": "", + "fp32": 0.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3184,7 +3560,7 @@ "Precisions": [ { "int4": "", - "int8": 179.75, + "int8": 1025.52, "fp16": "", "fp32": "", "bf16": "" @@ -3196,19 +3572,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 11.76, - "fp16": 8.39, - "fp32": "", - "bf16": "" + "int8": 1969.75, + "fp16": "", + "fp32": 815.83, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3218,7 +3598,7 @@ "Precisions": [ { "int4": "", - "int8": 87.43, + "int8": 1.36, "fp16": "", "fp32": "", "bf16": "" @@ -3230,19 +3610,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.08, - "fp16": 5.89, - "fp32": "", - "bf16": "" + "int8": 390.17, + "fp16": "", + "fp32": 94.82, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3252,7 +3636,7 @@ "Precisions": [ { "int4": "", - "int8": 110.63, + "int8": 6.23, "fp16": "", "fp32": "", "bf16": "" @@ -3264,19 +3648,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 6.38, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1.6, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3286,7 +3674,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 209.14, "fp16": "", "fp32": "", "bf16": "" @@ -3298,19 +3686,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.39, + "int8": 685.79, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 242.78, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3320,7 +3712,7 @@ "Precisions": [ { "int4": "", - "int8": 408.09, + "int8": 2.71, "fp16": "", "fp32": "", "bf16": "" @@ -3332,19 +3724,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 23.39, - "fp16": 14.13, - "fp32": "", - "bf16": "" + "int8": 166.55, + "fp16": "", + "fp32": 64.31, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3354,7 +3750,7 @@ "Precisions": [ { "int4": "", - "int8": 39.66, + "int8": 12.75, "fp16": "", "fp32": "", "bf16": "" @@ -3366,19 +3762,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 88.41, + "fp16": 74.04, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3388,7 +3788,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 12.15, "fp16": "", "fp32": "", "bf16": "" @@ -3400,19 +3800,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 54.9, - "fp16": 49.62, + "int8": 37.81, + "fp16": 34.74, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3422,7 +3826,7 @@ "Precisions": [ { "int4": "", - "int8": 20.12, + "int8": 27.47, "fp16": "", "fp32": "", "bf16": "" @@ -3434,19 +3838,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.85, + "int8": "", "fp16": "", - "fp32": 3.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 0.27, + "token_fp16": 2.55 } ], "Unit": "FPS", @@ -3456,8 +3864,8 @@ "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 3688.24, + "fp16": 390.94, "fp32": "", "bf16": "" } @@ -3468,19 +3876,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 1.5, - "bf16": "" + "int8": 1966.11, + "fp16": 1346.18, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3490,7 +3902,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.79, "fp16": "", "fp32": "", "bf16": "" @@ -3502,19 +3914,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.12, + "int8": "", "fp16": "", - "fp32": 2.56, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 3.7, + "token_int8": 0.49, + "token_fp16": 3.91 } ], "Unit": "FPS", @@ -3523,9 +3939,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 269.82, + "int8": 2003.58, + "fp16": 255.57, "fp32": "", "bf16": "" } @@ -3536,19 +3952,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 9.46, - "fp16": "", - "fp32": 5.5, - "bf16": "" + "int8": 771.23, + "fp16": 382.83, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3558,7 +3978,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.58, "fp16": "", "fp32": "", "bf16": "" @@ -3570,19 +3990,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.05, - "fp16": "", - "fp32": 2.57, - "bf16": "" + "int8": 705.76, + "fp16": 453.35, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3592,7 +4016,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.67, "fp16": "", "fp32": "", "bf16": "" @@ -3604,19 +4028,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", - "fp32": 5.4, - "bf16": "" + "int8": 126.18, + "fp16": 129.18, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3626,7 +4054,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.71, "fp16": "", "fp32": "", "bf16": "" @@ -3638,19 +4066,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.41, - "fp16": "", - "fp32": 3.19, - "bf16": "" + "int8": 164.18, + "fp16": 107.12, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3672,19 +4104,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.01, - "fp16": "", - "fp32": 4.49, - "bf16": "" + "int8": 195.27, + "fp16": 164.33, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3706,19 +4142,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 22.64, + "int8": "", "fp16": "", - "fp32": 12.18, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 8.94, + "token_int8": "", + "token_fp16": 0.94 } ], "Unit": "FPS", @@ -3727,9 +4167,9 @@ "latency": { "Precisions": [ { - "int4": "", + "int4": 111.74, "int8": "", - "fp16": "", + "fp16": 1056.4, "fp32": "", "bf16": "" } @@ -3740,19 +4180,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.39, + "int8": "", "fp16": "", - "fp32": 6.33, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 10.82, + "token_int8": 6.3, + "token_fp16": 1.1 } ], "Unit": "FPS", @@ -3761,9 +4205,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 92.41, + "int8": 158.68, + "fp16": 906.89, "fp32": "", "bf16": "" } @@ -3774,19 +4218,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 14.28, + "int8": "", "fp16": "", - "fp32": 11.21, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 14.62, + "token_int8": 8.53, + "token_fp16": "" } ], "Unit": "FPS", @@ -3795,8 +4243,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 71.92, + "int4": 68.39, + "int8": 117.1, "fp16": "", "fp32": "", "bf16": "" @@ -3808,19 +4256,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.25, + "int8": "", "fp16": "", - "fp32": 4.96, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 14.82, + "token_int8": 7.84, + "token_fp16": 4.04 } ], "Unit": "FPS", @@ -3829,9 +4281,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 139.91, - "fp16": "", + "int4": 67.44, + "int8": 127.51, + "fp16": 247.29, "fp32": "", "bf16": "" } @@ -3842,19 +4294,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 18.99, + "int8": "", "fp16": "", - "fp32": 11.5, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.17, + "token_int8": 20.38, + "token_fp16": 10.76 } ], "Unit": "FPS", @@ -3863,9 +4319,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 55.68, - "fp16": "", + "int4": 38.21, + "int8": 49.06, + "fp16": 92.92, "fp32": "", "bf16": "" } @@ -3876,19 +4332,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.48, - "fp16": "", - "fp32": 27.9, - "bf16": "" + "int8": 2.35, + "fp16": 1.58, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3898,7 +4358,7 @@ "Precisions": [ { "int4": "", - "int8": 27.49, + "int8": 421.72, "fp16": "", "fp32": "", "bf16": "" @@ -3910,19 +4370,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 59.12, + "int8": "", "fp16": "", - "fp32": 32.99, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 15.03, + "token_int8": 8.94, + "token_fp16": "" } ], "Unit": "FPS", @@ -3931,8 +4395,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 20.56, + "int4": 66.52, + "int8": 111.8, "fp16": "", "fp32": "", "bf16": "" @@ -3944,19 +4408,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 44.97, - "fp16": "", - "fp32": 24.31, - "bf16": "" + "int8": 1293.98, + "fp16": 1371.59, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -3966,7 +4434,7 @@ "Precisions": [ { "int4": "", - "int8": 32.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -3978,19 +4446,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 56.87, + "int8": "", "fp16": "", - "fp32": 36.06, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 23.61, + "token_int8": 18.01, + "token_fp16": 9.36 } ], "Unit": "FPS", @@ -3999,9 +4471,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 28.54, - "fp16": "", + "int4": 42.34, + "int8": 55.51, + "fp16": 106.82, "fp32": "", "bf16": "" } @@ -4012,19 +4484,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 152.17, + "int8": "", "fp16": "", - "fp32": 93.19, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 16.68, + "token_int8": 9.5, + "token_fp16": "" } ], "Unit": "FPS", @@ -4033,8 +4509,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 9.14, + "int4": 59.95, + "int8": 105.26, "fp16": "", "fp32": "", "bf16": "" @@ -4046,19 +4522,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 59.7, - "fp16": "", - "fp32": 42.45, - "bf16": "" + "int8": 563.96, + "fp16": 416.13, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4068,7 +4548,7 @@ "Precisions": [ { "int4": "", - "int8": 17.62, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4080,19 +4560,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 75.02, - "fp16": "", - "fp32": 41.47, - "bf16": "" + "int8": 21.26, + "fp16": 12.84, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4102,7 +4586,7 @@ "Precisions": [ { "int4": "", - "int8": 14.65, + "int8": 47.61, "fp16": "", "fp32": "", "bf16": "" @@ -4114,19 +4598,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.75, - "fp16": "", - "fp32": 21.83, - "bf16": "" + "int8": 1030.66, + "fp16": 811.13, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4136,7 +4624,7 @@ "Precisions": [ { "int4": "", - "int8": 20.19, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4148,19 +4636,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 112.1, - "fp16": "", - "fp32": 61.82, - "bf16": "" + "int8": 403.44, + "fp16": 306.22, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4170,7 +4662,7 @@ "Precisions": [ { "int4": "", - "int8": 11.47, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4182,19 +4674,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 62.08, + "int8": 223.99, "fp16": "", - "fp32": 39.8, - "bf16": "" + "fp32": 189.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4204,7 +4700,7 @@ "Precisions": [ { "int4": "", - "int8": 26.84, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4216,19 +4712,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", "Model": "efficientdet-d0", - "Checked": "true", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 83.44, + "int8": 174.87, "fp16": "", - "fp32": 43.76, - "bf16": "" + "fp32": 149.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4238,7 +4738,7 @@ "Precisions": [ { "int4": "", - "int8": 16.75, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4250,19 +4750,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 52.39, + "int8": 7.24, "fp16": "", - "fp32": 38.12, - "bf16": "" + "fp32": 3.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4272,7 +4776,7 @@ "Precisions": [ { "int4": "", - "int8": 17.79, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4284,19 +4788,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 65.23, + "int8": 4846.91, "fp16": "", - "fp32": 41.09, - "bf16": "" + "fp32": 2888.98, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4306,7 +4814,7 @@ "Precisions": [ { "int4": "", - "int8": 18.79, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4318,19 +4826,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 220.01, + "int8": 1975.45, "fp16": "", - "fp32": 126.72, - "bf16": "" + "fp32": 922.35, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4340,7 +4852,7 @@ "Precisions": [ { "int4": "", - "int8": 7.33, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4352,19 +4864,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 15.45, + "int8": "", "fp16": "", - "fp32": 12.76, - "bf16": "" + "fp32": 20.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4374,7 +4890,7 @@ "Precisions": [ { "int4": "", - "int8": 66.3, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4386,19 +4902,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 97.27, + "int8": "", "fp16": "", - "fp32": 48.02, - "bf16": "" + "fp32": 585.46, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4408,7 +4928,7 @@ "Precisions": [ { "int4": "", - "int8": 13.26, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4420,19 +4940,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 35.92, + "int8": 343.07, "fp16": "", - "fp32": 30.28, - "bf16": "" + "fp32": 274.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4442,7 +4966,7 @@ "Precisions": [ { "int4": "", - "int8": 28.54, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -4454,19 +4978,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 271.61, + "int8": 44.06, "fp16": "", - "fp32": 166.53, - "bf16": "" + "fp32": 16.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4476,7 +5004,7 @@ "Precisions": [ { "int4": "", - "int8": 11.3, + "int8": 41.27, "fp16": "", "fp32": "", "bf16": "" @@ -4488,19 +5016,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 576.89, + "int8": 53.32, "fp16": "", - "fp32": 325.0, - "bf16": "" + "fp32": 38.06, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4510,7 +5042,7 @@ "Precisions": [ { "int4": "", - "int8": 7.03, + "int8": 28.44, "fp16": "", "fp32": "", "bf16": "" @@ -4522,19 +5054,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1014.6, + "int8": 0.65, "fp16": "", - "fp32": 582.63, - "bf16": "" + "fp32": 0.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4544,7 +5080,7 @@ "Precisions": [ { "int4": "", - "int8": 4.2, + "int8": 2598.78, "fp16": "", "fp32": "", "bf16": "" @@ -4556,19 +5092,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1454.4, + "int8": 917.84, "fp16": "", - "fp32": 872.52, - "bf16": 1037.91 + "fp32": 490.87, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4578,10 +5118,10 @@ "Precisions": [ { "int4": "", - "int8": 4.6, + "int8": 2.07, "fp16": "", "fp32": "", - "bf16": 4.96 + "bf16": "" } ], "Unit": "ms", @@ -4590,19 +5130,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1736.65, + "int8": 194.09, "fp16": "", - "fp32": 1132.21, - "bf16": 1408.76 + "fp32": 52.09, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4612,10 +5156,10 @@ "Precisions": [ { "int4": "", - "int8": 4.6, + "int8": 9.58, "fp16": "", "fp32": "", - "bf16": 4.68 + "bf16": "" } ], "Unit": "ms", @@ -4624,19 +5168,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 413.78, + "int8": 3.52, "fp16": "", - "fp32": 257.07, - "bf16": "" + "fp32": 0.87, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4646,7 +5194,7 @@ "Precisions": [ { "int4": "", - "int8": 8.54, + "int8": 493.86, "fp16": "", "fp32": "", "bf16": "" @@ -4658,19 +5206,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 497.16, + "int8": 380.37, "fp16": "", - "fp32": 303.25, - "bf16": "" + "fp32": 135.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4680,7 +5232,7 @@ "Precisions": [ { "int4": "", - "int8": 5.86, + "int8": 4.64, "fp16": "", "fp32": "", "bf16": "" @@ -4692,19 +5244,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 97.68, + "int8": 80.52, "fp16": "", - "fp32": 67.18, - "bf16": "" + "fp32": 34.88, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4714,7 +5270,7 @@ "Precisions": [ { "int4": "", - "int8": 21.46, + "int8": 20.34, "fp16": "", "fp32": "", "bf16": "" @@ -4726,19 +5282,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 70.93, - "fp16": "", - "fp32": 46.39, - "bf16": "" + "int8": 265.97, + "fp16": 198.16, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4748,7 +5308,7 @@ "Precisions": [ { "int4": "", - "int8": 25.46, + "int8": 5.25, "fp16": "", "fp32": "", "bf16": "" @@ -4760,19 +5320,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", "Model": "efficientdet-d0", - "Checked": "true", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 92.57, - "fp16": "", - "fp32": 61.6, - "bf16": "" + "int8": 13.69, + "fp16": 13.65, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4782,7 +5346,7 @@ "Precisions": [ { "int4": "", - "int8": 13.58, + "int8": 119.56, "fp16": "", "fp32": "", "bf16": "" @@ -4794,19 +5358,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 456.95, - "fp16": 402.8, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": 0.24, + "token_fp16": 4.4 } ], "Unit": "FPS", @@ -4816,8 +5384,8 @@ "Precisions": [ { "int4": "", - "int8": 2.93, - "fp16": "", + "int8": 4094.9, + "fp16": 226.87, "fp32": "", "bf16": "" } @@ -4828,19 +5396,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 3799.36, + "fp16": 3178.95, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4850,7 +5422,7 @@ "Precisions": [ { "int4": "", - "int8": 4.8, + "int8": 0.46, "fp16": "", "fp32": "", "bf16": "" @@ -4862,19 +5434,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 147.83, - "fp16": 120.48, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 3.31, + "token_int8": 0.72, + "token_fp16": 6.86 } ], "Unit": "FPS", @@ -4883,9 +5459,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 9.67, - "fp16": "", + "int4": 301.49, + "int8": 1378.29, + "fp16": 145.76, "fp32": "", "bf16": "" } @@ -4896,19 +5472,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.71, - "fp16": 34.85, + "int8": 2161.26, + "fp16": 948.32, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4918,7 +5498,7 @@ "Precisions": [ { "int4": "", - "int8": 27.94, + "int8": 0.79, "fp16": "", "fp32": "", "bf16": "" @@ -4930,19 +5510,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.99, - "fp16": 34.96, + "int8": 230.18, + "fp16": 192.78, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4952,7 +5536,7 @@ "Precisions": [ { "int4": "", - "int8": 27.51, + "int8": 8.29, "fp16": "", "fp32": "", "bf16": "" @@ -4964,19 +5548,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 13.38, - "fp16": 13.66, + "int8": 401.12, + "fp16": 497.56, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -4986,7 +5574,7 @@ "Precisions": [ { "int4": "", - "int8": 124.82, + "int8": 3.97, "fp16": "", "fp32": "", "bf16": "" @@ -4998,19 +5586,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 40.02, - "fp16": 34.39, + "int8": 225.83, + "fp16": 298.39, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5020,7 +5612,7 @@ "Precisions": [ { "int4": "", - "int8": 34.08, + "int8": 3.93, "fp16": "", "fp32": "", "bf16": "" @@ -5032,19 +5624,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.7, - "fp16": 25.09, + "int8": 114.57, + "fp16": 121.87, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5054,7 +5650,7 @@ "Precisions": [ { "int4": "", - "int8": 63.02, + "int8": 10.22, "fp16": "", "fp32": "", "bf16": "" @@ -5066,19 +5662,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 73.6, - "fp16": 58.55, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 14.49, + "token_int8": 8.34, + "token_fp16": 0.59 } ], "Unit": "FPS", @@ -5087,9 +5687,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 24.96, - "fp16": "", + "int4": 68.99, + "int8": 119.77, + "fp16": 1691.52, "fp32": "", "bf16": "" } @@ -5100,19 +5700,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 68.83, - "fp16": 51.74, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 17.63, + "token_int8": 9.8, + "token_fp16": 0.71 } ], "Unit": "FPS", @@ -5121,9 +5725,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 19.37, - "fp16": "", + "int4": 56.72, + "int8": 102.04, + "fp16": 1402.74, "fp32": "", "bf16": "" } @@ -5134,19 +5738,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 80.63, - "fp16": 60.04, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 18.48, + "token_int8": 11.87, + "token_fp16": 6.44 } ], "Unit": "FPS", @@ -5155,9 +5763,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 14.48, - "fp16": "", + "int4": 54.09, + "int8": 84.18, + "fp16": 155.17, "fp32": "", "bf16": "" } @@ -5168,19 +5776,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 91.41, + "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 20.41, + "token_int8": 11.07, + "token_fp16": 5.81 } ], "Unit": "FPS", @@ -5189,9 +5801,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 48.98, + "int8": 90.29, + "fp16": 171.98, "fp32": "", "bf16": "" } @@ -5202,19 +5814,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 56.47, - "fp16": 42.15, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 36.58, + "token_int8": 23.94, + "token_fp16": 12.86 } ], "Unit": "FPS", @@ -5223,9 +5839,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 24.14, - "fp16": "", + "int4": 27.33, + "int8": 41.77, + "fp16": 77.71, "fp32": "", "bf16": "" } @@ -5236,19 +5852,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 128.8, - "fp16": 97.86, + "int8": 10.4, + "fp16": 5.7, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5258,7 +5878,7 @@ "Precisions": [ { "int4": "", - "int8": 12.63, + "int8": 109.21, "fp16": "", "fp32": "", "bf16": "" @@ -5270,19 +5890,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 100.05, - "fp16": 74.24, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 20.06, + "token_int8": 11.6, + "token_fp16": 6.05 } ], "Unit": "FPS", @@ -5291,9 +5915,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 13.33, - "fp16": "", + "int4": 49.85, + "int8": 86.18, + "fp16": 165.15, "fp32": "", "bf16": "" } @@ -5304,19 +5928,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 1007.75, + "fp16": 862.8, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5326,7 +5954,7 @@ "Precisions": [ { "int4": "", - "int8": 14.0, + "int8": 1.2, "fp16": "", "fp32": "", "bf16": "" @@ -5338,19 +5966,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 33.67, - "fp16": 30.88, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 31.27, + "token_int8": 20.55, + "token_fp16": 11.04 } ], "Unit": "FPS", @@ -5359,9 +5991,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 38.07, - "fp16": "", + "int4": 31.97, + "int8": 48.66, + "fp16": 90.57, "fp32": "", "bf16": "" } @@ -5372,19 +6004,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 201.37, - "fp16": 162.08, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 20.99, + "token_int8": 12.69, + "token_fp16": 6.07 } ], "Unit": "FPS", @@ -5393,9 +6029,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 6.86, - "fp16": "", + "int4": 47.64, + "int8": 78.78, + "fp16": 164.54, "fp32": "", "bf16": "" } @@ -5406,19 +6042,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 169.74, - "fp16": 143.49, + "int8": 830.46, + "fp16": 585.38, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5428,7 +6068,7 @@ "Precisions": [ { "int4": "", - "int8": 7.91, + "int8": 1.23, "fp16": "", "fp32": "", "bf16": "" @@ -5440,19 +6080,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 163.17, - "fp16": 185.29, + "int8": 57.99, + "fp16": 32.18, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5462,7 +6106,7 @@ "Precisions": [ { "int4": "", - "int8": 8.23, + "int8": 26.21, "fp16": "", "fp32": "", "bf16": "" @@ -5474,19 +6118,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39.25, - "fp16": "", - "fp32": 29.57, - "bf16": "" + "int8": 485.85, + "fp16": 555.71, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5496,7 +6144,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.75, "fp16": "", "fp32": "", "bf16": "" @@ -5508,19 +6156,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 22.93, - "fp16": "", - "fp32": 23.89, - "bf16": "" + "int8": 362.75, + "fp16": 375.06, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5530,7 +6182,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.3, "fp16": "", "fp32": "", "bf16": "" @@ -5542,19 +6194,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 57.64, + "int8": 34.21, "fp16": "", - "fp32": 48.78, - "bf16": "" + "fp32": 15.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5578,17 +6234,21 @@ { "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", "Model": "efficientdet-d0", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 47.95, "fp16": "", - "fp32": 30.78, - "bf16": "" + "fp32": 29.38, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5610,19 +6270,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 74.05, + "int8": 0.5, "fp16": "", - "fp32": 48.82, - "bf16": "" + "fp32": 0.18, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5644,19 +6308,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 100.6, + "int8": 742.67, "fp16": "", - "fp32": 65.57, - "bf16": "" + "fp32": 331.98, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5678,19 +6346,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 57.55, + "int8": 162.84, "fp16": "", - "fp32": 28.28, - "bf16": "" + "fp32": 51.66, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5712,9 +6384,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -5723,8 +6396,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 75.36, - "bf16": "" + "fp32": 1.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5746,19 +6422,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 88.7, + "int8": 328.29, "fp16": "", - "fp32": 59.09, - "bf16": "" + "fp32": 115.41, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5780,9 +6460,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -5791,8 +6472,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 59.29, - "bf16": "" + "fp32": 41.68, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5814,19 +6498,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 37.02, + "int8": 79.4, "fp16": "", - "fp32": 28.15, - "bf16": "" + "fp32": 35.44, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5848,19 +6536,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 191.27, + "int8": 31.55, "fp16": "", - "fp32": 150.88, - "bf16": "" + "fp32": 12.38, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5870,7 +6562,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 46.55, "fp16": "", "fp32": "", "bf16": "" @@ -5882,19 +6574,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", "Model": "efficientdet-d0", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 123.24, + "int8": 43.39, "fp16": "", - "fp32": 97.13, - "bf16": "" + "fp32": 23.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5904,7 +6600,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 33.09, "fp16": "", "fp32": "", "bf16": "" @@ -5916,19 +6612,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.17, + "int8": 0.45, "fp16": "", - "fp32": 0.04, - "bf16": "" + "fp32": 0.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5938,7 +6638,7 @@ "Precisions": [ { "int4": "", - "int8": 5769.81, + "int8": 2440.72, "fp16": "", "fp32": "", "bf16": "" @@ -5950,19 +6650,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.3, + "int8": 789.02, "fp16": "", - "fp32": 0.14, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -5972,7 +6676,7 @@ "Precisions": [ { "int4": "", - "int8": 3268.87, + "int8": 1.9, "fp16": "", "fp32": "", "bf16": "" @@ -5984,19 +6688,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.44, + "int8": 147.74, "fp16": "", - "fp32": 0.18, - "bf16": "" + "fp32": 38.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6006,7 +6714,7 @@ "Precisions": [ { "int4": "", - "int8": 2406.55, + "int8": 11.4, "fp16": "", "fp32": "", "bf16": "" @@ -6019,18 +6727,22 @@ }, { "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.47, + "int8": 2.66, "fp16": "", - "fp32": 0.12, - "bf16": "" + "fp32": 0.77, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6040,7 +6752,7 @@ "Precisions": [ { "int4": "", - "int8": 2337.51, + "int8": 511.09, "fp16": "", "fp32": "", "bf16": "" @@ -6052,19 +6764,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.58, + "int8": 313.17, "fp16": "", - "fp32": 0.16, - "bf16": "" + "fp32": 95.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6074,7 +6790,7 @@ "Precisions": [ { "int4": "", - "int8": 2064.41, + "int8": 4.81, "fp16": "", "fp32": "", "bf16": "" @@ -6086,19 +6802,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.71, + "int8": "", "fp16": "", - "fp32": 0.5, - "bf16": "" + "fp32": 31.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6108,7 +6828,7 @@ "Precisions": [ { "int4": "", - "int8": 708.93, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6120,19 +6840,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.48, + "int8": 67.43, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 26.68, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6142,7 +6866,7 @@ "Precisions": [ { "int4": "", - "int8": 2022.46, + "int8": 20.62, "fp16": "", "fp32": "", "bf16": "" @@ -6154,19 +6878,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.72, - "fp16": "", - "fp32": 0.19, - "bf16": "" + "int8": 46.15, + "fp16": 38.3, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6176,7 +6904,7 @@ "Precisions": [ { "int4": "", - "int8": 1351.31, + "int8": 19.82, "fp16": "", "fp32": "", "bf16": "" @@ -6188,19 +6916,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.52, - "fp16": "", - "fp32": 0.14, - "bf16": "" + "int8": 64.24, + "fp16": 50.43, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6210,7 +6942,7 @@ "Precisions": [ { "int4": "", - "int8": 1804.22, + "int8": 20.17, "fp16": "", "fp32": "", "bf16": "" @@ -6222,19 +6954,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.24, - "fp16": "", - "fp32": 0.35, - "bf16": "" + "int8": 0.5, + "fp16": 0.51, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6244,7 +6980,7 @@ "Precisions": [ { "int4": "", - "int8": 936.49, + "int8": 1499.27, "fp16": "", "fp32": "", "bf16": "" @@ -6256,19 +6992,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.64, - "fp16": "", - "fp32": 0.18, - "bf16": "" + "int8": 768.31, + "fp16": 485.7, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6278,7 +7018,7 @@ "Precisions": [ { "int4": "", - "int8": 1922.82, + "int8": 1.7, "fp16": "", "fp32": "", "bf16": "" @@ -6290,19 +7030,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.91, - "fp16": "", + "int8": 208.55, + "fp16": 117.84, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6312,7 +7056,7 @@ "Precisions": [ { "int4": "", - "int8": 1366.6, + "int8": 5.0, "fp16": "", "fp32": "", "bf16": "" @@ -6324,19 +7068,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.37, - "fp16": "", - "fp32": 0.15, - "bf16": "" + "int8": 5.64, + "fp16": 2.72, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6346,7 +7094,7 @@ "Precisions": [ { "int4": "", - "int8": 2090.62, + "int8": 172.69, "fp16": "", "fp32": "", "bf16": "" @@ -6358,19 +7106,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.47, - "fp16": "", - "fp32": 0.17, - "bf16": "" + "int8": 382.92, + "fp16": 223.39, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6380,7 +7132,7 @@ "Precisions": [ { "int4": "", - "int8": 2235.88, + "int8": 3.11, "fp16": "", "fp32": "", "bf16": "" @@ -6392,19 +7144,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.48, - "fp16": "", - "fp32": 0.71, - "bf16": "" + "int8": 126.83, + "fp16": 77.91, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6414,7 +7170,7 @@ "Precisions": [ { "int4": "", - "int8": 558.42, + "int8": 8.1, "fp16": "", "fp32": "", "bf16": "" @@ -6426,19 +7182,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.75, + "int8": 49.68, "fp16": "", - "fp32": 0.29, - "bf16": "" + "fp32": 26.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6448,7 +7208,7 @@ "Precisions": [ { "int4": "", - "int8": 1441.19, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6460,19 +7220,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.29, + "int8": 73.94, "fp16": "", - "fp32": 0.15, - "bf16": "" + "fp32": 48.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6482,7 +7246,7 @@ "Precisions": [ { "int4": "", - "int8": 3400.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6494,19 +7258,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.25, + "int8": 0.69, "fp16": "", - "fp32": 0.89, - "bf16": "" + "fp32": 0.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6516,7 +7284,7 @@ "Precisions": [ { "int4": "", - "int8": 650.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6528,19 +7296,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.7, + "int8": 1050.26, "fp16": "", - "fp32": 2.25, - "bf16": "" + "fp32": 535.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6550,7 +7322,7 @@ "Precisions": [ { "int4": "", - "int8": 251.14, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6562,19 +7334,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 14.54, + "int8": 234.19, "fp16": "", - "fp32": 3.46, - "bf16": "" + "fp32": 87.89, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6584,7 +7360,7 @@ "Precisions": [ { "int4": "", - "int8": 160.14, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6596,19 +7372,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 63.19, + "int8": 4.74, "fp16": "", - "fp32": 5.23, - "bf16": 37.95 + "fp32": 1.74, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6618,10 +7398,10 @@ "Precisions": [ { "int4": "", - "int8": 60.21, + "int8": "", "fp16": "", "fp32": "", - "bf16": 83.04 + "bf16": "" } ], "Unit": "ms", @@ -6630,19 +7410,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 75.35, + "int8": 466.65, "fp16": "", - "fp32": 6.47, - "bf16": 48.43 + "fp32": 188.83, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6652,10 +7436,10 @@ "Precisions": [ { "int4": "", - "int8": 55.97, + "int8": "", "fp16": "", "fp32": "", - "bf16": 73.23 + "bf16": "" } ], "Unit": "ms", @@ -6664,19 +7448,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.51, + "int8": "", "fp16": "", - "fp32": 1.65, - "bf16": "" + "fp32": 65.34, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6686,7 +7474,7 @@ "Precisions": [ { "int4": "", - "int8": 322.06, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6698,19 +7486,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.4, + "int8": 125.18, "fp16": "", - "fp32": 1.76, - "bf16": "" + "fp32": 58.13, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6720,7 +7512,7 @@ "Precisions": [ { "int4": "", - "int8": 286.63, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -6732,19 +7524,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.18, + "int8": 39.97, "fp16": "", - "fp32": 0.32, - "bf16": "" + "fp32": 15.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6754,7 +7550,7 @@ "Precisions": [ { "int4": "", - "int8": 999.99, + "int8": 40.14, "fp16": "", "fp32": "", "bf16": "" @@ -6766,19 +7562,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.9, + "int8": 56.15, "fp16": "", - "fp32": 0.21, - "bf16": "" + "fp32": 35.76, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6788,7 +7588,7 @@ "Precisions": [ { "int4": "", - "int8": 1330.81, + "int8": 28.73, "fp16": "", "fp32": "", "bf16": "" @@ -6800,19 +7600,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.12, + "int8": 0.57, "fp16": "", - "fp32": 0.27, - "bf16": "" + "fp32": 0.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6822,7 +7626,7 @@ "Precisions": [ { "int4": "", - "int8": 1255.21, + "int8": 2069.28, "fp16": "", "fp32": "", "bf16": "" @@ -6834,19 +7638,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39.0, - "fp16": 21.24, - "fp32": "", - "bf16": "" + "int8": 951.93, + "fp16": "", + "fp32": 463.06, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6856,7 +7664,7 @@ "Precisions": [ { "int4": "", - "int8": 52.08, + "int8": 1.74, "fp16": "", "fp32": "", "bf16": "" @@ -6868,19 +7676,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 184.54, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 52.88, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6890,7 +7702,7 @@ "Precisions": [ { "int4": "", - "int8": 48.01, + "int8": 9.61, "fp16": "", "fp32": "", "bf16": "" @@ -6902,19 +7714,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.33, + "int8": 3.16, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 0.92, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6924,7 +7740,7 @@ "Precisions": [ { "int4": "", - "int8": 179.63, + "int8": 466.34, "fp16": "", "fp32": "", "bf16": "" @@ -6936,19 +7752,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.48, - "fp16": 0.52, - "fp32": "", - "bf16": "" + "int8": 383.62, + "fp16": "", + "fp32": 134.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6958,7 +7778,7 @@ "Precisions": [ { "int4": "", - "int8": 2109.61, + "int8": 4.16, "fp16": "", "fp32": "", "bf16": "" @@ -6970,19 +7790,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.53, - "fp16": 0.52, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 43.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -6992,7 +7816,7 @@ "Precisions": [ { "int4": "", - "int8": 1494.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7004,19 +7828,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.54, - "fp16": 0.58, - "fp32": "", - "bf16": "" + "int8": 91.3, + "fp16": "", + "fp32": 36.39, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7026,7 +7854,7 @@ "Precisions": [ { "int4": "", - "int8": 1486.87, + "int8": 18.15, "fp16": "", "fp32": "", "bf16": "" @@ -7038,19 +7866,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.83, - "fp16": 0.89, + "int8": 47.17, + "fp16": 39.79, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7060,7 +7892,7 @@ "Precisions": [ { "int4": "", - "int8": 1111.35, + "int8": 18.45, "fp16": "", "fp32": "", "bf16": "" @@ -7072,19 +7904,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.57, - "fp16": 0.55, + "int8": 80.6, + "fp16": 59.92, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7094,7 +7930,7 @@ "Precisions": [ { "int4": "", - "int8": 1643.33, + "int8": 14.61, "fp16": "", "fp32": "", "bf16": "" @@ -7106,19 +7942,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.03, - "fp16": 1.14, + "int8": 0.52, + "fp16": 0.58, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7128,7 +7968,7 @@ "Precisions": [ { "int4": "", - "int8": 973.29, + "int8": 1506.76, "fp16": "", "fp32": "", "bf16": "" @@ -7140,19 +7980,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.73, - "fp16": 0.77, + "int8": 778.4, + "fp16": 509.56, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7162,7 +8006,7 @@ "Precisions": [ { "int4": "", - "int8": 1184.14, + "int8": 1.48, "fp16": "", "fp32": "", "bf16": "" @@ -7174,19 +8018,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 225.12, + "fp16": 127.27, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7196,7 +8044,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 4.31, "fp16": "", "fp32": "", "bf16": "" @@ -7208,19 +8056,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.97, - "fp16": 1.64, + "int8": 5.79, + "fp16": 2.86, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7230,7 +8082,7 @@ "Precisions": [ { "int4": "", - "int8": 481.56, + "int8": 144.71, "fp16": "", "fp32": "", "bf16": "" @@ -7242,19 +8094,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 404.76, + "fp16": 237.61, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7264,7 +8120,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.75, "fp16": "", "fp32": "", "bf16": "" @@ -7276,19 +8132,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.69, - "fp16": 7.44, + "int8": 131.89, + "fp16": 83.17, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7298,7 +8158,7 @@ "Precisions": [ { "int4": "", - "int8": 91.66, + "int8": 7.11, "fp16": "", "fp32": "", "bf16": "" @@ -7310,19 +8170,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.56, + "int8": 120.44, "fp16": "", - "fp32": 0.51, - "bf16": "" + "fp32": 47.21, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7332,7 +8196,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 13.32, "fp16": "", "fp32": "", "bf16": "" @@ -7344,19 +8208,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 148.91, "fp16": "", - "fp32": 0.19, - "bf16": "" + "fp32": 93.08, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7366,7 +8234,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 9.22, "fp16": "", "fp32": "", "bf16": "" @@ -7378,19 +8246,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.7, + "int8": "", "fp16": "", - "fp32": 0.31, - "bf16": "" + "fp32": 0.49, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7400,7 +8272,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 733.91, "fp16": "", "fp32": "", "bf16": "" @@ -7412,19 +8284,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.17, + "int8": 2974.41, "fp16": "", - "fp32": 0.65, - "bf16": "" + "fp32": 1317.04, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7434,7 +8310,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.69, "fp16": "", "fp32": "", "bf16": "" @@ -7446,19 +8322,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.67, + "int8": 537.98, "fp16": "", - "fp32": 0.33, - "bf16": "" + "fp32": 148.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7468,7 +8348,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.82, "fp16": "", "fp32": "", "bf16": "" @@ -7480,19 +8360,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.8, "fp16": "", - "fp32": 0.71, - "bf16": "" + "fp32": 2.47, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7502,7 +8386,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 133.73, "fp16": "", "fp32": "", "bf16": "" @@ -7514,19 +8398,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.81, + "int8": 1068.19, "fp16": "", - "fp32": 0.43, - "bf16": "" + "fp32": 379.85, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7536,7 +8424,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.33, "fp16": "", "fp32": "", "bf16": "" @@ -7548,9 +8436,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -7559,8 +8448,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 0.46, - "bf16": "" + "fp32": 122.62, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7582,19 +8474,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.37, + "int8": 266.57, "fp16": "", - "fp32": 1.2, - "bf16": "" + "fp32": 102.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7604,7 +8500,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 5.27, "fp16": "", "fp32": "", "bf16": "" @@ -7616,19 +8512,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 84.71, "fp16": "", - "fp32": 0.58, - "bf16": "" + "fp32": 51.06, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7650,19 +8550,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 274.01, + "int8": 98.02, "fp16": "", - "fp32": 168.87, - "bf16": "" + "fp32": 65.51, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7672,7 +8576,7 @@ "Precisions": [ { "int4": "", - "int8": 4.07, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7684,19 +8588,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 133.76, + "int8": 1.16, "fp16": "", - "fp32": 80.44, - "bf16": "" + "fp32": 0.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7706,7 +8614,7 @@ "Precisions": [ { "int4": "", - "int8": 7.82, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7718,19 +8626,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 300.95, + "int8": 1353.32, "fp16": "", - "fp32": 133.55, - "bf16": "" + "fp32": 683.15, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7740,7 +8652,7 @@ "Precisions": [ { "int4": "", - "int8": 3.62, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7752,19 +8664,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 534.91, + "int8": 365.63, "fp16": "", - "fp32": 406.53, - "bf16": "" + "fp32": 164.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7774,7 +8690,7 @@ "Precisions": [ { "int4": "", - "int8": 2.01, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7786,19 +8702,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 893.76, + "int8": 8.65, "fp16": "", - "fp32": 462.2, - "bf16": "" + "fp32": 3.77, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7808,7 +8728,7 @@ "Precisions": [ { "int4": "", - "int8": 1.62, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7820,19 +8740,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 826.85, + "int8": 657.26, "fp16": "", - "fp32": 315.6, - "bf16": "" + "fp32": 293.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7842,7 +8766,7 @@ "Precisions": [ { "int4": "", - "int8": 1.9, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7854,9 +8778,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -7865,8 +8790,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 462.14, - "bf16": "" + "fp32": 107.24, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7876,7 +8804,7 @@ "Precisions": [ { "int4": "", - "int8": 1.72, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7888,19 +8816,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2989.92, + "int8": 182.9, "fp16": "", - "fp32": 1328.99, - "bf16": "" + "fp32": 101.97, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7910,7 +8842,7 @@ "Precisions": [ { "int4": "", - "int8": 0.69, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -7922,19 +8854,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 848.04, + "int8": 50.21, "fp16": "", - "fp32": 626.94, - "bf16": "" + "fp32": 18.33, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7944,7 +8880,7 @@ "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 22.66, "fp16": "", "fp32": "", "bf16": "" @@ -7957,18 +8893,22 @@ }, { "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1284.93, + "int8": 71.27, "fp16": "", - "fp32": 509.36, - "bf16": "" + "fp32": 41.39, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -7978,7 +8918,7 @@ "Precisions": [ { "int4": "", - "int8": 0.96, + "int8": 14.62, "fp16": "", "fp32": "", "bf16": "" @@ -7990,19 +8930,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 975.3, + "int8": 0.71, "fp16": "", - "fp32": 312.11, - "bf16": "" + "fp32": 0.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8012,7 +8956,7 @@ "Precisions": [ { "int4": "", - "int8": 1.21, + "int8": 1361.21, "fp16": "", "fp32": "", "bf16": "" @@ -8024,19 +8968,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", "Model": "mobilenet-v2", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1966.6, + "int8": 1291.06, "fp16": "", - "fp32": 958.97, - "bf16": "" + "fp32": 507.09, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8046,7 +8994,7 @@ "Precisions": [ { "int4": "", - "int8": 0.96, + "int8": 0.95, "fp16": "", "fp32": "", "bf16": "" @@ -8058,19 +9006,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1039.53, + "int8": 224.68, "fp16": "", - "fp32": 513.13, - "bf16": "" + "fp32": 60.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8080,7 +9032,7 @@ "Precisions": [ { "int4": "", - "int8": 1.61, + "int8": 4.95, "fp16": "", "fp32": "", "bf16": "" @@ -8092,19 +9044,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.84, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1.01, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8114,7 +9070,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 250.45, "fp16": "", "fp32": "", "bf16": "" @@ -8126,19 +9082,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 737.89, + "int8": 491.99, "fp16": "", - "fp32": 488.9, - "bf16": "" + "fp32": 146.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8148,7 +9108,7 @@ "Precisions": [ { "int4": "", - "int8": 1.44, + "int8": 2.2, "fp16": "", "fp32": "", "bf16": "" @@ -8160,19 +9120,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 904.6, + "int8": "", "fp16": "", - "fp32": 570.78, - "bf16": "" + "fp32": 48.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8182,7 +9146,7 @@ "Precisions": [ { "int4": "", - "int8": 1.56, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8194,19 +9158,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4254.47, + "int8": 106.45, "fp16": "", - "fp32": 2047.25, - "bf16": "" + "fp32": 40.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8216,7 +9184,7 @@ "Precisions": [ { "int4": "", - "int8": 0.6, + "int8": 10.2, "fp16": "", "fp32": "", "bf16": "" @@ -8228,19 +9196,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 296.47, - "fp16": "", - "fp32": 183.87, - "bf16": "" + "int8": 68.4, + "fp16": 53.22, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8250,7 +9222,7 @@ "Precisions": [ { "int4": "", - "int8": 3.81, + "int8": 17.09, "fp16": "", "fp32": "", "bf16": "" @@ -8262,19 +9234,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1451.27, - "fp16": "", - "fp32": 666.39, - "bf16": "" + "int8": 91.46, + "fp16": 72.22, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8284,7 +9260,7 @@ "Precisions": [ { "int4": "", - "int8": 1.2, + "int8": 17.92, "fp16": "", "fp32": "", "bf16": "" @@ -8296,19 +9272,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 519.78, - "fp16": "", - "fp32": 425.23, - "bf16": "" + "int8": 0.82, + "fp16": 0.88, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8318,7 +9298,7 @@ "Precisions": [ { "int4": "", - "int8": 2.07, + "int8": 1113.84, "fp16": "", "fp32": "", "bf16": "" @@ -8330,19 +9310,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5407.63, - "fp16": "", - "fp32": 1924.43, - "bf16": "" + "int8": 729.72, + "fp16": 569.2, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8352,7 +9336,7 @@ "Precisions": [ { "int4": "", - "int8": 1.51, + "int8": 2.05, "fp16": "", "fp32": "", "bf16": "" @@ -8364,19 +9348,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 15016.47, - "fp16": "", - "fp32": 4645.46, - "bf16": "" + "int8": 262.94, + "fp16": 174.98, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8386,7 +9374,7 @@ "Precisions": [ { "int4": "", - "int8": 0.94, + "int8": 4.82, "fp16": "", "fp32": "", "bf16": "" @@ -8398,19 +9386,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 22842.63, - "fp16": "", - "fp32": 7144.4, - "bf16": "" + "int8": 8.29, + "fp16": 4.67, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8420,7 +9412,7 @@ "Precisions": [ { "int4": "", - "int8": 0.58, + "int8": 118.28, "fp16": "", "fp32": "", "bf16": "" @@ -8432,19 +9424,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 38642.1, - "fp16": "", - "fp32": 10319.56, - "bf16": 25708.49 + "int8": 447.59, + "fp16": 299.29, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8454,10 +9450,10 @@ "Precisions": [ { "int4": "", - "int8": 0.64, + "int8": 3.33, "fp16": "", "fp32": "", - "bf16": 0.65 + "bf16": "" } ], "Unit": "ms", @@ -8466,19 +9462,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 39913.42, - "fp16": "", - "fp32": 15945.06, - "bf16": "" + "int8": 161.26, + "fp16": 111.45, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8488,10 +9488,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.1, "fp16": "", "fp32": "", - "bf16": 0.75 + "bf16": "" } ], "Unit": "ms", @@ -8500,19 +9500,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10312.71, + "int8": 50.01, "fp16": "", - "fp32": 3331.01, - "bf16": "" + "fp32": 25.82, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8522,7 +9526,7 @@ "Precisions": [ { "int4": "", - "int8": 1.24, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8534,19 +9538,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 11955.69, + "int8": 57.69, "fp16": "", - "fp32": 3563.39, - "bf16": "" + "fp32": 28.41, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8568,19 +9576,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1984.45, + "int8": 0.69, "fp16": "", - "fp32": 814.84, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8590,7 +9602,7 @@ "Precisions": [ { "int4": "", - "int8": 1.35, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8602,19 +9614,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", "Model": "mobilenet-v2", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1516.8, + "int8": 958.94, "fp16": "", - "fp32": 588.51, - "bf16": "" + "fp32": 350.53, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8624,7 +9640,7 @@ "Precisions": [ { "int4": "", - "int8": 1.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8636,19 +9652,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1651.84, + "int8": 230.4, "fp16": "", - "fp32": 775.78, - "bf16": "" + "fp32": 85.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8658,7 +9678,7 @@ "Precisions": [ { "int4": "", - "int8": 1.02, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8670,19 +9690,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2423.58, - "fp16": 2444.62, - "fp32": "", - "bf16": "" + "int8": 4.44, + "fp16": "", + "fp32": 1.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8692,7 +9716,7 @@ "Precisions": [ { "int4": "", - "int8": 0.64, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8704,19 +9728,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 456.16, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 162.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8726,7 +9754,7 @@ "Precisions": [ { "int4": "", - "int8": 0.99, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8738,19 +9766,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1862.03, - "fp16": 1564.35, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 55.98, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8760,7 +9792,7 @@ "Precisions": [ { "int4": "", - "int8": 0.78, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8772,19 +9804,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1956.69, - "fp16": 1343.46, - "fp32": "", - "bf16": "" + "int8": 103.63, + "fp16": "", + "fp32": 53.56, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8794,7 +9830,7 @@ "Precisions": [ { "int4": "", - "int8": 0.82, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -8806,19 +9842,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1999.97, - "fp16": 1359.27, - "fp32": "", - "bf16": "" + "int8": 38.28, + "fp16": "", + "fp32": 13.87, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8828,7 +9868,7 @@ "Precisions": [ { "int4": "", - "int8": 0.74, + "int8": 28.41, "fp16": "", "fp32": "", "bf16": "" @@ -8840,19 +9880,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3751.95, - "fp16": 2877.38, - "fp32": "", - "bf16": "" + "int8": 53.34, + "fp16": "", + "fp32": 22.26, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8862,7 +9906,7 @@ "Precisions": [ { "int4": "", - "int8": 0.4, + "int8": 20.12, "fp16": "", "fp32": "", "bf16": "" @@ -8874,19 +9918,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 415.36, - "fp16": 324.74, - "fp32": "", - "bf16": "" + "int8": 0.52, + "fp16": "", + "fp32": 0.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8896,7 +9944,7 @@ "Precisions": [ { "int4": "", - "int8": 3.52, + "int8": 1805.69, "fp16": "", "fp32": "", "bf16": "" @@ -8908,19 +9956,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 188.4, - "fp16": 223.08, - "fp32": "", - "bf16": "" + "int8": 972.25, + "fp16": "", + "fp32": 311.82, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8930,7 +9982,7 @@ "Precisions": [ { "int4": "", - "int8": 7.66, + "int8": 1.2, "fp16": "", "fp32": "", "bf16": "" @@ -8942,19 +9994,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 673.52, - "fp16": 505.05, - "fp32": "", - "bf16": "" + "int8": 174.69, + "fp16": "", + "fp32": 45.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8964,7 +10020,7 @@ "Precisions": [ { "int4": "", - "int8": 2.74, + "int8": 6.4, "fp16": "", "fp32": "", "bf16": "" @@ -8976,19 +10032,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 779.69, - "fp16": 496.63, - "fp32": "", - "bf16": "" + "int8": 2.72, + "fp16": "", + "fp32": 0.78, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -8998,7 +10058,7 @@ "Precisions": [ { "int4": "", - "int8": 1.63, + "int8": 335.04, "fp16": "", "fp32": "", "bf16": "" @@ -9010,19 +10070,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 386.67, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 99.8, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9032,7 +10096,7 @@ "Precisions": [ { "int4": "", - "int8": 1.46, + "int8": 2.82, "fp16": "", "fp32": "", "bf16": "" @@ -9044,9 +10108,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -9054,9 +10119,12 @@ { "int4": "", "int8": "", - "fp16": 574.04, - "fp32": "", - "bf16": "" + "fp16": "", + "fp32": 32.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9078,19 +10146,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 630.96, - "fp16": 442.65, - "fp32": "", - "bf16": "" + "int8": 76.54, + "fp16": "", + "fp32": 27.6, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9100,7 +10172,7 @@ "Precisions": [ { "int4": "", - "int8": 1.74, + "int8": 13.2, "fp16": "", "fp32": "", "bf16": "" @@ -9112,19 +10184,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1281.44, - "fp16": 911.94, + "int8": 45.77, + "fp16": 40.93, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9134,7 +10210,7 @@ "Precisions": [ { "int4": "", - "int8": 1.08, + "int8": 21.21, "fp16": "", "fp32": "", "bf16": "" @@ -9146,19 +10222,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 890.07, - "fp16": 624.4, + "int8": 56.2, + "fp16": 41.8, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9168,7 +10248,7 @@ "Precisions": [ { "int4": "", - "int8": 1.41, + "int8": 23.38, "fp16": "", "fp32": "", "bf16": "" @@ -9180,19 +10260,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 0.56, + "fp16": 0.54, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9202,7 +10286,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1606.31, "fp16": "", "fp32": "", "bf16": "" @@ -9214,19 +10298,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 iGPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 339.71, - "fp16": 267.18, + "int8": 648.66, + "fp16": 431.47, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9236,7 +10324,7 @@ "Precisions": [ { "int4": "", - "int8": 3.83, + "int8": 1.76, "fp16": "", "fp32": "", "bf16": "" @@ -9248,19 +10336,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1298.67, - "fp16": 1337.24, + "int8": 208.21, + "fp16": 122.24, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9270,7 +10362,7 @@ "Precisions": [ { "int4": "", - "int8": 0.78, + "int8": 5.47, "fp16": "", "fp32": "", "bf16": "" @@ -9282,19 +10374,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1422.4, - "fp16": 1361.15, + "int8": 5.71, + "fp16": 3.09, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9304,7 +10400,7 @@ "Precisions": [ { "int4": "", - "int8": 1.27, + "int8": 173.5, "fp16": "", "fp32": "", "bf16": "" @@ -9316,19 +10412,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1200.66, - "fp16": 1554.15, + "int8": 348.95, + "fp16": 224.45, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9338,7 +10438,7 @@ "Precisions": [ { "int4": "", - "int8": 1.23, + "int8": 3.56, "fp16": "", "fp32": "", "bf16": "" @@ -9350,19 +10450,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 478.43, - "fp16": "", - "fp32": 307.02, - "bf16": "" + "int8": 113.89, + "fp16": 78.71, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9372,7 +10476,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 9.49, "fp16": "", "fp32": "", "bf16": "" @@ -9384,19 +10488,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 228.63, + "int8": 111.58, "fp16": "", - "fp32": 220.31, - "bf16": "" + "fp32": 57.55, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9418,19 +10526,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 524.29, + "int8": 141.13, "fp16": "", - "fp32": 394.1, - "bf16": "" + "fp32": 75.23, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9452,19 +10564,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.63, "fp16": "", - "fp32": 350.27, - "bf16": "" + "fp32": 0.68, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9486,19 +10602,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", "Model": "mobilenet-v2", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1049.71, + "int8": 2287.47, "fp16": "", - "fp32": 538.46, - "bf16": "" + "fp32": 1150.08, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9520,19 +10640,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 532.56, "fp16": "", - "fp32": 680.55, - "bf16": "" + "fp32": 180.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9554,19 +10678,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 10.33, "fp16": "", - "fp32": 347.8, - "bf16": "" + "fp32": 3.81, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9589,18 +10717,22 @@ }, { "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2306.0, + "int8": 1013.57, "fp16": "", - "fp32": 1096.89, - "bf16": "" + "fp32": 403.5, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9622,19 +10754,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1219.99, + "int8": "", "fp16": "", - "fp32": 644.18, - "bf16": "" + "fp32": 133.88, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9656,19 +10792,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1610.28, + "int8": 268.57, "fp16": "", - "fp32": 845.71, - "bf16": "" + "fp32": 120.55, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9690,19 +10830,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 484.88, + "int8": 87.88, "fp16": "", - "fp32": 280.8, - "bf16": "" + "fp32": 34.76, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9712,7 +10856,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 16.26, "fp16": "", "fp32": "", "bf16": "" @@ -9724,19 +10868,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4622.78, + "int8": 113.82, "fp16": "", - "fp32": 2338.78, - "bf16": "" + "fp32": 62.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9746,7 +10894,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.46, "fp16": "", "fp32": "", "bf16": "" @@ -9758,19 +10906,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.27, "fp16": "", - "fp32": 1684.08, - "bf16": "" + "fp32": 0.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9780,7 +10932,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 886.78, "fp16": "", "fp32": "", "bf16": "" @@ -9792,19 +10944,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 45.28, + "int8": 1982.75, "fp16": "", - "fp32": 18.84, - "bf16": "" + "fp32": 968.72, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9814,7 +10970,7 @@ "Precisions": [ { "int4": "", - "int8": 23.81, + "int8": 0.89, "fp16": "", "fp32": "", "bf16": "" @@ -9826,19 +10982,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 19.87, + "int8": 429.58, "fp16": "", - "fp32": 8.15, - "bf16": "" + "fp32": 107.58, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9848,7 +11008,7 @@ "Precisions": [ { "int4": "", - "int8": 51.41, + "int8": 3.47, "fp16": "", "fp32": "", "bf16": "" @@ -9860,19 +11020,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 51.65, + "int8": 7.11, "fp16": "", - "fp32": 14.46, - "bf16": "" + "fp32": 1.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9882,7 +11046,7 @@ "Precisions": [ { "int4": "", - "int8": 19.81, + "int8": 159.25, "fp16": "", "fp32": "", "bf16": "" @@ -9894,19 +11058,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 96.85, + "int8": 854.13, "fp16": "", - "fp32": 50.34, - "bf16": "" + "fp32": 289.32, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9916,7 +11084,7 @@ "Precisions": [ { "int4": "", - "int8": 10.76, + "int8": 1.72, "fp16": "", "fp32": "", "bf16": "" @@ -9928,19 +11096,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 144.67, + "int8": "", "fp16": "", - "fp32": 72.98, - "bf16": "" + "fp32": 90.72, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9950,7 +11122,7 @@ "Precisions": [ { "int4": "", - "int8": 8.21, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -9962,19 +11134,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 154.38, + "int8": 206.32, "fp16": "", - "fp32": 40.41, - "bf16": "" + "fp32": 78.09, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -9984,7 +11160,7 @@ "Precisions": [ { "int4": "", - "int8": 11.07, + "int8": 6.49, "fp16": "", "fp32": "", "bf16": "" @@ -9996,19 +11172,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 185.28, - "fp16": "", - "fp32": 53.47, - "bf16": "" + "int8": 89.81, + "fp16": 69.99, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10018,7 +11198,7 @@ "Precisions": [ { "int4": "", - "int8": 9.56, + "int8": 12.71, "fp16": "", "fp32": "", "bf16": "" @@ -10030,19 +11210,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 543.91, - "fp16": "", - "fp32": 151.19, - "bf16": "" + "int8": 128.07, + "fp16": 97.39, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10052,7 +11236,7 @@ "Precisions": [ { "int4": "", - "int8": 2.82, + "int8": 12.87, "fp16": "", "fp32": "", "bf16": "" @@ -10064,19 +11248,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 151.17, - "fp16": "", - "fp32": 75.67, - "bf16": "" + "int8": 1.04, + "fp16": 1.15, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10086,7 +11274,7 @@ "Precisions": [ { "int4": "", - "int8": 7.12, + "int8": 972.87, "fp16": "", "fp32": "", "bf16": "" @@ -10098,19 +11286,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 225.84, - "fp16": "", - "fp32": 61.28, - "bf16": "" + "int8": 1281.93, + "fp16": 912.69, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10120,7 +11312,7 @@ "Precisions": [ { "int4": "", - "int8": 4.95, + "int8": 1.08, "fp16": "", "fp32": "", "bf16": "" @@ -10132,19 +11324,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", "Model": "resnet-50", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 174.35, - "fp16": "", - "fp32": 45.46, - "bf16": "" + "int8": 381.27, + "fp16": 226.42, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10154,7 +11350,7 @@ "Precisions": [ { "int4": "", - "int8": 6.41, + "int8": 3.22, "fp16": "", "fp32": "", "bf16": "" @@ -10166,19 +11362,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 413.95, - "fp16": "", - "fp32": 107.82, - "bf16": "" + "int8": 10.47, + "fp16": 6.14, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10188,7 +11388,7 @@ "Precisions": [ { "int4": "", - "int8": 3.45, + "int8": 100.17, "fp16": "", "fp32": "", "bf16": "" @@ -10200,19 +11400,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 203.93, - "fp16": "", - "fp32": 59.04, - "bf16": "" + "int8": 744.92, + "fp16": 407.72, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10222,7 +11426,7 @@ "Precisions": [ { "int4": "", - "int8": 8.97, + "int8": 1.87, "fp16": "", "fp32": "", "bf16": "" @@ -10234,19 +11438,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 298.27, - "fp16": "", + "int8": 215.67, + "fp16": 148.01, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10256,7 +11464,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 5.58, "fp16": "", "fp32": "", "bf16": "" @@ -10268,19 +11476,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 122.15, + "int8": 61.33, "fp16": "", - "fp32": 60.34, - "bf16": "" + "fp32": 32.27, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10290,7 +11502,7 @@ "Precisions": [ { "int4": "", - "int8": 7.21, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10302,19 +11514,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 152.19, + "int8": 88.48, "fp16": "", - "fp32": 71.16, - "bf16": "" + "fp32": 59.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10324,7 +11540,7 @@ "Precisions": [ { "int4": "", - "int8": 7.72, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10336,19 +11552,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 766.25, + "int8": 0.81, "fp16": "", - "fp32": 233.37, - "bf16": "" + "fp32": 0.43, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10358,7 +11578,7 @@ "Precisions": [ { "int4": "", - "int8": 2.16, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10370,19 +11590,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 48.79, + "int8": 1218.37, "fp16": "", - "fp32": 20.22, - "bf16": "" + "fp32": 644.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10392,7 +11616,7 @@ "Precisions": [ { "int4": "", - "int8": 21.9, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10404,19 +11628,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 245.58, + "int8": 284.91, "fp16": "", - "fp32": 121.38, - "bf16": "" + "fp32": 109.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10426,7 +11654,7 @@ "Precisions": [ { "int4": "", - "int8": 5.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10438,19 +11666,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 92.24, + "int8": 5.67, "fp16": "", - "fp32": 49.79, - "bf16": "" + "fp32": 2.15, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10460,7 +11692,7 @@ "Precisions": [ { "int4": "", - "int8": 11.19, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10472,19 +11704,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 971.95, + "int8": 554.73, "fp16": "", - "fp32": 269.81, - "bf16": "" + "fp32": 228.8, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10494,7 +11730,7 @@ "Precisions": [ { "int4": "", - "int8": 3.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10506,19 +11742,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2968.31, + "int8": "", "fp16": "", - "fp32": 754.35, - "bf16": "" + "fp32": 80.32, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10528,7 +11768,7 @@ "Precisions": [ { "int4": "", - "int8": 1.61, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10540,19 +11780,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4948.09, + "int8": 154.56, "fp16": "", - "fp32": 1155.67, - "bf16": "" + "fp32": 72.19, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10562,7 +11806,7 @@ "Precisions": [ { "int4": "", - "int8": 1.06, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10574,19 +11818,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 19506.31, + "int8": 44.62, "fp16": "", - "fp32": 1609.79, - "bf16": 7600.31 + "fp32": 17.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10596,10 +11844,10 @@ "Precisions": [ { "int4": "", - "int8": 0.99, + "int8": 37.64, "fp16": "", "fp32": "", - "bf16": 1.24 + "bf16": "" } ], "Unit": "ms", @@ -10608,19 +11856,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21749.45, + "int8": 61.85, "fp16": "", - "fp32": 2011.71, - "bf16": 13645.99 + "fp32": 39.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10630,10 +11882,10 @@ "Precisions": [ { "int4": "", - "int8": 0.98, + "int8": 26.95, "fp16": "", "fp32": "", - "bf16": 1.33 + "bf16": "" } ], "Unit": "ms", @@ -10642,19 +11894,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2122.71, + "int8": 0.64, "fp16": "", - "fp32": 565.52, - "bf16": "" + "fp32": 0.17, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10664,7 +11920,7 @@ "Precisions": [ { "int4": "", - "int8": 1.88, + "int8": 1935.64, "fp16": "", "fp32": "", "bf16": "" @@ -10676,19 +11932,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2249.41, + "int8": 1042.94, "fp16": "", - "fp32": 563.79, - "bf16": "" + "fp32": 515.99, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10698,7 +11958,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.61, "fp16": "", "fp32": "", "bf16": "" @@ -10710,19 +11970,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", "Model": "resnet-50", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 392.72, + "int8": 203.02, "fp16": "", - "fp32": 95.29, - "bf16": "" + "fp32": 59.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10732,7 +11996,7 @@ "Precisions": [ { "int4": "", - "int8": 6.4, + "int8": 9.0, "fp16": "", "fp32": "", "bf16": "" @@ -10744,19 +12008,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 290.74, + "int8": 3.48, "fp16": "", - "fp32": 70.82, - "bf16": "" + "fp32": 1.03, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10766,7 +12034,7 @@ "Precisions": [ { "int4": "", - "int8": 7.55, + "int8": 439.19, "fp16": "", "fp32": "", "bf16": "" @@ -10778,19 +12046,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 318.23, + "int8": 422.9, "fp16": "", - "fp32": 86.35, - "bf16": "" + "fp32": 151.69, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10800,7 +12072,7 @@ "Precisions": [ { "int4": "", - "int8": 4.59, + "int8": 3.87, "fp16": "", "fp32": "", "bf16": "" @@ -10812,19 +12084,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2318.55, - "fp16": 1375.72, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 48.93, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10834,7 +12110,7 @@ "Precisions": [ { "int4": "", - "int8": 0.87, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -10846,19 +12122,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 101.73, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 40.76, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10868,7 +12148,7 @@ "Precisions": [ { "int4": "", - "int8": 1.42, + "int8": 16.99, "fp16": "", "fp32": "", "bf16": "" @@ -10880,19 +12160,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 854.09, - "fp16": 539.78, + "int8": 67.08, + "fp16": 52.9, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10902,7 +12186,7 @@ "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 14.38, "fp16": "", "fp32": "", "bf16": "" @@ -10914,19 +12198,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 768.87, - "fp16": 382.94, + "int8": 98.8, + "fp16": 73.53, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -10936,7 +12224,7 @@ "Precisions": [ { "int4": "", - "int8": 1.57, + "int8": 13.41, "fp16": "", "fp32": "", "bf16": "" @@ -10948,19 +12236,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 772.54, - "fp16": 383.99, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 6.21, + "token_int8": 3.88, + "token_fp16": "" } ], "Unit": "FPS", @@ -10969,8 +12261,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.54, + "int4": 160.82, + "int8": 257.32, "fp16": "", "fp32": "", "bf16": "" @@ -10982,19 +12274,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2022.98, - "fp16": 975.93, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 7.25, + "token_int8": 4.27, + "token_fp16": "" } ], "Unit": "FPS", @@ -11003,8 +12299,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 0.75, + "int4": 137.82, + "int8": 233.92, "fp16": "", "fp32": "", "bf16": "" @@ -11016,19 +12312,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 106.37, - "fp16": 64.66, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 8.53, + "token_int8": 5.74, + "token_fp16": "" } ], "Unit": "FPS", @@ -11037,8 +12337,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 10.57, + "int4": 117.18, + "int8": 174.01, "fp16": "", "fp32": "", "bf16": "" @@ -11050,19 +12350,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 48.1, - "fp16": 51.69, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 8.49, + "token_int8": 5.06, + "token_fp16": "" } ], "Unit": "FPS", @@ -11071,8 +12375,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 23.01, + "int4": 117.69, + "int8": 197.3, "fp16": "", "fp32": "", "bf16": "" @@ -11084,19 +12388,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 203.75, - "fp16": 118.64, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 16.79, + "token_int8": 11.89, + "token_fp16": 6.7 } ], "Unit": "FPS", @@ -11105,9 +12413,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 6.33, - "fp16": "", + "int4": 59.54, + "int8": 84.05, + "fp16": 149.13, "fp32": "", "bf16": "" } @@ -11118,19 +12426,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 213.35, - "fp16": 119.51, + "int8": 0.73, + "fp16": 0.77, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11140,7 +12452,7 @@ "Precisions": [ { "int4": "", - "int8": 4.89, + "int8": 1191.59, "fp16": "", "fp32": "", "bf16": "" @@ -11152,19 +12464,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 225.63, - "fp16": 128.49, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 8.86, + "token_int8": 5.44, + "token_fp16": "" } ], "Unit": "FPS", @@ -11173,8 +12489,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 4.26, + "int4": 112.76, + "int8": 183.5, "fp16": "", "fp32": "", "bf16": "" @@ -11186,19 +12502,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": 174.47, + "int8": 869.88, + "fp16": 621.94, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11208,7 +12528,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.38, "fp16": "", "fp32": "", "bf16": "" @@ -11220,19 +12540,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 203.92, - "fp16": 125.66, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 14.78, + "token_int8": 9.98, + "token_fp16": 5.45 } ], "Unit": "FPS", @@ -11241,9 +12565,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 5.53, - "fp16": "", + "int4": 67.65, + "int8": 100.19, + "fp16": 183.48, "fp32": "", "bf16": "" } @@ -11254,19 +12578,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 387.63, - "fp16": 228.73, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 9.11, + "token_int8": 5.39, + "token_fp16": "" } ], "Unit": "FPS", @@ -11275,8 +12603,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 3.23, + "int4": 109.74, + "int8": 185.49, "fp16": "", "fp32": "", "bf16": "" @@ -11290,17 +12618,21 @@ { "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", "Model": "resnet-50", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 278.1, - "fp16": 165.78, + "int8": 277.06, + "fp16": 164.27, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11310,7 +12642,7 @@ "Precisions": [ { "int4": "", - "int8": 3.86, + "int8": 3.85, "fp16": "", "fp32": "", "bf16": "" @@ -11322,19 +12654,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 7.1, + "fp16": 3.99, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11344,7 +12680,7 @@ "Precisions": [ { "int4": "", - "int8": 3.5, + "int8": 126.73, "fp16": "", "fp32": "", "bf16": "" @@ -11356,19 +12692,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 81.72, - "fp16": 49.75, + "int8": 484.13, + "fp16": 298.47, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11378,7 +12718,7 @@ "Precisions": [ { "int4": "", - "int8": 13.15, + "int8": 2.49, "fp16": "", "fp32": "", "bf16": "" @@ -11390,19 +12730,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 556.79, - "fp16": 393.72, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11412,8 +12756,8 @@ "Precisions": [ { "int4": "", - "int8": 1.7, - "fp16": "", + "int8": 29.54, + "fp16": 29.97, "fp32": "", "bf16": "" } @@ -11424,19 +12768,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 568.08, - "fp16": 375.74, + "int8": 162.35, + "fp16": 106.83, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11446,7 +12794,7 @@ "Precisions": [ { "int4": "", - "int8": 2.26, + "int8": 6.38, "fp16": "", "fp32": "", "bf16": "" @@ -11458,19 +12806,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1167.79, - "fp16": 621.08, - "fp32": "", - "bf16": "" + "int8": 170.14, + "fp16": "", + "fp32": 67.07, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11480,7 +12832,7 @@ "Precisions": [ { "int4": "", - "int8": 1.49, + "int8": 10.73, "fp16": "", "fp32": "", "bf16": "" @@ -11492,19 +12844,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 129.36, + "int8": 219.8, "fp16": "", - "fp32": 55.58, - "bf16": "" + "fp32": 126.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11514,7 +12870,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 7.34, "fp16": "", "fp32": "", "bf16": "" @@ -11526,19 +12882,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 59.45, + "int8": "", "fp16": "", - "fp32": 54.27, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 9.42, + "token_int8": 6.89, + "token_fp16": 3.59 } ], "Unit": "FPS", @@ -11547,9 +12907,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 106.06, + "int8": 144.95, + "fp16": 278.42, "fp32": "", "bf16": "" } @@ -11560,19 +12920,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 196.98, + "int8": "", "fp16": "", - "fp32": 115.77, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 10.65, + "token_int8": 7.46, + "token_fp16": 3.83 } ], "Unit": "FPS", @@ -11581,9 +12945,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 93.82, + "int8": 133.88, + "fp16": 260.66, "fp32": "", "bf16": "" } @@ -11594,9 +12958,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -11605,8 +12970,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 53.71, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 13.44, + "token_int8": 9.29, + "token_fp16": 4.94 } ], "Unit": "FPS", @@ -11615,9 +12983,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 74.39, + "int8": 107.62, + "fp16": 202.32, "fp32": "", "bf16": "" } @@ -11628,19 +12996,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 235.76, + "int8": "", "fp16": "", - "fp32": 88.15, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 11.91, + "token_int8": 8.65, + "token_fp16": 4.48 } ], "Unit": "FPS", @@ -11649,9 +13021,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 83.93, + "int8": 115.48, + "fp16": 223.15, "fp32": "", "bf16": "" } @@ -11662,19 +13034,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 368.58, + "int8": "", "fp16": "", - "fp32": 166.93, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 25.41, + "token_int8": 18.99, + "token_fp16": 10.18 } ], "Unit": "FPS", @@ -11683,9 +13059,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 39.35, + "int8": 52.64, + "fp16": 98.23, "fp32": "", "bf16": "" } @@ -11696,19 +13072,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 224.44, + "int8": 2.49, "fp16": "", - "fp32": 81.26, - "bf16": "" + "fp32": 0.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11718,7 +13098,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 562.6, "fp16": "", "fp32": "", "bf16": "" @@ -11730,9 +13110,10 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { @@ -11742,7 +13123,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 14.41, + "token_int8": 9.12, + "token_fp16": 4.71 } ], "Unit": "FPS", @@ -11751,9 +13135,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 69.39, + "int8": 109.54, + "fp16": 211.91, "fp32": "", "bf16": "" } @@ -11764,19 +13148,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 285.59, + "int8": 4239.14, "fp16": "", - "fp32": 110.56, - "bf16": "" + "fp32": 2047.2, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11786,7 +13174,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.6, "fp16": "", "fp32": "", "bf16": "" @@ -11798,19 +13186,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 433.32, + "int8": "", "fp16": "", - "fp32": 147.73, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 15.66, + "token_fp16": 8.52 } ], "Unit": "FPS", @@ -11820,8 +13212,8 @@ "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 63.84, + "fp16": 117.37, "fp32": "", "bf16": "" } @@ -11832,19 +13224,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 112.49, + "int8": "", "fp16": "", - "fp32": 42.68, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 13.1, + "token_int8": 9.24, + "token_fp16": 4.75 } ], "Unit": "FPS", @@ -11853,9 +13249,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 76.33, + "int8": 108.16, + "fp16": 210.38, "fp32": "", "bf16": "" } @@ -11866,19 +13262,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", "Model": "resnet-50", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1107.14, + "int8": 762.32, "fp16": "", - "fp32": 471.86, - "bf16": "" + "fp32": 234.53, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11888,7 +13288,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.17, "fp16": "", "fp32": "", "bf16": "" @@ -11900,19 +13300,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 12.97, "fp16": "", - "fp32": 257.94, - "bf16": "" + "fp32": 3.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11922,7 +13326,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 102.02, "fp16": "", "fp32": "", "bf16": "" @@ -11934,19 +13338,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.76, + "int8": 1606.89, "fp16": "", - "fp32": 0.31, - "bf16": "" + "fp32": 589.62, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11956,7 +13364,7 @@ "Precisions": [ { "int4": "", - "int8": 1318.07, + "int8": 1.08, "fp16": "", "fp32": "", "bf16": "" @@ -11968,19 +13376,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.33, + "int8": "", "fp16": "", - "fp32": 0.13, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -11990,8 +13402,8 @@ "Precisions": [ { "int4": "", - "int8": 2998.35, - "fp16": "", + "int8": 40.27, + "fp16": 39.61, "fp32": "", "bf16": "" } @@ -12002,19 +13414,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.89, + "int8": "", "fp16": "", - "fp32": 0.23, - "bf16": "" + "fp32": 187.66, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12024,7 +13440,7 @@ "Precisions": [ { "int4": "", - "int8": 1117.77, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12036,19 +13452,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", + "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.67, + "int8": 389.04, "fp16": "", - "fp32": 0.89, - "bf16": "" + "fp32": 154.4, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12058,7 +13478,7 @@ "Precisions": [ { "int4": "", - "int8": 598.98, + "int8": 4.13, "fp16": "", "fp32": "", "bf16": "" @@ -12070,19 +13490,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.42, - "fp16": "", - "fp32": 1.29, - "bf16": "" + "int8": 385.87, + "fp16": 420.99, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12092,7 +13516,7 @@ "Precisions": [ { "int4": "", - "int8": 428.32, + "int8": 2.99, "fp16": "", "fp32": "", "bf16": "" @@ -12104,19 +13528,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.76, - "fp16": "", - "fp32": 0.79, - "bf16": "" + "int8": 426.56, + "fp16": 362.73, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12126,7 +13554,7 @@ "Precisions": [ { "int4": "", - "int8": 494.95, + "int8": 2.8, "fp16": "", "fp32": "", "bf16": "" @@ -12138,19 +13566,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.17, + "int8": "", "fp16": "", - "fp32": 0.92, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 22.66, + "token_int8": 18.13, + "token_fp16": "" } ], "Unit": "FPS", @@ -12159,8 +13591,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 463.7, + "int4": 44.13, + "int8": 55.13, "fp16": "", "fp32": "", "bf16": "" @@ -12172,19 +13604,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.97, + "int8": "", "fp16": "", - "fp32": 2.5, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 40.04, + "token_int8": 26.95, + "token_fp16": "" } ], "Unit": "FPS", @@ -12193,8 +13629,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 128.87, + "int4": 24.97, + "int8": 37.1, "fp16": "", "fp32": "", "bf16": "" @@ -12206,19 +13642,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.61, + "int8": "", "fp16": "", - "fp32": 1.34, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.22, + "token_int8": 33.88, + "token_fp16": 21.45 } ], "Unit": "FPS", @@ -12227,9 +13667,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 399.72, - "fp16": "", + "int4": 22.11, + "int8": 29.51, + "fp16": 46.62, "fp32": "", "bf16": "" } @@ -12240,19 +13680,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.96, + "int8": "", "fp16": "", - "fp32": 1.02, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.55, + "token_int8": 30.8, + "token_fp16": "" } ], "Unit": "FPS", @@ -12261,8 +13705,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 250.0, + "int4": 21.95, + "int8": 32.46, "fp16": "", "fp32": "", "bf16": "" @@ -12274,10 +13718,11 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ @@ -12286,7 +13731,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 69.44, + "token_int8": 57.9, + "token_fp16": 37.69 } ], "Unit": "FPS", @@ -12295,9 +13743,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 321.18, - "fp16": "", + "int4": 14.4, + "int8": 17.27, + "fp16": 26.53, "fp32": "", "bf16": "" } @@ -12308,19 +13756,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.82, - "fp16": "", - "fp32": 1.9, - "bf16": "" + "int8": 33.38, + "fp16": 19.04, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12330,7 +13782,7 @@ "Precisions": [ { "int4": "", - "int8": 169.83, + "int8": 48.67, "fp16": "", "fp32": "", "bf16": "" @@ -12342,19 +13794,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.5, + "int8": "", "fp16": "", - "fp32": 1.02, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.53, + "token_int8": 32.37, + "token_fp16": 20.21 } ], "Unit": "FPS", @@ -12363,9 +13819,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 436.59, - "fp16": "", + "int4": 21.96, + "int8": 30.89, + "fp16": 49.48, "fp32": "", "bf16": "" } @@ -12376,19 +13832,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.14, - "fp16": "", + "int8": 3134.27, + "fp16": 3004.5, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12398,7 +13858,7 @@ "Precisions": [ { "int4": "", - "int8": 247.47, + "int8": 0.57, "fp16": "", "fp32": "", "bf16": "" @@ -12410,19 +13870,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.0, + "int8": "", "fp16": "", - "fp32": 1.06, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 69.93, + "token_int8": 51.51, + "token_fp16": 32.84 } ], "Unit": "FPS", @@ -12431,9 +13895,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 374.78, - "fp16": "", + "int4": 14.3, + "int8": 19.41, + "fp16": 30.45, "fp32": "", "bf16": "" } @@ -12444,19 +13908,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.57, + "int8": "", "fp16": "", - "fp32": 1.28, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.8, + "token_int8": 32.78, + "token_fp16": "" } ], "Unit": "FPS", @@ -12465,8 +13933,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 413.46, + "int4": 21.83, + "int8": 30.5, "fp16": "", "fp32": "", "bf16": "" @@ -12478,19 +13946,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.97, - "fp16": "", - "fp32": 3.83, - "bf16": "" + "int8": 1921.18, + "fp16": 1329.28, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12500,7 +13972,7 @@ "Precisions": [ { "int4": "", - "int8": 101.21, + "int8": 0.78, "fp16": "", "fp32": "", "bf16": "" @@ -12512,19 +13984,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU-only", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.82, + "int8": 133.77, "fp16": "", - "fp32": 0.32, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12534,7 +14010,7 @@ "Precisions": [ { "int4": "", - "int8": 1223.85, + "int8": 13.93, "fp16": "", "fp32": "", "bf16": "" @@ -12546,19 +14022,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.34, - "fp16": "", - "fp32": 2.29, - "bf16": "" + "int8": 2200.83, + "fp16": 1665.15, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12568,7 +14048,7 @@ "Precisions": [ { "int4": "", - "int8": 239.97, + "int8": 0.78, "fp16": "", "fp32": "", "bf16": "" @@ -12580,19 +14060,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "stable-diffusion-v1-5", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.59, + "int8": "", "fp16": "", - "fp32": 0.85, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12602,8 +14086,8 @@ "Precisions": [ { "int4": "", - "int8": 628.98, - "fp16": "", + "int8": 2.33, + "fp16": 2.36, "fp32": "", "bf16": "" } @@ -12614,19 +14098,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 17.67, - "fp16": "", - "fp32": 4.59, - "bf16": "" + "int8": 759.93, + "fp16": 694.57, + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12636,7 +14124,7 @@ "Precisions": [ { "int4": "", - "int8": 115.61, + "int8": 1.96, "fp16": "", "fp32": "", "bf16": "" @@ -12648,19 +14136,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 58.24, + "int8": 36.93, "fp16": "", - "fp32": 15.05, - "bf16": "" + "fp32": 27.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12670,7 +14162,7 @@ "Precisions": [ { "int4": "", - "int8": 36.32, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12682,19 +14174,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 85.63, + "int8": 484.32, "fp16": "", - "fp32": 21.01, - "bf16": "" + "fp32": 278.4, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12704,7 +14200,7 @@ "Precisions": [ { "int4": "", - "int8": 25.38, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12716,19 +14212,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 438.33, + "int8": 112.23, "fp16": "", - "fp32": 30.86, - "bf16": 213.33 + "fp32": 42.14, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12738,10 +14238,10 @@ "Precisions": [ { "int4": "", - "int8": 7.94, + "int8": "", "fp16": "", "fp32": "", - "bf16": 13.88 + "bf16": "" } ], "Unit": "ms", @@ -12750,19 +14250,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 512.76, + "int8": 2.04, "fp16": "", - "fp32": 35.43, - "bf16": 276.38 + "fp32": 0.6, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12772,10 +14276,10 @@ "Precisions": [ { "int4": "", - "int8": 7.12, + "int8": "", "fp16": "", "fp32": "", - "bf16": 11.56 + "bf16": "" } ], "Unit": "ms", @@ -12784,19 +14288,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 41.98, + "int8": 216.96, "fp16": "", - "fp32": 10.9, - "bf16": "" + "fp32": 94.92, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12806,7 +14314,7 @@ "Precisions": [ { "int4": "", - "int8": 48.76, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12818,19 +14326,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 42.23, + "int8": "", "fp16": "", - "fp32": 10.5, - "bf16": "" + "fp32": 34.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12840,7 +14352,7 @@ "Precisions": [ { "int4": "", - "int8": 49.04, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12852,19 +14364,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 CPU+iGPU", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.45, + "int8": 61.06, "fp16": "", - "fp32": 1.66, - "bf16": "" + "fp32": 28.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12874,7 +14390,7 @@ "Precisions": [ { "int4": "", - "int8": 208.68, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -12886,19 +14402,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.88, + "int8": 15.44, "fp16": "", - "fp32": 1.23, - "bf16": "" + "fp32": 12.75, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12908,7 +14428,7 @@ "Precisions": [ { "int4": "", - "int8": 259.24, + "int8": 66.23, "fp16": "", "fp32": "", "bf16": "" @@ -12920,19 +14440,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.08, + "int8": 296.53, "fp16": "", - "fp32": 1.48, - "bf16": "" + "fp32": 183.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12942,7 +14466,7 @@ "Precisions": [ { "int4": "", - "int8": 234.74, + "int8": 3.8, "fp16": "", "fp32": "", "bf16": "" @@ -12954,19 +14478,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Processor N100 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 140.65, - "fp16": 92.11, - "fp32": "", - "bf16": "" + "int8": 48.77, + "fp16": "", + "fp32": 20.13, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -12976,7 +14504,7 @@ "Precisions": [ { "int4": "", - "int8": 15.32, + "int8": 21.88, "fp16": "", "fp32": "", "bf16": "" @@ -12988,19 +14516,23 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", + "Platform": "IntelĀ® Processor N100 CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.82, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 0.31, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13010,7 +14542,7 @@ "Precisions": [ { "int4": "", - "int8": 14.78, + "int8": 1224.62, "fp16": "", "fp32": "", "bf16": "" @@ -13022,19 +14554,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® Processor N100 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 35.4, - "fp16": 17.45, - "fp32": "", - "bf16": "" + "int8": 106.12, + "fp16": "", + "fp32": 49.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13044,7 +14580,7 @@ "Precisions": [ { "int4": "", - "int8": 35.98, + "int8": 9.72, "fp16": "", "fp32": "", "bf16": "" @@ -13056,19 +14592,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", + "Platform": "IntelĀ® Processor N100 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.16, - "fp16": 1.32, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 15.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13078,7 +14618,7 @@ "Precisions": [ { "int4": "", - "int8": 472.12, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -13090,19 +14630,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", + "Platform": "IntelĀ® Processor N100 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.16, - "fp16": 1.16, - "fp32": "", - "bf16": "" + "int8": 23.65, + "fp16": "", + "fp32": 12.86, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13112,7 +14656,7 @@ "Precisions": [ { "int4": "", - "int8": 870.37, + "int8": 43.43, "fp16": "", "fp32": "", "bf16": "" @@ -13124,19 +14668,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", + "Platform": "IntelĀ® Processor N100 iGPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.09, - "fp16": 2.78, + "int8": 33.69, + "fp16": 30.91, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13146,7 +14694,7 @@ "Precisions": [ { "int4": "", - "int8": 210.29, + "int8": 38.02, "fp16": "", "fp32": "", "bf16": "" @@ -13158,19 +14706,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 iGPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.8, - "fp16": 2.81, + "int8": 337.95, + "fp16": 267.38, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13180,7 +14732,7 @@ "Precisions": [ { "int4": "", - "int8": 164.94, + "int8": 3.84, "fp16": "", "fp32": "", "bf16": "" @@ -13192,19 +14744,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 iGPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.92, - "fp16": 2.89, + "int8": 81.72, + "fp16": 49.76, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13214,7 +14770,7 @@ "Precisions": [ { "int4": "", - "int8": 143.88, + "int8": 13.15, "fp16": "", "fp32": "", "bf16": "" @@ -13226,19 +14782,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", + "Platform": "IntelĀ® Processor N100 iGPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.32, - "fp16": 4.74, + "int8": 1.62, + "fp16": 1.01, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13248,7 +14808,7 @@ "Precisions": [ { "int4": "", - "int8": 117.76, + "int8": 622.97, "fp16": "", "fp32": "", "bf16": "" @@ -13260,19 +14820,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 iGPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.43, - "fp16": 3.17, + "int8": 164.31, + "fp16": 106.85, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13282,7 +14846,7 @@ "Precisions": [ { "int4": "", - "int8": 170.95, + "int8": 7.35, "fp16": "", "fp32": "", "bf16": "" @@ -13294,19 +14858,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® Processor N100 iGPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 10.46, - "fp16": 6.1, + "int8": 47.04, + "fp16": 34.97, "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13316,7 +14884,7 @@ "Precisions": [ { "int4": "", - "int8": 100.05, + "int8": 23.03, "fp16": "", "fp32": "", "bf16": "" @@ -13328,19 +14896,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 7.12, - "fp16": 4.0, - "fp32": "", - "bf16": "" + "int8": 218.18, + "fp16": "", + "fp32": 80.36, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13350,7 +14922,7 @@ "Precisions": [ { "int4": "", - "int8": 126.18, + "int8": 14.4, "fp16": "", "fp32": "", "bf16": "" @@ -13362,19 +14934,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 271.94, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 167.25, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13384,7 +14960,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.07, "fp16": "", "fp32": "", "bf16": "" @@ -13396,19 +14972,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.62, - "fp16": 1.01, - "fp32": "", - "bf16": "" + "int8": 3.26, + "fp16": "", + "fp32": 0.9, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13418,7 +14998,7 @@ "Precisions": [ { "int4": "", - "int8": 623.25, + "int8": 637.88, "fp16": "", "fp32": "", "bf16": "" @@ -13430,19 +15010,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 21.37, - "fp16": 12.68, - "fp32": "", - "bf16": "" + "int8": 5417.98, + "fp16": "", + "fp32": 1926.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13452,7 +15036,7 @@ "Precisions": [ { "int4": "", - "int8": 47.25, + "int8": 1.45, "fp16": "", "fp32": "", "bf16": "" @@ -13464,19 +15048,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": 10.09, - "fp32": "", - "bf16": "" + "int8": 979.5, + "fp16": "", + "fp32": 267.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13486,7 +15074,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 3.06, "fp16": "", "fp32": "", "bf16": "" @@ -13498,19 +15086,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 65.86, - "fp16": 39.71, - "fp32": "", - "bf16": "" + "int8": 17.65, + "fp16": "", + "fp32": 4.58, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13520,7 +15112,7 @@ "Precisions": [ { "int4": "", - "int8": 21.36, + "int8": 116.19, "fp16": "", "fp32": "", "bf16": "" @@ -13532,19 +15124,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.53, + "int8": 2104.85, "fp16": "", - "fp32": 0.87, - "bf16": "" + "fp32": 639.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13554,7 +15150,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.56, "fp16": "", "fp32": "", "bf16": "" @@ -13566,19 +15162,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1.25, + "int8": "", "fp16": "", - "fp32": 1.08, - "bf16": "" + "fp32": 206.18, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13600,19 +15200,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.37, + "int8": 440.56, "fp16": "", - "fp32": 2.71, - "bf16": "" + "fp32": 173.57, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13622,7 +15226,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 5.93, "fp16": "", "fp32": "", "bf16": "" @@ -13634,19 +15238,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 3.52, + "int8": 426.19, "fp16": "", - "fp32": 1.13, - "bf16": "" + "fp32": 162.63, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13656,7 +15264,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 11.09, "fp16": "", "fp32": "", "bf16": "" @@ -13668,19 +15276,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.77, + "int8": 411.51, "fp16": "", - "fp32": 1.75, - "bf16": "" + "fp32": 254.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13690,7 +15302,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 8.51, "fp16": "", "fp32": "", "bf16": "" @@ -13702,19 +15314,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", + "Parameters": { + "throughput": { "Precisions": [ { "int4": "", - "int8": 8.81, + "int8": 6.45, "fp16": "", - "fp32": 3.81, - "bf16": "" + "fp32": 1.65, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13724,7 +15340,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 321.85, "fp16": "", "fp32": "", "bf16": "" @@ -13736,19 +15352,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.33, + "int8": 10273.19, "fp16": "", - "fp32": 1.68, - "bf16": "" + "fp32": 3342.96, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13758,7 +15378,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.21, "fp16": "", "fp32": "", "bf16": "" @@ -13770,19 +15390,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 2125.81, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 570.61, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13792,7 +15416,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.84, "fp16": "", "fp32": "", "bf16": "" @@ -13804,19 +15428,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 5.71, + "int8": 41.83, "fp16": "", - "fp32": 2.16, - "bf16": "" + "fp32": 10.91, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13826,7 +15454,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 49.53, "fp16": "", "fp32": "", "bf16": "" @@ -13838,19 +15466,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 4376.71, "fp16": "", - "fp32": 3.05, - "bf16": "" + "fp32": 1244.57, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13860,7 +15492,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.22, "fp16": "", "fp32": "", "bf16": "" @@ -13872,19 +15504,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2.05, + "int8": "", "fp16": "", - "fp32": 0.61, - "bf16": "" + "fp32": 383.86, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13906,19 +15542,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 18.73, + "int8": 749.14, "fp16": "", - "fp32": 9.5, - "bf16": "" + "fp32": 338.04, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13928,7 +15568,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 4.21, "fp16": "", "fp32": "", "bf16": "" @@ -13940,19 +15580,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.04, + "int8": 622.71, "fp16": "", - "fp32": 4.95, - "bf16": "" + "fp32": 240.52, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13962,7 +15606,7 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 6.4, "fp16": "", "fp32": "", "bf16": "" @@ -13974,19 +15618,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 97.87, + "int8": 721.9, "fp16": "", - "fp32": 45.25, - "bf16": "" + "fp32": 423.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -13996,7 +15644,7 @@ "Precisions": [ { "int4": "", - "int8": 10.56, + "int8": 4.83, "fp16": "", "fp32": "", "bf16": "" @@ -14008,19 +15656,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 45.67, + "int8": 10.46, "fp16": "", - "fp32": 21.53, - "bf16": "" + "fp32": 2.45, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14030,7 +15682,7 @@ "Precisions": [ { "int4": "", - "int8": 22.79, + "int8": 221.46, "fp16": "", "fp32": "", "bf16": "" @@ -14042,19 +15694,23 @@ } }, { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 115.01, + "int8": 16509.95, "fp16": "", - "fp32": 36.97, - "bf16": "" + "fp32": 5201.56, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14064,7 +15720,7 @@ "Precisions": [ { "int4": "", - "int8": 9.06, + "int8": 0.59, "fp16": "", "fp32": "", "bf16": "" @@ -14076,19 +15732,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 211.59, + "int8": 3352.09, "fp16": "", - "fp32": 119.52, - "bf16": "" + "fp32": 825.5, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14098,7 +15758,7 @@ "Precisions": [ { "int4": "", - "int8": 4.95, + "int8": 1.34, "fp16": "", "fp32": "", "bf16": "" @@ -14110,19 +15770,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 327.96, + "int8": 60.91, "fp16": "", - "fp32": 163.65, - "bf16": "" + "fp32": 15.11, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14132,7 +15796,7 @@ "Precisions": [ { "int4": "", - "int8": 3.63, + "int8": 36.91, "fp16": "", "fp32": "", "bf16": "" @@ -14144,19 +15808,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 326.48, + "int8": 6975.09, "fp16": "", - "fp32": 98.82, - "bf16": "" + "fp32": 1755.62, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14166,7 +15834,7 @@ "Precisions": [ { "int4": "", - "int8": 4.67, + "int8": 0.77, "fp16": "", "fp32": "", "bf16": "" @@ -14178,19 +15846,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 383.99, + "int8": "", "fp16": "", - "fp32": 136.38, - "bf16": "" + "fp32": 571.3, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14200,7 +15872,7 @@ "Precisions": [ { "int4": "", - "int8": 4.16, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -14212,19 +15884,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Gold 6338N CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1073.43, + "int8": 1224.86, "fp16": "", - "fp32": 385.05, - "bf16": "" + "fp32": 495.73, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14234,7 +15910,7 @@ "Precisions": [ { "int4": "", - "int8": 1.31, + "int8": 2.98, "fp16": "", "fp32": "", "bf16": "" @@ -14246,19 +15922,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 335.03, + "int8": 587.54, "fp16": "", - "fp32": 182.29, - "bf16": "" + "fp32": 225.64, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14268,7 +15948,7 @@ "Precisions": [ { "int4": "", - "int8": 3.15, + "int8": 9.18, "fp16": "", "fp32": "", "bf16": "" @@ -14280,19 +15960,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 495.53, + "int8": 580.8, "fp16": "", - "fp32": 147.22, - "bf16": "" + "fp32": 343.39, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14302,7 +15986,7 @@ "Precisions": [ { "int4": "", - "int8": 2.19, + "int8": 6.9, "fp16": "", "fp32": "", "bf16": "" @@ -14314,19 +15998,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 385.05, + "int8": 8.58, "fp16": "", - "fp32": 100.34, - "bf16": "" + "fp32": 2.26, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14336,7 +16024,7 @@ "Precisions": [ { "int4": "", - "int8": 2.83, + "int8": 248.72, "fp16": "", "fp32": "", "bf16": "" @@ -14348,19 +16036,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 832.22, + "int8": 14930.31, "fp16": "", - "fp32": 284.89, - "bf16": "" + "fp32": 4646.16, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14370,7 +16062,7 @@ "Precisions": [ { "int4": "", - "int8": 1.73, + "int8": 0.93, "fp16": "", "fp32": "", "bf16": "" @@ -14382,19 +16074,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 423.45, + "int8": 2965.31, "fp16": "", - "fp32": 153.0, - "bf16": "" + "fp32": 761.01, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14404,7 +16100,7 @@ "Precisions": [ { "int4": "", - "int8": 3.88, + "int8": 1.59, "fp16": "", "fp32": "", "bf16": "" @@ -14416,19 +16112,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 58.15, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 15.0, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14438,7 +16138,7 @@ "Precisions": [ { "int4": "", - "int8": 2.45, + "int8": 37.18, "fp16": "", "fp32": "", "bf16": "" @@ -14450,19 +16150,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 275.79, + "int8": 6130.48, "fp16": "", - "fp32": 151.19, - "bf16": "" + "fp32": 1654.84, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14472,7 +16176,7 @@ "Precisions": [ { "int4": "", - "int8": 3.22, + "int8": 1.2, "fp16": "", "fp32": "", "bf16": "" @@ -14484,19 +16188,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 351.76, + "int8": "", "fp16": "", - "fp32": 183.93, - "bf16": "" + "fp32": 512.57, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14506,7 +16214,7 @@ "Precisions": [ { "int4": "", - "int8": 3.41, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -14518,19 +16226,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1607.16, + "int8": 996.59, "fp16": "", - "fp32": 587.94, - "bf16": "" + "fp32": 452.05, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14540,7 +16252,7 @@ "Precisions": [ { "int4": "", - "int8": 1.09, + "int8": 3.6, "fp16": "", "fp32": "", "bf16": "" @@ -14552,19 +16264,23 @@ } }, { - "Platform": "IntelĀ® Processor N100 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 105.81, + "int8": 881.04, "fp16": "", - "fp32": 49.62, - "bf16": "" + "fp32": 338.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14574,7 +16290,7 @@ "Precisions": [ { "int4": "", - "int8": 9.76, + "int8": 5.18, "fp16": "", "fp32": "", "bf16": "" @@ -14586,19 +16302,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 575.93, + "int8": 1009.71, "fp16": "", - "fp32": 272.53, - "bf16": "" + "fp32": 562.38, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14608,7 +16328,7 @@ "Precisions": [ { "int4": "", - "int8": 2.33, + "int8": 4.28, "fp16": "", "fp32": "", "bf16": "" @@ -14620,19 +16340,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 202.32, + "int8": "", "fp16": "", - "fp32": 120.89, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 20.78, + "token_int8": 14.18, + "token_fp16": 7.72 } ], "Unit": "FPS", @@ -14641,9 +16365,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 5.11, - "fp16": "", + "int4": 48.12, + "int8": 70.5, + "fp16": 129.51, "fp32": "", "bf16": "" } @@ -14654,19 +16378,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2068.2, + "int8": "", "fp16": "", - "fp32": 637.61, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 22.79, + "token_int8": 15.56, + "token_fp16": 8.48 } ], "Unit": "FPS", @@ -14675,9 +16403,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.66, - "fp16": "", + "int4": 43.86, + "int8": 64.26, + "fp16": 117.92, "fp32": "", "bf16": "" } @@ -14688,19 +16416,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6032.65, + "int8": "", "fp16": "", - "fp32": 1652.7, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 25.41, + "token_int8": 18.68, + "token_fp16": 10.61 } ], "Unit": "FPS", @@ -14709,9 +16441,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.23, - "fp16": "", + "int4": 39.34, + "int8": 53.51, + "fp16": 94.17, "fp32": "", "bf16": "" } @@ -14723,8 +16455,9 @@ }, { "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Model": "llama-3-8b", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { @@ -14733,8 +16466,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": 2525.11, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.07, + "token_int8": 17.66, + "token_fp16": 9.72 } ], "Unit": "FPS", @@ -14743,9 +16479,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 0.69, - "fp16": "", + "int4": 38.35, + "int8": 56.62, + "fp16": 102.88, "fp32": "", "bf16": "" } @@ -14756,19 +16492,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": false, + "whats_new_model": true, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 23599.57, + "int8": "", "fp16": "", - "fp32": 3405.56, - "bf16": 12205.1 + "fp32": "", + "bf16": "", + "token_int4": 46.81, + "token_int8": 33.54, + "token_fp16": 19.32 } ], "Unit": "FPS", @@ -14777,11 +16517,11 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 0.74, - "fp16": "", + "int4": 21.36, + "int8": 29.81, + "fp16": 51.74, "fp32": "", - "bf16": 0.87 + "bf16": "" } ], "Unit": "ms", @@ -14790,19 +16530,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 14.73, "fp16": "", - "fp32": 4698.93, - "bf16": 16520.05 + "fp32": 3.42, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -14812,10 +16556,10 @@ "Precisions": [ { "int4": "", - "int8": 0.83, + "int8": "", "fp16": "", "fp32": "", - "bf16": 1.2 + "bf16": "" } ], "Unit": "ms", @@ -14824,19 +16568,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4267.65, + "int8": "", "fp16": "", - "fp32": 1238.3, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.89, + "token_int8": 18.54, + "token_fp16": 10.22 } ], "Unit": "FPS", @@ -14845,9 +16593,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.29, - "fp16": "", + "int4": 37.18, + "int8": 53.93, + "fp16": 97.8, "fp32": "", "bf16": "" } @@ -14858,155 +16606,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4642.89, - "fp16": "", - "fp32": 1227.04, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 0.93, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 682.74, - "fp16": "", - "fp32": 243.93, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 533.14, - "fp16": "", - "fp32": 179.15, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 663.32, - "fp16": "", - "fp32": 226.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.25, + "int8": 22703.47, "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2300.75, - "fp16": 1706.66, - "fp32": "", - "bf16": "" + "fp32": 6937.71, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -15016,7 +16632,7 @@ "Precisions": [ { "int4": "", - "int8": 0.91, + "int8": 0.58, "fp16": "", "fp32": "", "bf16": "" @@ -15028,10 +16644,11 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ @@ -15040,75 +16657,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.28, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1024.85, - "fp16": 777.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.33, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 704.76, - "fp16": 451.92, - "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 39.41, + "token_int8": 29.28, + "token_fp16": 17.35 } ], "Unit": "FPS", @@ -15117,9 +16669,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.65, - "fp16": "", + "int4": 25.37, + "int8": 34.15, + "fp16": 57.61, "fp32": "", "bf16": "" } @@ -15130,53 +16682,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "qwen2-7b", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 705.61, - "fp16": 458.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.66, + "int8": "", "fp16": "", "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 229.58, - "fp16": 198.86, - "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 28.26, + "token_int8": 19.32, + "token_fp16": 10.27 } ], "Unit": "FPS", @@ -15185,9 +16707,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 7.67, - "fp16": "", + "int4": 35.38, + "int8": 51.74, + "fp16": 97.35, "fp32": "", "bf16": "" } @@ -15198,9267 +16720,23 @@ } }, { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 210.98, - "fp16": 137.08, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.21, + "int8": 4874.95, "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 92.82, - "fp16": 95.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 13.65, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 396.49, - "fp16": 222.23, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.31, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 392.98, - "fp16": 229.59, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.07, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 405.21, - "fp16": 238.92, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.72, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 298.42, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 360.04, - "fp16": 225.51, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.42, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 748.06, - "fp16": 416.28, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 488.8, - "fp16": 302.47, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.51, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 164.35, - "fp16": 106.86, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 7.36, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 979.63, - "fp16": 802.16, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.15, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 974.05, - "fp16": 701.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 985.08, - "fp16": 990.93, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 1.87, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 234.86, - "fp16": "", - "fp32": 116.72, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 111.52, - "fp16": "", - "fp32": 98.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 318.36, - "fp16": "", - "fp32": 194.83, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 117.54, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 471.49, - "fp16": "", - "fp32": 189.58, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 650.89, - "fp16": "", - "fp32": 298.25, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 160.27, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 380.11, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 554.71, - "fp16": "", - "fp32": 229.14, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 350.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 217.21, - "fp16": "", - "fp32": 95.95, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1045.56, - "fp16": "", - "fp32": 608.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 748.16, - "fp16": "", - "fp32": 419.13, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1.49, - "fp16": "", - "fp32": 0.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 672.94, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.54, - "fp16": "", - "fp32": 1.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 403.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.62, - "fp16": "", - "fp32": 2.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 306.85, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.8, - "fp16": "", - "fp32": 0.82, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 348.07, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4.66, - "fp16": "", - "fp32": 0.83, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 310.23, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.67, - "fp16": "", - "fp32": 3.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 93.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.95, - "fp16": "", - "fp32": 2.06, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 247.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.54, - "fp16": "", - "fp32": 1.66, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 154.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 4.97, - "fp16": "", - "fp32": 1.24, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 206.27, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 9.59, - "fp16": "", - "fp32": 2.4, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 121.76, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 5.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 292.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.22, - "fp16": "", - "fp32": 1.55, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 178.96, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.06, - "fp16": "", - "fp32": 1.66, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 254.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.79, - "fp16": "", - "fp32": 1.91, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 282.74, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.74, - "fp16": "", - "fp32": 4.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 72.32, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.13, - "fp16": "", - "fp32": 3.29, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 174.28, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.45, - "fp16": "", - "fp32": 1.35, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 415.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 28.98, - "fp16": "", - "fp32": 7.41, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 69.96, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 96.65, - "fp16": "", - "fp32": 22.64, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 23.08, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 139.91, - "fp16": "", - "fp32": 33.26, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 15.34, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 511.53, - "fp16": "", - "fp32": 48.78, - "bf16": 276.34 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.66, - "fp16": "", - "fp32": "", - "bf16": 9.97 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 617.57, - "fp16": "", - "fp32": 56.93, - "bf16": 334.64 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.71, - "fp16": "", - "fp32": "", - "bf16": 7.81 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 69.09, - "fp16": "", - "fp32": 16.42, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 29.59, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 69.98, - "fp16": "", - "fp32": 16.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 30.05, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.9, - "fp16": "", - "fp32": 1.98, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 152.81, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.79, - "fp16": "", - "fp32": 1.6, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 179.19, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.92, - "fp16": "", - "fp32": 2.48, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 185.72, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 276.55, - "fp16": 194.61, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.54, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 55.61, - "fp16": 34.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.05, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 9.86, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 9.91, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 46.18, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 3.4, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 294.79, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1.98, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 505.59, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.29, - "fp16": 4.48, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 121.49, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 10.02, - "fp16": 4.68, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 79.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 9.86, - "fp16": 4.73, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 82.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 14.62, - "fp16": 7.64, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 63.94, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": 5.13, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 93.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.44, - "fp16": 9.72, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 54.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.84, - "fp16": 6.49, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 68.04, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.56, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 390.36, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 36.35, - "fp16": 23.49, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 25.29, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.67, - "fp16": 18.53, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 105.25, - "fp16": 74.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.37, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.85, - "fp16": "", - "fp32": 4.55, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 1.47, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.38, - "fp16": "", - "fp32": 2.41, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 15.79, - "fp16": "", - "fp32": 6.34, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.73, - "fp16": "", - "fp32": 2.82, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 6.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 9.45, - "fp16": "", - "fp32": 3.58, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 5.04, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.21, - "fp16": "", - "fp32": 16.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 7.99, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 5.65, - "fp16": "", - "fp32": 2.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 182.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2.58, - "fp16": "", - "fp32": 1.04, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 393.57, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.38, - "fp16": "", - "fp32": 1.86, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 158.93, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.59, - "fp16": "", - "fp32": 6.77, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 80.57, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.57, - "fp16": "", - "fp32": 9.33, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 57.66, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 18.83, - "fp16": "", - "fp32": 5.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 76.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 23.43, - "fp16": "", - "fp32": 7.32, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 70.0, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 66.59, - "fp16": "", - "fp32": 20.16, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.62, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 19.4, - "fp16": "", - "fp32": 10.16, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 52.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 27.81, - "fp16": "", - "fp32": 7.85, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 37.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 20.29, - "fp16": "", - "fp32": 5.69, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 49.78, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 49.8, - "fp16": "", - "fp32": 14.8, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 25.53, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 25.87, - "fp16": "", - "fp32": 8.21, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 65.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 37.03, - "fp16": "", - "fp32": 10.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 36.87, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 15.32, - "fp16": "", - "fp32": 7.94, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 50.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 19.78, - "fp16": "", - "fp32": 9.0, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 54.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 95.49, - "fp16": "", - "fp32": 30.36, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 15.47, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 6.13, - "fp16": "", - "fp32": 2.62, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 167.91, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 32.67, - "fp16": "", - "fp32": 14.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 34.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 12.06, - "fp16": "", - "fp32": 6.75, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 83.73, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 117.81, - "fp16": "", - "fp32": 35.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.72, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 334.59, - "fp16": "", - "fp32": 102.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.17, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 533.18, - "fp16": "", - "fp32": 154.64, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.76, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1419.98, - "fp16": "", - "fp32": 214.44, - "bf16": 830.31 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.62, - "fp16": "", - "fp32": "", - "bf16": 5.98 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1755.21, - "fp16": "", - "fp32": 265.47, - "bf16": 1055.23 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.46, - "fp16": "", - "fp32": "", - "bf16": 5.21 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 251.62, - "fp16": "", - "fp32": 74.96, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 265.17, - "fp16": "", - "fp32": 76.67, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.63, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 44.74, - "fp16": "", - "fp32": 12.8, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 36.67, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 33.1, - "fp16": "", - "fp32": 9.15, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 45.86, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 288.05, - "fp16": 333.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.92, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 105.43, - "fp16": 93.59, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.01, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 50.31, - "fp16": 40.99, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 21.31, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 50.34, - "fp16": 41.31, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 21.03, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 14.53, - "fp16": 9.52, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 71.11, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 7.37, - "fp16": 7.57, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 141.78, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.36, - "fp16": 17.91, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 34.6, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 33.47, - "fp16": 18.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 24.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 35.33, - "fp16": 18.84, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 24.68, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 48.16, - "fp16": 28.25, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 22.07, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 33.73, - "fp16": 18.76, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 29.57, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 63.37, - "fp16": 36.52, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 17.02, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 44.64, - "fp16": 24.85, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 20.43, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 53.52, - "fp16": 30.53, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 17.5, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 11.12, - "fp16": 7.41, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 91.66, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 132.43, - "fp16": 70.84, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 7.23, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 120.91, - "fp16": 58.45, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.06, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 17.32, - "fp16": "", - "fp32": 7.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 8.67, - "fp16": "", - "fp32": 7.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 32.84, - "fp16": "", - "fp32": 18.05, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 7.54, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 32.16, - "fp16": "", - "fp32": 12.68, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 55.36, - "fp16": "", - "fp32": 25.15, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 29.16, - "fp16": "", - "fp32": 11.65, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 24.74, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 39.77, - "fp16": "", - "fp32": 15.13, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 52.61, - "fp16": "", - "fp32": 20.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 14.47, - "fp16": "", - "fp32": 4.84, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 120.56, - "fp16": "", - "fp32": 58.53, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 70.85, - "fp16": "", - "fp32": 32.17, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 21.65, - "fp16": "", - "fp32": 11.8, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 47.52, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 10.28, - "fp16": "", - "fp32": 5.11, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 100.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 25.98, - "fp16": "", - "fp32": 9.65, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 40.09, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i3-8100 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 53.64, - "fp16": "", - "fp32": 32.13, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.04, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-10500TE CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 81.71, - "fp16": "", - "fp32": 45.23, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 13.68, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 70.36, - "fp16": "", - "fp32": 27.96, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.88, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 91.48, - "fp16": "", - "fp32": 36.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 18.13, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-13600K CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 272.16, - "fp16": "", - "fp32": 103.27, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.12, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-8500 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 84.24, - "fp16": "", - "fp32": 49.32, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.97, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 111.42, - "fp16": "", - "fp32": 40.37, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 76.56, - "fp16": "", - "fp32": 27.61, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 13.37, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 200.97, - "fp16": "", - "fp32": 75.16, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.52, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 101.43, - "fp16": "", - "fp32": 40.83, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 16.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.34, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-8700T CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 70.91, - "fp16": "", - "fp32": 40.94, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 11.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-10900TE CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 89.73, - "fp16": "", - "fp32": 48.88, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 12.74, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 387.96, - "fp16": "", - "fp32": 154.46, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.11, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 23.66, - "fp16": "", - "fp32": 12.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 43.5, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® W1290P CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 136.65, - "fp16": "", - "fp32": 71.98, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® E-2124G CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 51.95, - "fp16": "", - "fp32": 32.5, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 19.42, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Gold 5218T CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 442.04, - "fp16": "", - "fp32": 173.4, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.02, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8280 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 997.47, - "fp16": "", - "fp32": 452.11, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 1718.36, - "fp16": "", - "fp32": 701.27, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.39, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 2411.67, - "fp16": "", - "fp32": 959.63, - "bf16": 2381.88 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.11, - "fp16": "", - "fp32": "", - "bf16": 2.53 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 1262.79, - "bf16": 3461.09 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.15, - "fp16": "", - "fp32": "", - "bf16": 2.62 - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Gold 6238L CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 757.45, - "fp16": "", - "fp32": 337.98, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 4.31, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Silver 4316 CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 859.9, - "fp16": "", - "fp32": 340.51, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.38, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 166.62, - "fp16": "", - "fp32": 64.56, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 12.93, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 127.76, - "fp16": "", - "fp32": 46.89, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 14.67, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 139.58, - "fp16": "", - "fp32": 59.12, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.46, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 803.05, - "fp16": 733.02, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.32, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.14, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 322.1, - "fp16": 299.84, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.75, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H NPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 126.74, - "fp16": 129.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H NPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 127.62, - "fp16": 131.24, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.95, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V NPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 390.98, - "fp16": 506.71, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.51, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 60.91, - "fp16": 44.63, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 18.54, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 31.72, - "fp16": 33.09, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 35.85, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 121.95, - "fp16": 81.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 10.32, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 130.79, - "fp16": 80.07, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 8.12, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 133.28, - "fp16": 83.52, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.92, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 164.13, - "fp16": 112.15, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 117.62, - "fp16": 77.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 9.6, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 218.62, - "fp16": 149.58, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.56, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 164.07, - "fp16": 107.64, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 6.35, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 5.83, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 47.04, - "fp16": 34.98, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 23.03, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 388.85, - "fp16": 291.51, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 2.61, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 386.42, - "fp16": 271.08, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.16, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 442.05, - "fp16": 412.46, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": 3.33, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® x7425E CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 68.47, - "fp16": "", - "fp32": 36.85, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® AtomĀ® X6425E CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 36.31, - "fp16": "", - "fp32": 34.01, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® CeleronĀ® 6305E CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 114.66, - "fp16": "", - "fp32": 78.23, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1235U Processor CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 36.38, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i5-1335U Processor CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 126.5, - "fp16": "", - "fp32": 58.24, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185G7 CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 184.9, - "fp16": "", - "fp32": 103.88, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1185GRE CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 100.37, - "fp16": "", - "fp32": 51.71, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-12700H CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": 117.6, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1355U Processor CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 154.41, - "fp16": "", - "fp32": 72.66, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 227.78, - "fp16": "", - "fp32": 103.62, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Processor N100 CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Mobile Platforms (IntelĀ® Atomā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 61.09, - "fp16": "", - "fp32": 28.95, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 155H CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 369.1, - "fp16": "", - "fp32": 228.3, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": 253.52, - "fp16": "", - "fp32": 149.21, - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 15.4, - "int8": 11.0, - "fp16": 5.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 64.94, - "int8": 90.95, - "fp16": 176.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 30.9, - "int8": 22.8, - "fp16": 12.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 32.39, - "int8": 43.95, - "fp16": 78.58, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 33.7, - "int8": 25.7, - "fp16": 16.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 29.69, - "int8": 38.85, - "fp16": 61.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 39.5, - "int8": 29.1, - "fp16": 18.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 25.3, - "int8": 34.31, - "fp16": 53.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.4, - "int8": 6.2, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 106.9, - "int8": 162.58, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 56.8, - "int8": 45.2, - "fp16": 26.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 17.61, - "int8": 22.14, - "fp16": 37.67, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 45.5, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 21.99, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.4, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 87.86, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 15.8, - "int8": 11.3, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 63.49, - "int8": 88.47, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 18.0, - "int8": 12.6, - "fp16": 7.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 55.41, - "int8": 79.46, - "fp16": 142.26, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.8, - "int8": 9.4, - "fp16": 4.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 67.35, - "int8": 106.1, - "fp16": 205.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 28.4, - "int8": 19.7, - "fp16": 10.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 35.25, - "int8": 50.64, - "fp16": 91.76, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 30.6, - "int8": 21.6, - "fp16": 14.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 32.72, - "int8": 46.36, - "fp16": 71.16, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 35.7, - "int8": 24.3, - "fp16": 15.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 28.02, - "int8": 41.12, - "fp16": 62.78, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.8, - "int8": 5.4, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 102.44, - "int8": 186.87, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 44.7, - "int8": 32.1, - "fp16": 21.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 22.37, - "int8": 31.17, - "fp16": 47.04, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 34.7, - "int8": 25.8, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 28.79, - "int8": 38.77, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.7, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 174.81, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 6.5, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 153.92, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.2, - "int8": 9.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 70.34, - "int8": 102.9, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 17.6, - "int8": 10.8, - "fp16": 6.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 56.96, - "int8": 92.92, - "fp16": 164.05, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 13.4, - "int8": 9.3, - "fp16": 5.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 74.6, - "int8": 107.69, - "fp16": 201.87, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 25.2, - "int8": 18.4, - "fp16": 10.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 39.73, - "int8": 54.31, - "fp16": 94.63, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 26.7, - "int8": 20.3, - "fp16": 14.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 37.4, - "int8": 49.27, - "fp16": 70.22, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 32.5, - "int8": 23.1, - "fp16": 16.4, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 30.75, - "int8": 43.23, - "fp16": 60.97, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 8.6, - "int8": 5.3, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 116.29, - "int8": 189.6, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 45.3, - "int8": 35.9, - "fp16": 21.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 22.08, - "int8": 27.87, - "fp16": 45.77, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 38.1, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 26.27, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.4, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 184.58, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 13.3, - "int8": 9.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 75.47, - "int8": 109.71, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "llama-2-7b-chat", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 15.6, - "int8": 10.4, - "fp16": 5.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 64.18, - "int8": 95.84, - "fp16": 171.66, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.8, - "int8": 8.7, - "fp16": 4.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 84.87, - "int8": 115.0, - "fp16": 223.46, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 26.1, - "int8": 17.6, - "fp16": 9.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 38.25, - "int8": 56.78, - "fp16": 102.16, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 25.3, - "int8": 19.3, - "fp16": 13.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 39.49, - "int8": 51.8, - "fp16": 75.23, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 28.9, - "int8": 21.2, - "fp16": 15.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 34.63, - "int8": 47.15, - "fp16": 66.83, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 4.8, - "int8": 4.9, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 209.01, - "int8": 205.41, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 45.6, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 21.92, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 35.3, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 28.33, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.7, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 175.65, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.3, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 70.04, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 17.0, - "int8": 9.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 58.82, - "int8": 102.99, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.4, - "int8": 9.1, - "fp16": 4.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 69.6, - "int8": 109.3, - "fp16": 212.27, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 27.1, - "int8": 18.3, - "fp16": 10.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 36.86, - "int8": 54.59, - "fp16": 97.42, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 27.8, - "int8": 20.0, - "fp16": 13.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 35.91, - "int8": 49.99, - "fp16": 72.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 33.0, - "int8": 22.7, - "fp16": 15.8, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 30.3, - "int8": 44.07, - "fp16": 63.31, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.0, - "int8": 5.0, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 111.21, - "int8": 201.82, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 46.0, - "int8": 34.4, - "fp16": 21.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 21.76, - "int8": 29.06, - "fp16": 47.66, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 38.0, - "int8": "", - "fp16": 17.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 26.33, - "int8": "", - "fp16": 55.81, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.1, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 90.09, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 5.8, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 171.03, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 14.6, - "int8": 9.3, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 68.27, - "int8": 108.02, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 16.7, - "int8": 10.2, - "fp16": 5.7, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 60.02, - "int8": 98.33, - "fp16": 176.33, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 23.3, - "int8": 15.6, - "fp16": 8.5, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 42.95, - "int8": 64.22, - "fp16": 117.2, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 40.8, - "int8": 29.1, - "fp16": 17.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 24.53, - "int8": 34.38, - "fp16": 58.33, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 43.6, - "int8": 33.3, - "fp16": 22.0, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 22.94, - "int8": 30.03, - "fp16": 45.44, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 52.2, - "int8": 39.0, - "fp16": 25.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 19.15, - "int8": 25.66, - "fp16": 38.58, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H CPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 11.6, - "int8": 8.6, - "fp16": 6.9, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 86.42, - "int8": 116.68, - "fp16": 145.89, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 170 dGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 55.6, - "int8": 46.6, - "fp16": 31.6, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 17.98, - "int8": 21.47, - "fp16": 31.63, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 41.9, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 23.88, - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Accelerator Platforms", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 18.8, - "int8": 10.7, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 53.33, - "int8": 93.53, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 9.7, - "int8": 6.1, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 102.87, - "int8": 165.24, - "fp16": "", - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 165H iGPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 23.1, - "int8": 15.7, - "fp16": 9.1, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 43.28, - "int8": 63.82, - "fp16": 109.36, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": 25.6, - "int8": 16.7, - "fp16": 10.3, - "fp32": "", - "bf16": "" - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - }, - "latency": { - "Precisions": [ - { - "int4": 38.99, - "int8": 59.98, - "fp16": 96.89, - "fp32": "", - "bf16": "" - } - ], - "Unit": "ms", - "UnitDesc": "lower is better" - } - } - }, - { - "Platform": "IntelĀ® Coreā„¢ i9-13900K CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "int4": "", - "int8": "", - "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1144.73, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24468,8 +16746,8 @@ "Precisions": [ { "int4": "", - "int8": 42.87, - "fp16": 41.99, + "int8": 1.07, + "fp16": "", "fp32": "", "bf16": "" } @@ -24481,18 +16759,22 @@ }, { "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 84.6, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 20.95, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24502,8 +16784,8 @@ "Precisions": [ { "int4": "", - "int8": 14.24, - "fp16": 14.44, + "int8": "", + "fp16": "", "fp32": "", "bf16": "" } @@ -24514,19 +16796,23 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 10174.18, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 2524.59, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24536,8 +16822,8 @@ "Precisions": [ { "int4": "", - "int8": 4.67, - "fp16": 4.55, + "int8": 0.7, + "fp16": "", "fp32": "", "bf16": "" } @@ -24548,9 +16834,10 @@ } }, { - "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", "Model": "stable-diffusion-v1-5", - "Checked": "true", + "featured_SKU": false, + "whats_new_model": false, "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { @@ -24560,7 +16847,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24570,8 +16860,8 @@ "Precisions": [ { "int4": "", - "int8": 4.13, - "fp16": 4.04, + "int8": 13.34, + "fp16": 13.66, "fp32": "", "bf16": "" } @@ -24582,10 +16872,11 @@ } }, { - "Platform": "IntelĀ® Arcā„¢ A-Series Graphics dGPU", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "yolo11", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ @@ -24593,8 +16884,11 @@ "int4": "", "int8": "", "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 803.12, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24604,8 +16898,8 @@ "Precisions": [ { "int4": "", - "int8": 2.55, - "fp16": 2.47, + "int8": "", + "fp16": "", "fp32": "", "bf16": "" } @@ -24616,19 +16910,23 @@ } }, { - "Platform": "IntelĀ® Data Center GPU Flex 140 dGPU", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Accelerator Platforms", + "Platform": "IntelĀ® XeonĀ® Platinum 8380 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 1704.08, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 697.23, + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24638,8 +16936,8 @@ "Precisions": [ { "int4": "", - "int8": 10.8, - "fp16": 10.89, + "int8": 2.36, + "fp16": "", "fp32": "", "bf16": "" } @@ -24650,19 +16948,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ i7-1360P iGPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 3023.92, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 483.11, + "bf16": 1976.63, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24672,10 +16974,10 @@ "Precisions": [ { "int4": "", - "int8": 33.5, - "fp16": 32.47, + "int8": 3.79, + "fp16": "", "fp32": "", - "bf16": "" + "bf16": 4.84 } ], "Unit": "ms", @@ -24684,19 +16986,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 9 processor 288V iGPU-only", - "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 1445.78, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 861.51, + "bf16": 1021.75, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24706,10 +17012,10 @@ "Precisions": [ { "int4": "", - "int8": 5.67, - "fp16": 5.33, + "int8": 4.69, + "fp16": "", "fp32": "", - "bf16": "" + "bf16": 5.16 } ], "Unit": "ms", @@ -24718,19 +17024,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 43.86, + "int8": "", "fp16": "", - "fp32": 19.46, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 22.71, + "token_int8": 16.83, + "token_fp16": 10.76 } ], "Unit": "FPS", @@ -24739,9 +17049,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 30.78, - "fp16": "", + "int4": 44.03, + "int8": 59.39, + "fp16": 92.87, "fp32": "", "bf16": "" } @@ -24752,19 +17062,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "bert-large-uncased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.63, + "int8": "", "fp16": "", - "fp32": 1.74, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 23.7, + "token_int8": 16.93, + "token_fp16": 11.27 } ], "Unit": "FPS", @@ -24773,9 +17087,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 281.45, - "fp16": "", + "int4": 42.19, + "int8": 59.04, + "fp16": 88.67, "fp32": "", "bf16": "" } @@ -24786,19 +17100,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 67.33, + "int8": "", "fp16": "", - "fp32": 45.22, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.11, + "token_int8": 20.1, + "token_fp16": 14.19 } ], "Unit": "FPS", @@ -24807,9 +17125,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 18.61, - "fp16": "", + "int4": 38.29, + "int8": 49.73, + "fp16": 70.45, "fp32": "", "bf16": "" } @@ -24820,19 +17138,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 0.83, + "int8": "", "fp16": "", - "fp32": 0.2, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 26.02, + "token_int8": 18.97, + "token_fp16": 13.23 } ], "Unit": "FPS", @@ -24841,9 +17163,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1662.21, - "fp16": "", + "int4": 38.42, + "int8": 52.71, + "fp16": 75.57, "fp32": "", "bf16": "" } @@ -24854,19 +17176,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1206.23, + "int8": "", "fp16": "", - "fp32": 587.85, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 45.68, + "token_int8": 36.96, + "token_fp16": 27.27 } ], "Unit": "FPS", @@ -24875,9 +17201,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 1.27, - "fp16": "", + "int4": 21.89, + "int8": 27.05, + "fp16": 36.67, "fp32": "", "bf16": "" } @@ -24888,19 +17214,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 248.53, + "int8": 62.13, "fp16": "", - "fp32": 60.14, - "bf16": "" + "fp32": 5.19, + "bf16": 37.54, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24910,10 +17240,10 @@ "Precisions": [ { "int4": "", - "int8": 5.96, + "int8": 58.49, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 81.95 } ], "Unit": "ms", @@ -24922,19 +17252,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 510.87, + "int8": "", "fp16": "", - "fp32": 166.45, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 27.42, + "token_int8": 19.9, + "token_fp16": 13.72 } ], "Unit": "FPS", @@ -24943,9 +17277,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 2.76, - "fp16": "", + "int4": 36.46, + "int8": 50.24, + "fp16": 72.84, "fp32": "", "bf16": "" } @@ -24956,19 +17290,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4.55, + "int8": 38538.65, "fp16": "", - "fp32": 1.1, - "bf16": "" + "fp32": 10274.08, + "bf16": 25608.67, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -24978,10 +17316,10 @@ "Precisions": [ { "int4": "", - "int8": 310.48, + "int8": 0.65, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 0.66 } ], "Unit": "ms", @@ -24990,19 +17328,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.44, + "int8": "", "fp16": "", - "fp32": 1.82, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 33.53, + "token_fp16": 23.1 } ], "Unit": "FPS", @@ -25011,9 +17353,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 213.98, - "fp16": "", + "int4": 24.06, + "int8": 29.82, + "fp16": 43.29, "fp32": "", "bf16": "" } @@ -25024,19 +17366,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 31.79, + "int8": "", "fp16": "", - "fp32": 8.85, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 30.03, + "token_int8": 22.14, + "token_fp16": 13.95 } ], "Unit": "FPS", @@ -25045,9 +17391,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 42.2, - "fp16": "", + "int4": 33.3, + "int8": 45.16, + "fp16": 71.68, "fp32": "", "bf16": "" } @@ -25058,19 +17404,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU-only", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 116.4, + "int8": 19226.96, "fp16": "", - "fp32": 42.84, - "bf16": "" + "fp32": 1597.37, + "bf16": 7480.12, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25080,10 +17430,10 @@ "Precisions": [ { "int4": "", - "int8": 10.51, + "int8": 1.01, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 1.25 } ], "Unit": "ms", @@ -25092,19 +17442,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 279.89, - "fp16": 242.6, - "fp32": "", - "bf16": "" + "int8": 434.12, + "fp16": "", + "fp32": 30.6, + "bf16": 209.11, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25114,7 +17468,7 @@ "Precisions": [ { "int4": "", - "int8": 4.17, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -25126,19 +17480,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "bert-large-uncased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 43.58, - "fp16": 40.08, - "fp32": "", - "bf16": "" + "int8": 24134.02, + "fp16": "", + "fp32": 3392.4, + "bf16": 12168.49, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25148,10 +17506,10 @@ "Precisions": [ { "int4": "", - "int8": 20.29, + "int8": 0.74, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 0.89 } ], "Unit": "ms", @@ -25160,19 +17518,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 184.73, - "fp16": 167.81, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25182,8 +17544,8 @@ "Precisions": [ { "int4": "", - "int8": 7.43, - "fp16": "", + "int8": 4.62, + "fp16": 4.55, "fp32": "", "bf16": "" } @@ -25194,19 +17556,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "yolo11", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 8.7, - "fp16": 5.04, - "fp32": "", - "bf16": "" + "int8": "", + "fp16": "", + "fp32": 1034.68, + "bf16": 2068.81, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25216,7 +17582,7 @@ "Precisions": [ { "int4": "", - "int8": 106.68, + "int8": "", "fp16": "", "fp32": "", "bf16": "" @@ -25228,19 +17594,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8480+ CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1530.29, - "fp16": 1450.39, - "fp32": "", - "bf16": "" + "int8": 2380.51, + "fp16": "", + "fp32": 950.6, + "bf16": 2374.89, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25250,10 +17620,10 @@ "Precisions": [ { "int4": "", - "int8": 1.42, + "int8": 3.13, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 2.54 } ], "Unit": "ms", @@ -25262,19 +17632,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1023.16, - "fp16": 466.9, - "fp32": "", - "bf16": "" + "int8": 4671.04, + "fp16": "", + "fp32": 560.3, + "bf16": 3211.93, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25284,10 +17658,10 @@ "Precisions": [ { "int4": "", - "int8": 1.4, + "int8": 3.66, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 4.77 } ], "Unit": "ms", @@ -25296,19 +17670,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "efficientdet-d0", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1114.16, - "fp16": 946.07, - "fp32": "", - "bf16": "" + "int8": 1725.13, + "fp16": "", + "fp32": 1123.04, + "bf16": 1407.69, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25318,10 +17696,10 @@ "Precisions": [ { "int4": "", - "int8": 2.05, + "int8": 4.71, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 4.84 } ], "Unit": "ms", @@ -25330,19 +17708,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 45.06, - "fp16": 25.61, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 25.46, + "token_int8": 18.96, + "token_fp16": 12.14 } ], "Unit": "FPS", @@ -25351,9 +17733,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 21.18, - "fp16": "", + "int4": 39.27, + "int8": 52.74, + "fp16": 82.36, "fp32": "", "bf16": "" } @@ -25364,19 +17746,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 68.61, - "fp16": 48.46, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 27.1, + "token_int8": 19.33, + "token_fp16": 12.69 } ], "Unit": "FPS", @@ -25384,10 +17770,10 @@ }, "latency": { "Precisions": [ - { - "int4": "", - "int8": 12.01, - "fp16": "", + { + "int4": 36.9, + "int8": 51.72, + "fp16": 78.77, "fp32": "", "bf16": "" } @@ -25398,19 +17784,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 112.66, - "fp16": 119.27, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 31.71, + "token_int8": 23.05, + "token_fp16": 16.64 } ], "Unit": "FPS", @@ -25419,9 +17809,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 7.63, - "fp16": "", + "int4": 31.53, + "int8": 43.37, + "fp16": 60.07, "fp32": "", "bf16": "" } @@ -25432,19 +17822,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 326.29, - "fp16": 320.34, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 30.06, + "token_int8": 21.73, + "token_fp16": 14.93 } ], "Unit": "FPS", @@ -25453,9 +17847,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": 4.17, - "fp16": "", + "int4": 33.26, + "int8": 46.01, + "fp16": 66.97, "fp32": "", "bf16": "" } @@ -25466,19 +17860,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 211.04, + "int8": "", "fp16": "", - "fp32": 165.57, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 54.73, + "token_int8": 42.58, + "token_fp16": 31.51 } ], "Unit": "FPS", @@ -25487,9 +17885,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 18.27, + "int8": 23.48, + "fp16": 31.73, "fp32": "", "bf16": "" } @@ -25500,19 +17898,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "bert-large-uncased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 34.98, + "int8": 74.86, "fp16": "", - "fp32": 28.58, - "bf16": "" + "fp32": 6.39, + "bf16": 48.32, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25522,10 +17924,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 55.7, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 73.74 } ], "Unit": "ms", @@ -25534,19 +17936,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 166.87, + "int8": "", "fp16": "", - "fp32": 144.55, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 33.27, + "token_int8": 22.24, + "token_fp16": 15.74 } ], "Unit": "FPS", @@ -25555,9 +17961,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 30.05, + "int8": 44.96, + "fp16": 63.51, "fp32": "", "bf16": "" } @@ -25568,19 +17974,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "mobilenet-v2", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 6.73, + "int8": 39894.55, "fp16": "", - "fp32": 3.15, - "bf16": "" + "fp32": 15839.75, + "bf16": 29419.55, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25590,10 +18000,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.84, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 0.72 } ], "Unit": "ms", @@ -25602,19 +18012,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 1609.77, + "int8": "", "fp16": "", - "fp32": 1160.68, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": 40.45, + "token_fp16": 26.95 } ], "Unit": "FPS", @@ -25623,8 +18037,8 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 21.43, + "int8": 24.72, "fp16": "", "fp32": "", "bf16": "" @@ -25636,19 +18050,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 738.31, + "int8": "", "fp16": "", - "fp32": 365.53, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": 35.48, + "token_int8": 25.7, + "token_fp16": 16.1 } ], "Unit": "FPS", @@ -25657,9 +18075,9 @@ "latency": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 28.18, + "int8": 38.91, + "fp16": 62.09, "fp32": "", "bf16": "" } @@ -25670,19 +18088,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "resnet-50", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 911.79, + "int8": 21612.82, "fp16": "", - "fp32": 627.75, - "bf16": "" + "fp32": 2002.36, + "bf16": 13669.05, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25692,10 +18114,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 1.0, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 1.37 } ], "Unit": "ms", @@ -25704,19 +18126,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 31.73, + "int8": 513.09, "fp16": "", - "fp32": 16.1, - "bf16": "" + "fp32": 35.2, + "bf16": 275.94, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25738,19 +18164,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 51.44, + "int8": 26748.89, "fp16": "", - "fp32": 30.42, - "bf16": "" + "fp32": 4718.18, + "bf16": 16684.87, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25760,10 +18190,10 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 0.72, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 1.15 } ], "Unit": "ms", @@ -25772,19 +18202,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "stable-diffusion-v1-5", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 102.09, + "int8": "", "fp16": "", - "fp32": 85.74, - "bf16": "" + "fp32": "", + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25794,8 +18228,8 @@ "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 4.09, + "fp16": 3.99, "fp32": "", "bf16": "" } @@ -25806,19 +18240,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V CPU+iGPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "yolo11", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 315.12, + "int8": "", "fp16": "", - "fp32": 248.34, - "bf16": "" + "fp32": 1455.5, + "bf16": 2962.49, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25831,7 +18269,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": 3.19 } ], "Unit": "ms", @@ -25840,19 +18278,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "bert-base-cased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® Platinum 8580 CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 179.86, - "fp16": 141.28, - "fp32": "", - "bf16": "" + "int8": 3043.23, + "fp16": "", + "fp32": 1258.2, + "bf16": 3444.22, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25862,10 +18304,10 @@ "Precisions": [ { "int4": "", - "int8": 6.54, + "int8": 3.08, "fp16": "", "fp32": "", - "bf16": "" + "bf16": 2.56 } ], "Unit": "ms", @@ -25874,19 +18316,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "bert-large-uncased", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "bert-base-cased", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 31.04, + "int8": 8897.30, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 1217.03, + "bf16": 6414.49, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25896,9 +18342,9 @@ "Precisions": [ { "int4": "", - "int8": 42.23, + "int8": 7.74, "fp16": "", - "fp32": "", + "fp32": 14.8, "bf16": "" } ], @@ -25908,19 +18354,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", "Model": "efficientdet-d0", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 12.96, - "fp16": 13.4, - "fp32": "", - "bf16": "" + "int8": 3384.23, + "fp16": "", + "fp32": 2295.4, + "bf16": 2872.84, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25930,9 +18380,9 @@ "Precisions": [ { "int4": "", - "int8": 125.1, + "int8": 9.71, "fp16": "", - "fp32": "", + "fp32": 9.43, "bf16": "" } ], @@ -25942,19 +18392,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", "Model": "mask_rcnn_resnet50_atrous_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 149.52, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 11.97, + "bf16": 91.85, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25964,9 +18418,9 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 74.6, "fp16": "", - "fp32": "", + "fp32": 248.21, "bf16": "" } ], @@ -25976,19 +18430,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", "Model": "mobilenet-v2", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 4151.21, - "fp16": 3149.73, - "fp32": "", - "bf16": "" + "int8": 32737.09, + "fp16": "", + "fp32": 25621.92, + "bf16": 26297.21, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -25998,9 +18456,9 @@ "Precisions": [ { "int4": "", - "int8": 0.42, + "int8": 1.65, "fp16": "", - "fp32": "", + "fp32": 1.34, "bf16": "" } ], @@ -26010,19 +18468,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", "Model": "resnet-50", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 2038.58, - "fp16": 835.87, - "fp32": "", - "bf16": "" + "int8": 27670.82, + "fp16": "", + "fp32": 4254.94, + "bf16": 22432.74, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26032,9 +18494,9 @@ "Precisions": [ { "int4": "", - "int8": 0.77, + "int8": 2.28, "fp16": "", - "fp32": "", + "fp32": 3.69, "bf16": "" } ], @@ -26044,19 +18506,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "ssd_mobilenet_v1_coco", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "ssd-resnet34-1200", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 229.76, - "fp16": 200.09, - "fp32": "", - "bf16": "" + "int8": 1009.62, + "fp16": "", + "fp32": 77.99, + "bf16": 532.90, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26066,9 +18532,9 @@ "Precisions": [ { "int4": "", - "int8": 8.16, + "int8": 9.73, "fp16": "", - "fp32": "", + "fp32": 34.1, "bf16": "" } ], @@ -26078,19 +18544,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "ssd-resnet34-1200", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", + "int8": 29674.40, "fp16": "", - "fp32": "", - "bf16": "" + "fp32": 9800.83, + "bf16": 19479.18, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26100,9 +18570,9 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 2.08, "fp16": "", - "fp32": "", + "fp32": 2.45, "bf16": "" } ], @@ -26112,19 +18582,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "unet-camvid-onnx-0001", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "yolo_v8n", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": 37.1, - "fp32": "", - "bf16": "" + "int8": 5590.87, + "fp16": "", + "fp32": 2699.0, + "bf16": 6003.66, + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "FPS", @@ -26134,9 +18608,9 @@ "Precisions": [ { "int4": "", - "int8": "", + "int8": 6.56, "fp16": "", - "fp32": "", + "fp32": 5.59, "bf16": "" } ], @@ -26146,30 +18620,34 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "yolo_v5m", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "gemma-2-9b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 129.88, - "fp16": 105.29, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 136.4, + "token_int8": "", + "token_fp16": 53.6 } ], - "Unit": "FPS", + "Unit": "Tokens/sec", "UnitDesc": "higher is better" }, "latency": { "Precisions": [ { - "int4": "", - "int8": 9.44, - "fp16": "", + "int4": 7.3, + "int8": "", + "fp16": 18.7, "fp32": "", "bf16": "" } @@ -26180,30 +18658,34 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V NPU", - "Model": "yolo_v8n", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "glm-4-9b-chat", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": 399.28, - "fp16": 469.58, + "int8": "", + "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 116.5, + "token_int8": "", + "token_fp16": 51.9 } ], - "Unit": "FPS", + "Unit": "Tokens/sec", "UnitDesc": "higher is better" }, "latency": { "Precisions": [ { - "int4": "", - "int8": 3.94, - "fp16": "", + "int4": 8.6, + "int8": "", + "fp16": 19, "fp32": "", "bf16": "" } @@ -26214,19 +18696,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "chatglm2-6b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "llama-2-7b-chat", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 20.6, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 139.5, + "token_int8": "", + "token_fp16": 132 } ], "Unit": "Tokens/sec", @@ -26235,9 +18721,9 @@ "latency": { "Precisions": [ { - "int4": 48, + "int4": 7.2, "int8": "", - "fp16": "", + "fp16": 7.6, "fp32": "", "bf16": "" } @@ -26248,19 +18734,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "falcon-7b-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "llama-3.2-3b-instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 18.8, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 272.7, + "token_int8": 65, + "token_fp16": "" } ], "Unit": "Tokens/sec", @@ -26269,10 +18759,10 @@ "latency": { "Precisions": [ { - "int4": 53, + "int4": 3.7, "int8": "", "fp16": "", - "fp32": "", + "fp32": 15.4, "bf16": "" } ], @@ -26282,19 +18772,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "llama-2-7b-chat-hf", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "llama-3-8b", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 17.5, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 148.2, + "token_int8": "", + "token_fp16": 57.2 } ], "Unit": "Tokens/sec", @@ -26303,9 +18797,9 @@ "latency": { "Precisions": [ { - "int4": 57, + "int4": 6.7, "int8": "", - "fp16": "", + "fp16": 17.5, "fp32": "", "bf16": "" } @@ -26316,19 +18810,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "llama-3-8b", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "mistral-7b-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 19.2, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 126.4, + "token_int8": "", + "token_fp16": 61.4 } ], "Unit": "Tokens/sec", @@ -26337,9 +18835,9 @@ "latency": { "Precisions": [ { - "int4": 52, + "int4": 7.9, "int8": "", - "fp16": "", + "fp16": 16.3, "fp32": "", "bf16": "" } @@ -26350,19 +18848,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "mistral-7b-v0.1", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "phi-3-mini-4k-instruct", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 18.2, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 176.6, + "token_int8": "", + "token_fp16": 111.9 } ], "Unit": "Tokens/sec", @@ -26371,9 +18873,9 @@ "latency": { "Precisions": [ { - "int4": 54, + "int4": 5.7, "int8": "", - "fp16": "", + "fp16": 8.9, "fp32": "", "bf16": "" } @@ -26384,19 +18886,23 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", - "Model": "phi-3-mini-4k-instruct", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", + "Model": "qwen2-7b", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ { - "int4": 29.5, + "int4": "", "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": 164.4, + "token_int8": "", + "token_fp16": 62.2 } ], "Unit": "Tokens/sec", @@ -26405,9 +18911,9 @@ "latency": { "Precisions": [ { - "int4": 34, + "int4": 6.1, "int8": "", - "fp16": "", + "fp16": 16.1, "fp32": "", "bf16": "" } @@ -26418,10 +18924,11 @@ } }, { - "Platform": "IntelĀ® Coreā„¢ Ultra 7 processor 268V iGPU", + "Platform": "IntelĀ® XeonĀ® 6979P CPU-only", "Model": "stable-diffusion-v1-5", - "Checked": "true", - "PlatformType": "Client Platforms (IntelĀ® Coreā„¢)", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (IntelĀ® XeonĀ®)", "Parameters": { "throughput": { "Precisions": [ @@ -26430,7 +18937,10 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "" + "bf16": "", + "token_int4": "", + "token_int8": "", + "token_fp16": "" } ], "Unit": "n/a", @@ -26440,8 +18950,8 @@ "Precisions": [ { "int4": "", - "int8": 7.2, - "fp16": 6.9, + "int8": 4.0, + "fp16": 4.1, "fp32": "", "bf16": "" } diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json new file mode 100644 index 00000000000000..f96fb11e6b029d --- /dev/null +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json @@ -0,0 +1,96 @@ +[ + { + "Platform": "IntelĀ® XeonĀ® Platinum 8580", + "Model": "mistralai/Mistral-7B-v0.1", + "PlatformType": "None", + "Parameters": { + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": "350.06", + "0.6": "486.89", + "0.8": "575.92", + "2.0": "778.07" + } + }, + { + "Latency": { + "0.2": "60.93", + "0.6": "91.63", + "0.8": "113.61", + "2.0": "240.25" + } + } + ] + }, + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": "90.98", + "0.6": "266.24", + "0.8": "351.63", + "2.0": "195.16" + } + }, + { + "Latency": { + "0.2": "54.9", + "0.6": "78.78", + "0.8": "95.78", + "2.0": "352.23" + } + } + ] + } + } + }, + { + "Platform": "IntelĀ® XeonĀ® Platinum 8530", + "Model": "mistralai/Mistral-7B-v0.1", + "PlatformType": "None", + "Parameters": { + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": "350.06", + "0.6": "486.89", + "0.8": "575.92", + "2.0": "778.07" + } + }, + { + "Latency": { + "0.2": "60.93", + "0.6": "91.63", + "0.8": "113.61", + "2.0": "240.25" + } + } + ] + }, + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": "90.98", + "0.6": "266.24", + "0.8": "351.63", + "2.0": "195.16" + } + }, + { + "Latency": { + "0.2": "54.9", + "0.6": "78.78", + "0.8": "95.78", + "2.0": "352.23" + } + } + ] + } + } + } +] \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json index 6fb8d19e1a1adf..e5fe953b72bca1 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json +++ b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json @@ -1,4 +1,3 @@ - { "PrecisionsMap": { "INT4": "int4", @@ -8,8 +7,13 @@ "BF16": "bf16", "FP32_OV": "fp32_ov", "FP32_OVMS": "fp32_ovms", + "TOKEN_INT4": "token_int4", + "TOKEN_INT8": "token_int8", + "TOKEN_FP16": "token_fp16", "INT8_OV": "int8_ov", - "INT8_OVMS": "int8_ovms" + "INT8_OVMS": "int8_ovms", + "THROUGHPUT": "Throughput", + "LATENCY": "Latency" }, "ParametersMap": { "Throughput": "throughput", @@ -71,6 +75,45 @@ "data": null, "color": "#00536a", "label": "INT8 OVMS" + }, + "token_int4": { + "data": null, + "color": "#00536a", + "label": "T INT4" + }, + "token_int8": { + "data": null, + "color": "#00636a", + "label": "T INT8" + }, + "token_fp16": { + "data": null, + "color": "#00736a", + "label": "T INT16" + }, + "Vllm": { + "Latency": { + "data": null, + "color": "#FDB2BC", + "label": "Latency" + }, + "Throughput": { + "data": null, + "color": "#91CDF7", + "label": "Throughput" + } + }, + "Ovms": { + "Latency": { + "data": null, + "color": "#CDB2BC", + "label": "Latency" + }, + "Throughput": { + "data": null, + "color": "#12CDF7", + "label": "Throughput" + } } }, "Filters": [ @@ -80,13 +123,16 @@ "platformTypes": { "name": "ietype", "data": [ - "Client Platforms (IntelĀ® Coreā„¢)" + "None", + "IntelĀ® Coreā„¢, CPU-only", + "IntelĀ® Coreā„¢, iGPU-only", + "IntelĀ® Coreā„¢, NPU-only", + "IntelĀ® Coreā„¢, CPU+iGPU" ] }, "platforms": { "name": "platform", - "data": [ - ] + "data": [] }, "platformFilters": { "name": "coretype", @@ -97,21 +143,24 @@ "models": { "name": "networkmodel", "data": [ - "bert-base-cased" + "bert-base-cased", + "resnet-50", + "yolo_v8n", + "llama-3-8B" ] }, "parameters": { "name": "parameters", "data": [ - "Throughput", - "Latency" + "Vllm", + "Ovms" ] }, "precision": { "name": "precision", "data": [ - "INT4", - "INT8" + "THROUGHPUT", + "LATENCY" ] } } diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-155H.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-155H.csv new file mode 100644 index 00000000000000..0d16c5c4998329 --- /dev/null +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-155H.csv @@ -0,0 +1,156 @@ +Topology,Precision,Input Size,max rss memory,1st latency (ms),2nd latency (ms),2nd tok/sec +opt-125m-gptq,INT4-MIXED,32,965.9,29,7.7,129.87 +opt-125m-gptq,INT4-MIXED,1024,1507.9,113.1,7.8,128.21 +tiny-llama-1.1b-chat,INT4-MIXED,32,1831.8,46.5,16.7,59.88 +tiny-llama-1.1b-chat,INT4-MIXED,1024,1806.3,635,17.8,56.18 +qwen2-0.5b,INT4-MIXED,32,2551.7,61.4,18.3,54.64 +qwen2-0.5b,INT4-MIXED,1024,2976.6,356.1,19.2,52.08 +tiny-llama-1.1b-chat,INT8-CW,32,1987.4,56,21.6,46.30 +tiny-llama-1.1b-chat,INT8-CW,1024,2209.1,772.7,22.6,44.25 +qwen2-0.5b,INT8-CW,32,2484.9,57.3,22.8,43.86 +qwen2-0.5b,INT8-CW,1024,3102.5,407.1,23.9,41.84 +qwen2-1.5b,INT4-MIXED,32,4265.2,71.7,25.5,39.22 +qwen2-1.5b,INT4-MIXED,1024,4884.5,862.4,26.8,37.31 +dolly-v2-3b,INT4-MIXED,32,2401.3,89.6,27.5,36.36 +red-pajama-incite-chat-3b-v1,INT4-MIXED,32,2511.5,78.6,28.2,35.46 +phi-2,INT4-MIXED,32,2279.5,95.7,29.1,34.36 +minicpm-1b-sft,INT4-MIXED,31,2759.9,104.4,30.9,32.36 +phi-2,INT4-MIXED,32,2620.1,100.8,31,32.26 +stable-zephyr-3b-dpo,INT4-MIXED,30,2636.5,86.8,31.7,31.55 +dolly-v2-3b,INT4-MIXED,1024,3137.1,1782.9,32.2,31.06 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1020,3118.5,1831.7,33.3,30.03 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1024,2862.7,1821.1,33.5,29.85 +qwen2-1.5b,INT8-CW,32,4831.2,87,33.8,29.59 +opt-2.7b,INT4-MIXED,31,2898.3,73.2,33.9,29.50 +phi-2,INT4-MIXED,1024,2797.4,1887,34,29.41 +orca-mini-3b,INT4-MIXED,32,2877.8,100.3,35,28.57 +stablelm-3b-4e1t,INT4-MIXED,32,2669.4,94.7,35.3,28.33 +qwen2-1.5b,INT8-CW,1024,5455.8,1047.6,35.3,28.33 +minicpm-1b-sft,INT8-CW,31,3104.1,103.5,35.3,28.33 +phi-2,INT4-MIXED,1024,3039.8,1917.4,35.9,27.86 +stable-zephyr-3b-dpo,INT4-MIXED,946,3411.4,1695,37,27.03 +gemma-2b-it,INT4-MIXED,32,3991.7,116.1,37.9,26.39 +opt-2.7b,INT4-MIXED,937,3617.5,1764.9,38.2,26.18 +phi-3-mini-4k-instruct,INT4-MIXED,31,2935.3,111.6,38.2,26.18 +phi-3-mini-4k-instruct,INT4-MIXED,38,3102.4,134,38.4,26.04 +phi-3-mini-4k-instruct,INT4-MIXED,31,2986.1,114.1,38.9,25.71 +phi-3-mini-4k-instruct,INT4-MIXED,38,2977.4,131.1,39,25.64 +gemma-2b-it,INT4-MIXED,1024,4973.3,1249.2,39.7,25.19 +stablelm-3b-4e1t,INT4-MIXED,1024,3196.9,2045.4,39.9,25.06 +dolly-v2-3b,INT8-CW,32,3490.2,107.4,41.5,24.10 +red-pajama-incite-chat-3b-v1,INT8-CW,32,3457.9,105,42.5,23.53 +opt-2.7b,INT8-CW,31,3686.8,107.5,44.1,22.68 +phi-2,INT8-CW,32,3554.9,116.6,44.1,22.68 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3390.7,2277.1,44.2,22.62 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3643.6,2485,44.4,22.52 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3516.4,2280.9,44.5,22.47 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3537.2,2522.4,44.7,22.37 +orca-mini-3b,INT4-MIXED,1024,3557.3,1898.9,45,22.22 +minicpm-1b-sft,FP16,31,3814.4,97.9,45.4,22.03 +stablelm-3b-4e1t,INT8-CW,32,3486.9,100.5,46.1,21.69 +stable-zephyr-3b-dpo,INT8-CW,30,3516.7,101.9,46.1,21.69 +dolly-v2-3b,INT8-CW,1024,4265.9,2178.6,46.2,21.65 +red-pajama-incite-chat-3b-v1,INT8-CW,1020,3979.1,2219.7,47.2,21.19 +red-pajama-incite-chat-3b-v1,INT8-CW,1024,3975.5,2199.7,47.3,21.14 +opt-2.7b,INT8-CW,937,4358.6,1981.8,48.4,20.66 +phi-2,INT8-CW,1024,4058.1,2280.1,48.9,20.45 +gemma-2b-it,INT8-CW,32,4786.8,119.8,49.4,20.24 +chatglm3-6b,INT4-MIXED,32,4141.5,166.6,49.7,20.12 +stablelm-3b-4e1t,INT8-CW,1024,4054.8,2243.5,50.7,19.72 +stable-zephyr-3b-dpo,INT8-CW,946,4521.8,1816.4,51.3,19.49 +gemma-2b-it,INT8-CW,1024,5810.7,1580,51.3,19.49 +chatglm3-6b,INT4-MIXED,32,4651.4,164.7,51.6,19.38 +chatglm3-6b,INT4-MIXED,1024,4235.1,2818.7,52.3,19.12 +orca-mini-3b,INT8-CW,32,4162,109.2,53.3,18.76 +chatglm3-6b,INT4-MIXED,1024,4783.8,2869,54.4,18.38 +gpt-j-6b,INT4-MIXED,32,4667.3,176.7,56.3,17.76 +chatglm3-6b-gptq,INT4-MIXED,32,5369.4,173.9,58.9,16.98 +llama-2-7b-chat-hf,INT4-MIXED,32,4280,173.2,60.1,16.64 +phi-3-mini-4k-instruct,INT8-CW,31,4585.1,123,60.5,16.53 +phi-3-mini-4k-instruct,INT8-CW,38,4597,152,60.5,16.53 +chatglm2-6b,INT4-MIXED,32,4847.8,158.7,60.6,16.50 +vicuna-7b-v1.5,INT4-MIXED,32,4476.9,178.2,61.2,16.34 +chatglm3-6b-gptq,INT4-MIXED,1024,5217.6,2863.7,61.3,16.31 +mistral-7b-v0.1,INT4-MIXED,31,4413.6,194,61.7,16.21 +qwen2-7b,INT4-MIXED,32,7044.7,184.4,61.7,16.21 +mistral-7b-v0.1,INT4-MIXED,32,4427.6,193.3,61.8,16.18 +orca-mini-3b,INT8-CW,1024,4821.6,2239.1,62,16.13 +codegen25-7b,INT4-MIXED,32,4687.2,176.2,62.7,15.95 +chatglm2-6b,INT4-MIXED,1024,5165.9,3148,63,15.87 +llama-2-7b-gptq,INT4-MIXED,32,4632.8,175.2,63.4,15.77 +stablelm-7b,INT4-MIXED,32,5219.5,206.3,63.4,15.77 +qwen-7b-chat,INT4-MIXED,32,7805.6,193.8,63.6,15.72 +gpt-j-6b,INT4-MIXED,1024,5314.9,3111.8,63.6,15.72 +qwen2-7b,INT4-MIXED,1024,7716.2,3548.3,64.1,15.60 +llama-3-8b,INT4-MIXED,32,4910.9,204.8,64.7,15.46 +mistral-7b-v0.1,INT4-MIXED,1024,4720.8,3667.1,64.8,15.43 +mistral-7b-v0.1,INT4-MIXED,1007,4704.7,3685.4,64.9,15.41 +llama-3.1-8b,INT4-MIXED,31,4850.3,211.5,64.9,15.41 +phi-3-mini-4k-instruct,INT8-CW,1023,5128.6,2815.2,65.7,15.22 +phi-3-mini-4k-instruct,INT8-CW,1061,5155,3407.9,65.9,15.17 +mistral-7b-v0.1,INT4-MIXED,32,4939.3,192,66.5,15.04 +llama-3-8b,INT4-MIXED,33,4919.4,261.9,67.2,14.88 +llama-2-7b-chat-hf,INT4-MIXED,1024,4948.2,3811,67.3,14.86 +qwen1.5-7b-chat,INT4-MIXED,32,5943.1,180.5,67.7,14.77 +qwen-7b-chat-gptq,INT4-MIXED,32,8057,187,68.1,14.68 +llama-3-8b,INT4-MIXED,32,5503.5,198.4,68.1,14.68 +qwen-7b-chat,INT4-MIXED,32,8091.6,185.9,68.1,14.68 +llama-3-8b,INT4-MIXED,1024,5569.1,3920.5,68.2,14.66 +llama-3.1-8b,INT4-MIXED,31,5358.6,201,68.2,14.66 +stablelm-7b,INT4-MIXED,1020,5804.4,3726.6,68.8,14.53 +llama-3.1-8b,INT4-MIXED,31,5452.6,202.9,68.8,14.53 +llama-2-7b-chat-hf,INT4-MIXED,32,5023,165.7,69,14.49 +llama-3-8b,INT4-MIXED,32,5413.6,202,69.1,14.47 +llama-3-8b,INT4-MIXED,33,5440.4,262.1,69.2,14.45 +codegen25-7b,INT4-MIXED,1024,5434.6,3513.2,69.9,14.31 +mistral-7b-v0.1,INT4-MIXED,1024,5614.9,3819.1,70,14.29 +mistral-7b-v0.1,INT4-MIXED,31,4927.8,205,70.5,14.18 +llama-3-8b,INT4-MIXED,33,5498.9,270.7,70.6,14.16 +llama-3-8b,INT4-MIXED,1025,5577.4,4271.2,70.6,14.16 +llama-2-7b-gptq,INT4-MIXED,1024,5302.2,3529.4,70.7,14.14 +zephyr-7b-beta,INT4-MIXED,32,5212.4,190.6,71.2,14.04 +llama-3-8b,INT4-MIXED,1024,6161.1,3918,71.5,13.99 +llama-3-8b,INT4-MIXED,1025,6098,4441.8,72.3,13.83 +llama-3-8b,INT4-MIXED,1024,6071.7,3972.2,72.4,13.81 +mistral-7b-v0.1,INT4-MIXED,1007,5224.1,4153.4,73.8,13.55 +llama-3-8b,INT4-MIXED,1025,6156.9,4357,73.9,13.53 +zephyr-7b-beta,INT4-MIXED,1024,5511.6,3978,74.4,13.44 +opt-2.7b,FP16,31,9220.3,107.8,74.7,13.39 +dolly-v2-3b,FP16,32,6058.9,109.9,74.7,13.39 +qwen1.5-7b-chat,INT4-MIXED,1024,7063.2,3791.7,75,13.33 +qwen-7b-chat,INT4-MIXED,1024,8919.5,3763.9,75,13.33 +red-pajama-incite-chat-3b-v1,FP16,32,6036.5,107.5,75.9,13.18 +llama-2-7b-chat-hf,INT4-MIXED,1024,5716.8,4231.7,76.2,13.12 +phi-2,FP16,32,6090.1,115.2,77.1,12.97 +stable-zephyr-3b-dpo,FP16,30,6113.1,112.1,78.6,12.72 +qwen-7b-chat,INT4-MIXED,1024,9212.9,3857.4,78.6,12.72 +stablelm-3b-4e1t,FP16,32,6065.4,110.2,78.7,12.71 +opt-2.7b,FP16,937,9733.8,3750.8,78.8,12.69 +dolly-v2-3b,FP16,1024,6615.2,2230.9,79.1,12.64 +red-pajama-incite-chat-3b-v1,FP16,1020,6588.3,2259.4,80.2,12.47 +glm-4-9b,INT4-MIXED,33,6386.2,328,80.4,12.44 +red-pajama-incite-chat-3b-v1,FP16,1024,6570.3,2268.7,80.4,12.44 +baichuan2-7b-chat,INT4-MIXED,32,5977.9,201.7,81,12.35 +glm-4-9b,INT4-MIXED,32,6389.7,248.1,81,12.35 +phi-2,FP16,1024,6646.2,2406.7,81.4,12.29 +stable-zephyr-3b-dpo,FP16,946,6875.7,1868.2,82.9,12.06 +stablelm-3b-4e1t,FP16,1024,6636.1,2036.9,83,12.05 +chatglm2-6b,INT8-CW,32,6731.8,159.2,84.4,11.85 +glm-4-9b,INT4-MIXED,1025,7061.4,4939.2,85.2,11.74 +qwen-7b-chat-gptq,INT4-MIXED,1024,9175.3,3898,85.3,11.72 +gemma-7b-it,INT4-MIXED,32,7883.9,230.5,86,11.63 +gemma-7b-it,INT4-MIXED,32,8002.6,235,86.1,11.61 +glm-4-9b,INT4-MIXED,1024,7064.9,4411.2,86.2,11.60 +gpt-j-6b,INT8-CW,32,7009.2,176.8,86.4,11.57 +chatglm2-6b,INT8-CW,1024,7050.5,3871.6,86.8,11.52 +chatglm3-6b,INT8-CW,32,6755.9,159,86.8,11.52 +baichuan2-7b-chat,INT4-MIXED,1024,7033.3,4049,88.8,11.26 +chatglm3-6b,INT8-CW,1024,7076.5,3865.9,89.2,11.21 +qwen-7b-chat,INT4-MIXED,32,9245.7,176.3,90,11.11 +gemma-7b-it,INT4-MIXED,1024,9449.4,4305.8,93.2,10.73 +gpt-j-6b,INT8-CW,1024,7672.3,4181.1,93.5,10.70 +gemma-7b-it,INT4-MIXED,1024,9330.5,4222.5,93.7,10.67 +orca-mini-3b,FP16,32,7416.5,122.3,94.7,10.56 +codegen25-7b,INT8-CW,32,7557.6,170.7,98.4,10.16 +qwen-7b-chat,INT4-MIXED,1024,10371.1,4271.7,98.9,10.11 +llama-2-7b-chat-hf,INT8-CW,32,7390.6,171.6,99.9,10.01 diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv new file mode 100644 index 00000000000000..09799a2de31fe6 --- /dev/null +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv @@ -0,0 +1,182 @@ +Topology,Precision,Input Size,max rss memory,1st latency (ms),2nd latency (ms),2nd tok/sec +opt-125m-gptq,INT4-MIXED,1024,1513.6,81.9,7.8,128.21 +opt-125m-gptq,INT4-MIXED,32,979.9,50.4,7.9,126.58 +tiny-llama-1.1b-chat,INT4-MIXED,1024,1943.3,176.3,16.8,59.52 +tiny-llama-1.1b-chat,INT4-MIXED,32,1982.2,59.5,17.1,58.48 +qwen2-0.5b,INT4-MIXED,32,2678,117.3,18.7,53.48 +tiny-llama-1.1b-chat,INT8-CW,32,2080.9,59.4,19,52.63 +qwen2-0.5b,INT4-MIXED,1024,3036.1,165.5,19.2,52.08 +tiny-llama-1.1b-chat,INT8-CW,1024,2287,241.4,19.6,51.02 +qwen2-0.5b,INT8-CW,1024,3084.9,172.1,20,50.00 +qwen2-0.5b,INT8-CW,32,2518,105.5,21.4,46.73 +red-pajama-incite-chat-3b-v1,INT4-MIXED,32,2793.6,141.8,23.9,41.84 +qwen2-1.5b,INT4-MIXED,32,4515.4,118.7,24,41.67 +qwen2-1.5b,INT4-MIXED,1024,4930.1,229.6,24.3,41.15 +dolly-v2-3b,INT4-MIXED,32,2486.1,174,25.4,39.37 +phi-2,INT4-MIXED,32,2552.9,210.6,26.9,37.17 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1020,2934.1,464.5,27.5,36.36 +qwen2-1.5b,INT8-CW,32,4813.4,119.1,27.8,35.97 +opt-2.7b,INT4-MIXED,31,3172.5,131.9,28.5,35.09 +red-pajama-incite-chat-3b-v1,INT4-MIXED,1024,3038.2,447.1,28.6,34.97 +dolly-v2-3b,INT4-MIXED,1024,2947.4,409,28.8,34.72 +qwen2-1.5b,INT8-CW,1024,5394.8,327.9,29.3,34.13 +stable-zephyr-3b-dpo,INT4-MIXED,30,2728.1,131.2,29.8,33.56 +phi-2,INT4-MIXED,32,2805.1,208.3,30.2,33.11 +minicpm-1b-sft,INT8-CW,31,3104.2,147.8,30.9,32.36 +phi-2,INT4-MIXED,1024,3058.9,602.9,31.1,32.15 +minicpm-1b-sft,INT4-MIXED,31,2970.1,183.7,31.1,32.15 +stablelm-3b-4e1t,INT4-MIXED,32,3077.1,183.2,31.6,31.65 +opt-2.7b,INT4-MIXED,937,3416.7,429.4,31.6,31.65 +stable-zephyr-3b-dpo,INT4-MIXED,946,3211.8,428.8,32.3,30.96 +phi-3-mini-4k-instruct,INT4-MIXED,31,3014.5,116,32.5,30.77 +phi-3-mini-4k-instruct,INT4-MIXED,38,2957.4,153.9,32.5,30.77 +phi-2,INT4-MIXED,1024,3278.9,613.3,33.4,29.94 +phi-3-mini-4k-instruct,INT4-MIXED,38,3288.5,152.9,33.4,29.94 +phi-3-mini-4k-instruct,INT4-MIXED,31,3265.1,123.6,34.1,29.33 +gemma-2b-it,INT4-MIXED,32,4162.1,208.8,34.2,29.24 +stablelm-3b-4e1t,INT4-MIXED,1024,3525.8,524.5,35,28.57 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3427.8,777.5,36.5,27.40 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3405.4,554.1,36.7,27.25 +gemma-2b-it,INT4-MIXED,1024,5053.1,354.8,36.9,27.10 +minicpm-1b-sft,FP16,31,3595.5,124.9,36.9,27.10 +phi-3-mini-4k-instruct,INT4-MIXED,1061,3547.2,755.8,37.1,26.95 +phi-3-mini-4k-instruct,INT4-MIXED,1023,3528.4,536.4,37.4,26.74 +red-pajama-incite-chat-3b-v1,INT8-CW,32,3747.7,189.9,38.1,26.25 +opt-2.7b,INT8-CW,31,3810.7,145.7,38.5,25.97 +chatglm3-6b,INT4-MIXED,32,4120.7,67.3,38.7,25.84 +dolly-v2-3b,INT8-CW,32,3747,188.4,39.2,25.51 +chatglm3-6b,INT4-MIXED,32,4482.9,69.9,40.7,24.57 +chatglm3-6b,INT4-MIXED,1024,4146,606.8,41,24.39 +opt-2.7b,INT8-CW,937,4458.9,587.8,41.8,23.92 +red-pajama-incite-chat-3b-v1,INT8-CW,1024,4088.4,634.1,41.9,23.87 +red-pajama-incite-chat-3b-v1,INT8-CW,1020,4086.8,653.4,42,23.81 +phi-2,INT8-CW,32,3794.6,202.7,42.1,23.75 +chatglm3-6b,INT4-MIXED,1024,4446.7,598.6,42.3,23.64 +stablelm-3b-4e1t,INT8-CW,32,3652.5,146,42.6,23.47 +stable-zephyr-3b-dpo,INT8-CW,30,3768.6,151.9,42.6,23.47 +dolly-v2-3b,INT8-CW,1024,4092,603.1,42.9,23.31 +stablelm-3b-4e1t,INT8-CW,1024,4143.2,671.7,45.2,22.12 +gemma-2b-it,INT8-CW,32,4878.4,221.6,45.6,21.93 +phi-2,INT8-CW,1024,4153.6,810.3,46,21.74 +llama-2-7b-chat-hf,INT4-MIXED,32,4394.6,109.7,46.2,21.65 +chatglm3-6b-gptq,INT4-MIXED,32,5218.9,79.7,46.7,21.41 +stable-zephyr-3b-dpo,INT8-CW,946,4360.1,627.8,46.8,21.37 +vicuna-7b-v1.5,INT4-MIXED,32,4482.3,101.2,47.2,21.19 +gemma-2b-it,INT8-CW,1024,5837.1,507.1,48,20.83 +llama-2-7b-gptq,INT4-MIXED,32,4734.3,102.8,48.1,20.79 +orca-mini-3b,INT4-MIXED,32,2720.1,132,48.1,20.79 +qwen-7b-chat,INT4-MIXED,32,7803.7,178.5,48.3,20.70 +mistral-7b-v0.1,INT4-MIXED,31,4537.5,99,48.5,20.62 +codegen25-7b,INT4-MIXED,32,4723.3,108.5,48.5,20.62 +chatglm3-6b-gptq,INT4-MIXED,1024,5150.8,614.2,48.8,20.49 +mistral-7b-v0.1,INT4-MIXED,32,4572,102.9,48.8,20.49 +llama-3-8b,INT4-MIXED,33,4991.2,252.2,50.9,19.65 +qwen-7b-chat-gptq,INT4-MIXED,32,8088.4,212.6,51,19.61 +chatglm2-6b,INT4-MIXED,32,4960.6,105.5,51.2,19.53 +gpt-j-6b,INT4-MIXED,32,4699.5,259.2,51.4,19.46 +llama-3.1-8b,INT4-MIXED,31,4897.8,106.9,51.5,19.42 +llama-3-8b,INT4-MIXED,32,4999.7,105.9,51.6,19.38 +qwen-7b-chat,INT4-MIXED,32,8085.9,193.5,51.7,19.34 +falcon-7b-instruct,INT4-MIXED,32,5416.2,175,52.5,19.05 +mistral-7b-v0.1,INT4-MIXED,1007,4772.6,803,52.6,19.01 +qwen1.5-7b-chat,INT4-MIXED,32,6027.3,174.9,53,18.87 +mistral-7b-v0.1,INT4-MIXED,1024,4775,717.6,53,18.87 +llama-2-7b-chat-hf,INT4-MIXED,1024,4976.5,992.1,53.1,18.83 +qwen2-7b,INT4-MIXED,32,7087.1,138.1,53.3,18.76 +llama-2-7b-gptq,INT4-MIXED,1024,5351.2,711.6,53.7,18.62 +llama-3-8b,INT4-MIXED,32,5472.8,109.4,53.7,18.62 +phi-3-mini-4k-instruct,INT8-CW,38,4575.3,115.9,53.7,18.62 +stablelm-7b,INT4-MIXED,32,5213.7,128.5,53.8,18.59 +phi-3-mini-4k-instruct,INT8-CW,31,4571.8,118.9,53.8,18.59 +llama-3-8b,INT4-MIXED,33,5480.4,246.8,53.9,18.55 +llama-3-8b,INT4-MIXED,32,5528.2,144.9,54.3,18.42 +llama-3.1-8b,INT4-MIXED,31,5377.3,112.8,54.3,18.42 +chatglm2-6b,INT4-MIXED,1024,5232.3,759.6,54.6,18.32 +llama-3.1-8b,INT4-MIXED,31,5440.4,126.4,54.8,18.25 +llama-3-8b,INT4-MIXED,33,5532.8,248.2,54.9,18.21 +codegen25-7b,INT4-MIXED,1024,5412.9,714.8,55,18.18 +mistral-7b-v0.1,INT4-MIXED,32,4998.5,117.3,55.2,18.12 +mistral-7b-v0.1,INT4-MIXED,31,5000.2,122.4,55.6,17.99 +llama-3-8b,INT4-MIXED,1024,5594,953.5,56.6,17.67 +gpt-j-6b,INT4-MIXED,1024,5323.8,1254,56.8,17.61 +llama-3-8b,INT4-MIXED,1025,5596.7,1192.3,56.8,17.61 +qwen2-7b,INT4-MIXED,1024,7722.1,714.2,57,17.54 +phi-3-mini-4k-instruct,INT8-CW,1023,5067.1,818.5,57.4,17.42 +phi-3-mini-4k-instruct,INT8-CW,1061,5086.1,975.1,57.4,17.42 +llama-2-7b-chat-hf,INT4-MIXED,32,5087.7,126.2,57.9,17.27 +stablelm-7b,INT4-MIXED,1020,5780.5,1248.4,59,16.95 +llama-3-8b,INT4-MIXED,1025,6088.9,1381.5,59,16.95 +llama-3-8b,INT4-MIXED,1024,6084.8,931.2,59.2,16.89 +llama-3-8b,INT4-MIXED,1025,6141.2,1494.3,59.4,16.84 +llama-3-8b,INT4-MIXED,1024,6133.8,1075.2,59.6,16.78 +mistral-7b-v0.1,INT4-MIXED,1024,5472.6,794.3,59.7,16.75 +zephyr-7b-beta,INT4-MIXED,32,5328.5,103.5,59.8,16.72 +falcon-7b-instruct,INT4-MIXED,1024,5677.5,686.2,59.8,16.72 +mistral-7b-v0.1,INT4-MIXED,1007,5243.5,1074,59.9,16.69 +qwen1.5-7b-chat,INT4-MIXED,1024,7096.7,1132.7,60,16.67 +qwen-7b-chat,INT4-MIXED,1024,8872.6,792.8,61,16.39 +qwen-7b-chat,INT4-MIXED,1024,9164.4,822.6,63.3,15.80 +orca-mini-3b,INT8-CW,32,4221.7,170.6,63.5,15.75 +llama-2-7b-chat-hf,INT4-MIXED,1024,5708.1,1397.9,63.6,15.72 +glm-4-9b,INT4-MIXED,33,6402.9,307.1,63.8,15.67 +zephyr-7b-beta,INT4-MIXED,1024,5572.4,1156.4,64.3,15.55 +glm-4-9b,INT4-MIXED,32,6383.1,256.2,64.5,15.50 +baichuan2-7b-chat,INT4-MIXED,32,5926.3,191.8,65.8,15.20 +opt-2.7b,FP16,31,5886,112.2,68,14.71 +dolly-v2-3b,FP16,32,6161.5,147.5,69.5,14.39 +red-pajama-incite-chat-3b-v1,FP16,32,6265.4,146.2,69.6,14.37 +glm-4-9b,INT4-MIXED,1024,6994.5,1013.7,69.8,14.33 +opt-2.7b,FP16,937,6345,379.5,71.6,13.97 +glm-4-9b,INT4-MIXED,1025,7014.9,1416.8,72.5,13.79 +phi-2,FP16,32,6204.7,189.2,72.9,13.72 +stable-zephyr-3b-dpo,FP16,30,6221.4,159.7,73,13.70 +dolly-v2-3b,FP16,1024,6669.9,424.3,73.3,13.64 +red-pajama-incite-chat-3b-v1,FP16,1020,6658.8,484.7,73.4,13.62 +stablelm-3b-4e1t,FP16,32,6216.3,145.4,73.5,13.61 +qwen-7b-chat,INT4-MIXED,32,9294.9,144.4,73.8,13.55 +red-pajama-incite-chat-3b-v1,FP16,1024,6755.1,469.1,73.9,13.53 +qwen-7b-chat-gptq,INT4-MIXED,1024,9152.1,827.2,75.1,13.32 +gemma-7b-it,INT4-MIXED,32,7991.4,128.6,75.8,13.19 +chatglm2-6b,INT8-CW,32,6854.4,110.2,76.3,13.11 +chatglm3-6b,INT8-CW,32,6754.8,112.3,76.4,13.09 +stable-zephyr-3b-dpo,FP16,946,6940,428.6,76.7,13.04 +baichuan2-7b-chat,INT4-MIXED,1024,6930.2,1229.5,76.7,13.04 +gemma-7b-it,INT4-MIXED,32,8061.5,125.6,76.7,13.04 +stablelm-3b-4e1t,FP16,1024,6722.9,480.8,77,12.99 +phi-2,FP16,1024,6709.4,624.1,77.2,12.95 +chatglm2-6b,INT8-CW,1024,7132.9,1361.9,78.7,12.71 +chatglm3-6b,INT8-CW,1024,7037.5,1389.2,78.7,12.71 +qwen-7b-chat,INT4-MIXED,1024,10374.1,1357.5,81.1,12.33 +gemma-7b-it,INT4-MIXED,1024,9398,1268.5,82.7,12.09 +gemma-7b-it,INT4-MIXED,1024,9469.5,1268,83.2,12.02 +gpt-j-6b,INT8-CW,32,7126.5,255.2,87.2,11.47 +falcon-7b-instruct,INT8-CW,32,8287.6,131.1,88.4,11.31 +llama-2-7b-chat-hf,INT8-CW,32,7474.9,139.5,89.7,11.15 +codegen25-7b,INT8-CW,32,7559.4,138,90.8,11.01 +vicuna-7b-v1.5,INT8-CW,32,7390.8,136.6,90.8,11.01 +falcon-7b-instruct,INT8-CW,1024,8546.8,1205.9,92.2,10.85 +stablelm-7b,INT8-CW,32,8356.4,143,92.4,10.82 +qwen2-7b,INT8-CW,32,9940.7,132,92.5,10.81 +baichuan2-13b-chat,INT4-MIXED,32,9879.2,184.9,93.3,10.72 +phi-3-mini-4k-instruct,FP16,38,8290,125.2,93.4,10.71 +phi-3-mini-4k-instruct,FP16,31,8290.5,109.5,93.5,10.70 +gpt-j-6b,INT8-CW,1024,7759,1996.8,93.9,10.65 +llama-2-7b-chat-hf,INT8-CW,1024,8097.8,1701.6,94.7,10.56 +phi-3-medium-4k-instruct,INT4-MIXED,38,8210.4,527,95.1,10.52 +mistral-7b-v0.1,INT8-CW,31,7882.4,128.6,95.1,10.52 +vicuna-7b-v1.5,INT8-CW,1024,8013.2,1558.1,95.1,10.52 +mistral-7b-v0.1,INT8-CW,32,7886.9,140.6,95.2,10.50 +qwen2-7b,INT8-CW,1024,10573.1,1564.5,95.3,10.49 +codegen25-7b,INT8-CW,1024,8253.1,1526.3,95.7,10.45 +zephyr-7b-beta,INT8-CW,32,7785.3,144.4,95.8,10.44 +stablelm-7b,INT8-CW,1020,8921.9,1845,96.9,10.32 +mistral-7b-v0.1,INT8-CW,1007,8127.4,1648.4,97.4,10.27 +qwen-7b-chat,INT8-CW,32,11083.2,140.6,97.7,10.24 +qwen1.5-7b-chat,INT8-CW,32,8870,156.4,98.1,10.19 +llama-3.1-8b,INT8-CW,31,8600.3,189.2,98.4,10.16 +mistral-7b-v0.1,INT8-CW,1024,8134.7,1554.1,98.4,10.16 +qwen-14b-chat,INT4-MIXED,32,9876.2,192.3,98.6,10.14 +zephyr-7b-beta,INT8-CW,1024,8035.2,1580.4,98.8,10.12 +llama-3-8b,INT8-CW,32,8694.2,150.7,99.5,10.05 +llama-3-8b,INT8-CW,33,8700.4,175.4,99.8,10.02 +phi-3-mini-4k-instruct,FP16,1023,8795.2,601.3,99.9,10.01 diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_9-288V.csv similarity index 100% rename from docs/sphinx_setup/_static/benchmarks_files/llm_models.csv rename to docs/sphinx_setup/_static/benchmarks_files/llm_models_9-288V.csv diff --git a/docs/sphinx_setup/_static/download/supported_models.csv b/docs/sphinx_setup/_static/download/supported_models.csv index 1c7db3ab6bfc9d..87ea37b0f207c3 100644 --- a/docs/sphinx_setup/_static/download/supported_models.csv +++ b/docs/sphinx_setup/_static/download/supported_models.csv @@ -344,8 +344,7 @@ mask_rcnn_resnet101_atrous_coco,Instance Segmentation,tf,FP16-INT8,+,+, mask_rcnn_resnet50_atrous_coco,Instance Segmentation,tf,FP16,+,+, mask_rcnn_resnet50_atrous_coco,Instance Segmentation,tf,FP16-INT8,+,+, mask_rcnn_resnet50_atrous_coco,Instance Segmentation,tf,FP32,+,+, -MaskRCNN-12,Object Detection,onnx,FP16,,,+ -MaskRCNN-12,Object Detection,onnx,FP32,+,+,+ +MaskRCNN-12,Object Detection,onnx,FP32,+,+, mbart-large-50-many-to-one-mmt,Natural Language Processing,pytorch,intel-optimum default,,+, Meta-Llama-3-8B,Large Language Model,pytorch,intel-optimum default,,+, Meta-Llama-3-8B-Instruct,Large Language Model,pytorch,intel-optimum default,,+, diff --git a/docs/sphinx_setup/_static/html/modal.html b/docs/sphinx_setup/_static/html/modal.html index ac425599b821ce..38eb673824f97e 100644 --- a/docs/sphinx_setup/_static/html/modal.html +++ b/docs/sphinx_setup/_static/html/modal.html @@ -11,9 +11,6 @@

Configure Graphs

-
- Clear All -
diff --git a/docs/sphinx_setup/_static/html/modalLLM.html b/docs/sphinx_setup/_static/html/modalLLM.html new file mode 100644 index 00000000000000..37b569d0bd4078 --- /dev/null +++ b/docs/sphinx_setup/_static/html/modalLLM.html @@ -0,0 +1,92 @@ +
+
+

Benchmark Graph Builder

+ × +
+
+ + +
+
+
\ No newline at end of file diff --git a/docs/sphinx_setup/_static/js/graphs.js b/docs/sphinx_setup/_static/js/graphs.js index f8146c1580c58e..7171aed374dd99 100644 --- a/docs/sphinx_setup/_static/js/graphs.js +++ b/docs/sphinx_setup/_static/js/graphs.js @@ -1,7 +1,6 @@ // =================== GENERAL OUTPUT CONFIG ========================= class Filter { - // param: GraphData[], networkModels[] static FilterByNetworkModel(graphDataArr, networkModels) { const optionMap = new Map(); @@ -10,7 +9,6 @@ class Filter { .forEach(item => optionMap.set(item.Platform, item)); return Array.from(optionMap.values()); } - // param: GraphData[], ieType static ByIeTypes(graphDataArr, ieTypes) { const optionMap = new Map(); @@ -19,32 +17,26 @@ class Filter { .forEach(item => optionMap.set(item.Platform, item)); return Array.from(optionMap.values()); } - // param: GraphData[], ieType, networkModels static ByTypesAndModels(graphDataArr, ieTypes, models) { - const optionMap = new Map(); - graphDataArr - .filter(graphData => ieTypes.includes(graphData.PlatformType)) - .filter(graphData => models.includes(graphData.Model)) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); + return Array.from( + graphDataArr + .filter(({ PlatformType, Model }) => ieTypes.includes(PlatformType) && models.includes(Model)) + .reduce((map, item) => map.set(item.Platform, item), new Map()) + .values() + ); } - // param: GraphData[], clientPlatforms static ByIeKpis(graphDataArr, clientPlatforms) { - var kpis = [] - clientPlatforms.forEach((platformName) => { - graphDataArr.filter((data) => { - if (data.Platform.includes(platformName)) { - for (var key in data.Parameters) { - if (!kpis.includes(key)) kpis.push(key) - } + return Array.from( + graphDataArr.reduce((kpiSet, data) => { + if (clientPlatforms.some(platformName => data.Platform.includes(platformName))) { + Object.keys(data.Parameters).forEach(key => kpiSet.add(key)); } - }) - }) - return kpis; + return kpiSet; + }, new Set()) + ); } - // param: GraphData[] static getParameters(graphDataArr) { var parameters = [] @@ -55,7 +47,6 @@ class Filter { }) return parameters; } - // param: GraphData[] static getIeTypes(graphDataArr) { var kpis = [] @@ -66,36 +57,27 @@ class Filter { }) return kpis; } - // param: GraphData[], clientPlatforms[] static ByClientPlatforms(graphDataArr, platformsArr) { return graphDataArr.filter((data) => { return platformsArr.includes(data.Platform) }); } - - // param: GraphData[], coreTypes[] - static FilterByCoreTypes(graphDataArr, coreTypes) { - if (coreTypes) { - return graphDataArr.filter((data) => coreTypes.includes(data.PlatformType)); - } - return graphDataArr; - } } class Modal { static getPrecisionsLabels(graphDataArr) { - var kpis = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - data.Parameters[key].Precisions.forEach((key) => { - Object.keys(key).forEach((key) => { - if (!kpis.includes(key.toUpperCase())) kpis.push(key.toUpperCase()) + const kpisSet = new Set(); + graphDataArr.forEach(data => { + Object.values(data.Parameters).forEach(param => { + param.Precisions.forEach(precision => { + Object.keys(precision).forEach(key => { + kpisSet.add(key.toUpperCase()); }); - }) - } - }) - return kpis; + }); + }); + }); + return Array.from(kpisSet); } static getPrecisions(appConfig, labels) { @@ -111,26 +93,43 @@ class Modal { } } - class Graph { // functions to get unique keys static getNetworkModels(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.Model))); + return Array.from(new Set(graphDataArr.map(obj => obj.Model))) + .sort((a, b) => a.localeCompare(b)); } static getIeTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))); - } - static getCoreTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.ieType))); + return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))).sort((a, b) => a.localeCompare(b)); } // param: GraphData[] static getPlatformNames(graphDataArr) { - return graphDataArr.map((data) => data.Platform); + return graphDataArr.map((data) => data.Platform) + .sort((a, b) => a.localeCompare(b)); + } + + // param: GraphData[], engine: string, precisions: list + static getDatabyParameter(graphDataArr, engine, array) { + if (!Array.isArray(array[engine])) { + array[engine] = []; + } + array[engine].push(graphDataArr.Parameters[engine].Precisions); + return array; + } + + // this returns an object that is used to ender the chart + static getGraphConfig(engine, precisions, appConfig) { + return { + chartTitle: 'Throughput vs Latency', + iconClass: 'latency-icon', + datasets: precisions.map((precision) => appConfig.PrecisionData[engine][precision]), + unit: "None" + }; } // param: GraphData[], parameterName: string, precisions: list - static getDatabyParameter(graphDataArr, parameterName, precisions) { + static getDatabyParameterOld(graphDataArr, parameterName, precisions) { var array = []; graphDataArr.forEach((item) => { if (item.Parameters[parameterName] !== undefined) { @@ -149,7 +148,7 @@ class Graph { } // this returns an object that is used to ender the chart - static getGraphConfig(parameterName, item, precisions, appConfig) { + static getGraphConfigOld(parameterName, item, precisions, appConfig) { return { chartTitle: Graph.capitalizeFirstLetter(parameterName), iconClass: parameterName + '-icon', @@ -171,10 +170,11 @@ class ChartDisplay { $(document).ready(function () { - $('.ov-toolkit-benchmark-results').on('click', () => showModal("graph-data-ov.json")); - $('.ovms-toolkit-benchmark-results').on('click', () => showModal("graph-data-ovms.json")); - function clickBuildGraphs(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions) { - renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions); + $('.ov-toolkit-benchmark-results').on('click', () => showModal("graph-data-ov.json", false)); + $('.ovms-toolkit-benchmark-results').on('click', () => showModal("graph-data-ovms.json", false)); + $('.ovms-toolkit-benchmark-llm-result').on('click', () => showModal("graph-data-ovms-genai.json", true)); + function clickBuildGraphs(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions, isLLM) { + renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions, isLLM); $('.modal-footer').show(); $('#modal-display-graphs').show(); $('.edit-settings-btn').off('click').on('click', (event) => { @@ -186,7 +186,6 @@ $(document).ready(function () { $('.graph-chart-title-header').off('click').on('click', (event) => { var parent = event.target.parentElement; - if ($(parent).children('.chart-wrap,.empty-chart-container').is(":visible")) { $(parent).children('.chart-wrap,.empty-chart-container').hide(); $(parent).children('.chevron-right-btn').show(); @@ -204,16 +203,16 @@ $(document).ready(function () { $('body').css('overflow', 'auto'); } - function showModal(file) { + function showModal(file, isLLM) { $('body').css('overflow', 'hidden'); - fetch('../_static/benchmarks_files/data/'+ file) + fetch('../_static/benchmarks_files/data/' + file) .then((response) => response.json()) .then((jsonData) => { fetch('../_static/benchmarks_files/graph-config.json') .then((configResponse) => configResponse.json()) .then((appConfig) => { - renderModal(jsonData, appConfig) + renderModal(jsonData, appConfig, isLLM) }) }); } @@ -263,11 +262,12 @@ $(document).ready(function () { $('#build-graphs-btn').prop('disabled', true); } - function renderModal(graph, appConfig) { + function renderModal(graph, appConfig, isLLM) { + var modalPath = isLLM === true ? '../_static/html/modalLLM.html' : '../_static/html/modal.html' new Graph(graph); var networkModels = Graph.getNetworkModels(graph); var ieTypes = Graph.getIeTypes(graph); - fetch('../_static/html/modal.html').then((response) => response.text()).then((text) => { + fetch(modalPath).then((response) => response.text()).then((text) => { // generate and configure modal container var modal = $('
'); @@ -281,13 +281,13 @@ $(document).ready(function () { const models = networkModels.map((networkModel) => createCheckMark(networkModel, 'networkmodel')); modal.find('.models-column').append(models); - const selectAllModelsButton = createCheckMark('', 'networkmodel'); + const selectAllModelsButton = createCheckMark('', 'networkmodel', false , false); modal.find('.models-selectall').append(selectAllModelsButton); - const selectAllPlatformsButton = createCheckMark('', 'platform'); + const selectAllPlatformsButton = createCheckMark('', 'platform', false , false); modal.find('.platforms-selectall').append(selectAllPlatformsButton); - const precisions = Modal.getPrecisionsLabels(graph).map((precision) => createCheckMark(precision, 'precision', false)); + const precisions = Modal.getPrecisionsLabels(graph).map((precision) => createCheckMark(precision, 'precision', false , false)); modal.find('.precisions-column').append(precisions); selectAllCheckboxes(precisions); @@ -302,24 +302,20 @@ $(document).ready(function () { modal.find('#modal-display-graphs').hide(); modal.find('.ietype-column input').first().prop('checked', true); - const kpiLabels = Filter.getParameters(graph).map((parameter) => createCheckMark(parameter, 'kpi', false)); + const kpiLabels = Filter.getParameters(graph).map((parameter) => createCheckMark(parameter, 'kpi', false , true)); modal.find('.kpi-column').append(kpiLabels); $('body').prepend(modal); - preselectDefaultSettings(graph, modal, appConfig); - - //is not generic solution :( if (appConfig.DefaultSelections.platformTypes?.data?.includes('Select All')) { selectAllCheckboxes(iefilter); - }; + preselectDefaultSettings(graph, modal, appConfig); renderClientPlatforms(graph, modal); - $('.clear-all-btn').on('click', clearAll); $('#build-graphs-btn').on('click', () => { $('#modal-configure-graphs').hide(); - clickBuildGraphs(graph, appConfig, getSelectedNetworkModels(), getSelectedIeTypes(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(appConfig, getSelectedPrecisions())); + clickBuildGraphs(graph, appConfig, getSelectedNetworkModels(), getSelectedIeTypes(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(appConfig, getSelectedPrecisions()), isLLM); }); $('.modal-close').on('click', hideModal); $('.close-btn').on('click', hideModal); @@ -333,18 +329,18 @@ $(document).ready(function () { modal.find('.models-selectall input').on('click', function () { if ($(this).prop('checked')) selectAllCheckboxes(models); else deSelectAllCheckboxes(models); - + renderClientPlatforms(graph, modal) }); modal.find('.platforms-selectall input').on('click', function () { - if ($(this).prop('checked')) + if ($(this).prop('checked')) renderClientPlatforms(graph, modal) else { var enabledPlatforms = modal.find('.platforms-column .checkmark-container'); deSelectCheckbox(enabledPlatforms); }; - + }); modal.find('.models-column input').on('click', function () { @@ -393,49 +389,20 @@ $(document).ready(function () { precisions.prop('disabled', false); } - function clearAll() { - $('.modal-content-grid-container input:checkbox').each((index, object) => $(object).prop('checked', false)); - validatePrecisionSelection(); - validateSelections(); - } - function preselectDefaultSettings(graph, modal, appConfig) { - - const defaultSelections = appConfig.DefaultSelections; - - selectDefaultPlatformType(defaultSelections.platformTypes, graph, modal); - - applyPlatformFilters(defaultSelections.platformFilters, modal, graph); - - clearAllSettings(defaultSelections); - + selectDefaultPlatformType(appConfig.DefaultSelections.platformTypes, graph, modal); + clearAllSettings(appConfig.DefaultSelections); validateSelections(); validatePrecisionSelection(); } - function selectDefaultPlatformType(platformTypes, graph, modal) { if (!platformTypes) return; - const type = platformTypes.data[0]; $(`input[data-ietype="${type}"]`).prop('checked', true); renderClientPlatforms(graph, modal); } - - function applyPlatformFilters(platformFilters, modal, graph) { - if (!platformFilters) return; - - const filters = modal.find('.selectable-box-container').children('.selectable-box'); - filters.removeClass('selected'); - - platformFilters.data.forEach(selection => { - filters.filter(`[data-${platformFilters.name}="${selection}"]`).addClass('selected'); - }); - - renderClientPlatforms(graph, modal); - } - + function clearAllSettings(defaultSelections) { - clearAll(); Object.keys(defaultSelections).forEach(setting => { const { name, data } = defaultSelections[setting]; data.forEach(selection => { @@ -457,14 +424,15 @@ $(document).ready(function () { var platformNames = Graph.getPlatformNames(fPlatforms); $('.platforms-column .checkmark-container').remove(); - const clientPlatforms = platformNames.map((platform) => createCheckMark(platform, 'platform', true)); - + const clientPlatforms = platformNames.map((platform) => createCheckMark(platform, 'platform', true, false)); + var enabledPlatforms = filterPlatforms(graph, getSelectedIeTypes(), getSelectedNetworkModels()); enableCheckBoxes(clientPlatforms, enabledPlatforms); modal.find('.platforms-column').append(clientPlatforms); enableParmeters(graph, getSelectedClientPlatforms()); modal.find('.platforms-column input').on('click', validateSelections); + validateSelections(); } function enableParmeters(graph, clientPlatforms) { @@ -480,11 +448,12 @@ $(document).ready(function () { }) } - function createCheckMark(itemLabel, modelLabel, disabled) { + function createCheckMark(itemLabel, modelLabel, disabled, checked = false) { const item = $('