diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index 75d1882b8cee89..75c7ba90db7e76 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -64,7 +64,7 @@ implemented in your solutions. Click the buttons below to see the chosen benchma :outline: :expand: - :material-regular:`bar_chart;1.4em` OVMS for GenAI (incoming) + :material-regular:`bar_chart;1.4em` OVMS for GenAI diff --git a/docs/sphinx_setup/_static/js/graphs.js b/docs/sphinx_setup/_static/js/graphs.js index 4d621ce0780261..168c1c348e7a08 100644 --- a/docs/sphinx_setup/_static/js/graphs.js +++ b/docs/sphinx_setup/_static/js/graphs.js @@ -1,11 +1,196 @@ -// =================== ADDITIONAL OUTPUT CONFIG ========================= +// =================== GENERAL OUTPUT CONFIG ========================= + +class Filter { + // param: GraphData[], networkModels[] + static FilterByNetworkModel(graphDataArr, networkModels) { + const optionMap = new Map(); + networkModels.map((model) => graphDataArr.filter((graphData => graphData.Model === model))) + .flat(1) + .forEach(item => optionMap.set(item.Platform, item)); + return Array.from(optionMap.values()); + } + + // param: GraphData[], ieType + static ByIeTypes(graphDataArr, ieTypes) { + const optionMap = new Map(); + graphDataArr + .filter(graphData => ieTypes.includes(graphData.PlatformType)) + .forEach(item => optionMap.set(item.Platform, item)); + return Array.from(optionMap.values()); + } + + // param: GraphData[], ieType, networkModels + static ByTypesAndModels(graphDataArr, ieTypes, models) { + return Array.from( + graphDataArr + .filter(({ PlatformType, Model }) => ieTypes.includes(PlatformType) && models.includes(Model)) + .reduce((map, item) => map.set(item.Platform, item), new Map()) + .values() + ).sort((a, b) => a.Platform.localeCompare(b.Platform)); + } + + // param: GraphData[], clientPlatforms + static ByIeKpis(graphDataArr, clientPlatforms) { + return Array.from( + graphDataArr.reduce((kpiSet, data) => { + if (clientPlatforms.some(platformName => data.Platform.includes(platformName))) { + Object.keys(data.Parameters).forEach(key => kpiSet.add(key)); + } + return kpiSet; + }, new Set()) + ); + } + + // param: GraphData[] + static getParameters(graphDataArr) { + var parameters = [] + graphDataArr.filter((data) => { + for (var key in data.Parameters) { + if (!parameters.includes(Graph.capitalizeFirstLetter(key))) parameters.push(Graph.capitalizeFirstLetter(key)) + } + }) + return parameters; + } + + // param: GraphData[] + static getIeTypes(graphDataArr) { + var kpis = [] + graphDataArr.filter((data) => { + for (var key in data.Parameters) { + if (!kpis.includes(Graph.capitalizeFirstLetter(key))) kpis.push(Graph.capitalizeFirstLetter(key)) + } + }) + return kpis; + } + + // param: GraphData[], clientPlatforms[] + static ByClientPlatforms(graphDataArr, platformsArr) { + return graphDataArr.filter((data) => { + return platformsArr.includes(data.Platform) + }); + } + + // param: GraphData[], coreTypes[] + static FilterByCoreTypes(graphDataArr, coreTypes) { + if (coreTypes) { + return graphDataArr.filter((data) => coreTypes.includes(data.PlatformType)); + } + return graphDataArr; + } +} + +class Modal { + static getPrecisionsLabels(graphDataArr) { + const kpisSet = new Set(); + graphDataArr.forEach(data => { + Object.values(data.Parameters).forEach(param => { + param.Precisions.forEach(precision => { + Object.keys(precision).forEach(key => { + kpisSet.add(key.toUpperCase()); + }); + }); + }); + }); + return Array.from(kpisSet); + } + + static getPrecisions(appConfig, labels) { + return labels.map((label) => { + var prec = appConfig.PrecisionsMap[label]; + if (prec !== undefined) { + return prec; + } + else { + return "no name"; + } + }); + } +} + +class Graph { + // functions to get unique keys + static getNetworkModels(graphDataArr) { + return Array.from(new Set(graphDataArr.map(obj => obj.Model))) + .sort((a, b) => a.localeCompare(b)); + } + static getIeTypes(graphDataArr) { + return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))); + } + static getCoreTypes(graphDataArr) { + return Array.from(new Set(graphDataArr.map((obj) => obj.ieType))); + } + + // param: GraphData[] + static getPlatformNames(graphDataArr) { + return graphDataArr.map((data) => data.Platform); + } + + // param: GraphData[], engine: string, precisions: list + static getDatabyParameter(graphDataArr, engine, array) { + if (!Array.isArray(array[engine])) { + array[engine] = []; + } + array[engine].push(graphDataArr.Parameters[engine].Precisions); + return array; + } + + // this returns an object that is used to ender the chart + static getGraphConfig(engine, precisions, appConfig) { + return { + chartTitle: 'Throughput vs Latency', + iconClass: 'latency-icon', + datasets: precisions.map((precision) => appConfig.PrecisionData[engine][precision]), + unit: "None" + }; + } + + // param: GraphData[], parameterName: string, precisions: list + static getDatabyParameterOld(graphDataArr, parameterName, precisions) { + var array = []; + graphDataArr.forEach((item) => { + if (item.Parameters[parameterName] !== undefined) { + array.push(item.Parameters[parameterName].Precisions); + } + else { + var obj = {}; + precisions.forEach((prec) => { + obj[prec] = 0; + }) + array.push([obj]) + } + }) + return array; + + } + + // this returns an object that is used to ender the chart + static getGraphConfigOld(parameterName, item, precisions, appConfig) { + return { + chartTitle: Graph.capitalizeFirstLetter(parameterName), + iconClass: parameterName + '-icon', + unit: item.Parameters[parameterName]?.Unit, + datasets: precisions.map((precision) => appConfig.PrecisionData[precision]), + }; + } + static capitalizeFirstLetter(string) { + return string.charAt(0).toUpperCase() + string.slice(1); + } +} + +class ChartDisplay { + constructor(mode, numberOfCharts) { + this.mode = mode; + this.numberOfChartsInRow = numberOfCharts; + } +} $(document).ready(function () { - $('.ov-toolkit-benchmark-results').on('click', () => showModal("graph-data-ov.json")); - $('.ovms-toolkit-benchmark-results').on('click', () => showModal("graph-data-ovms.json")); - function clickBuildGraphs(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions) { - renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions); + $('.ov-toolkit-benchmark-results').on('click', () => showModal("graph-data-ov.json", false)); + $('.ovms-toolkit-benchmark-results').on('click', () => showModal("graph-data-ovms.json", false)); + $('.ovms-toolkit-benchmark-llm-result').on('click', () => showModal("graph-data-ovms-genai.json", true)); + function clickBuildGraphs(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions, isLLM) { + renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions, isLLM); $('.modal-footer').show(); $('#modal-display-graphs').show(); $('.edit-settings-btn').off('click').on('click', (event) => { @@ -34,16 +219,16 @@ $(document).ready(function () { $('body').css('overflow', 'auto'); } - function showModal(file) { + function showModal(file, isLLM) { $('body').css('overflow', 'hidden'); - fetch('../_static/benchmarks_files/data/'+ file) + fetch('../_static/benchmarks_files/data/' + file) .then((response) => response.json()) .then((jsonData) => { fetch('../_static/benchmarks_files/graph-config.json') .then((configResponse) => configResponse.json()) .then((appConfig) => { - renderModal(jsonData, appConfig) + renderModal(jsonData, appConfig, isLLM) }) }); } @@ -93,11 +278,12 @@ $(document).ready(function () { $('#build-graphs-btn').prop('disabled', true); } - function renderModal(graph, appConfig) { + function renderModal(graph, appConfig, isLLM) { + var modalPath = isLLM === true ? '../_static/html/modalLLM.html' : '../_static/html/modal.html' new Graph(graph); var networkModels = Graph.getNetworkModels(graph); var ieTypes = Graph.getIeTypes(graph); - fetch('../_static/html/modal.html').then((response) => response.text()).then((text) => { + fetch(modalPath).then((response) => response.text()).then((text) => { // generate and configure modal container var modal = $('
'); @@ -111,10 +297,10 @@ $(document).ready(function () { const models = networkModels.map((networkModel) => createCheckMark(networkModel, 'networkmodel')); modal.find('.models-column').append(models); - const selectAllModelsButton = createCheckMark('', 'networkmodel'); + const selectAllModelsButton = createCheckMark('', 'networkmodel'); modal.find('.models-selectall').append(selectAllModelsButton); - const selectAllPlatformsButton = createCheckMark('', 'platform'); + const selectAllPlatformsButton = createCheckMark('', 'platform'); modal.find('.platforms-selectall').append(selectAllPlatformsButton); const precisions = Modal.getPrecisionsLabels(graph).map((precision) => createCheckMark(precision, 'precision', false)); @@ -142,14 +328,14 @@ $(document).ready(function () { //is not generic solution :( if (appConfig.DefaultSelections.platformTypes?.data?.includes('Select All')) { selectAllCheckboxes(iefilter); - + }; renderClientPlatforms(graph, modal); $('.clear-all-btn').on('click', clearAll); $('#build-graphs-btn').on('click', () => { $('#modal-configure-graphs').hide(); - clickBuildGraphs(graph, appConfig, getSelectedNetworkModels(), getSelectedIeTypes(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(appConfig, getSelectedPrecisions())); + clickBuildGraphs(graph, appConfig, getSelectedNetworkModels(), getSelectedIeTypes(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(appConfig, getSelectedPrecisions()), isLLM); }); $('.modal-close').on('click', hideModal); $('.close-btn').on('click', hideModal); @@ -163,18 +349,18 @@ $(document).ready(function () { modal.find('.models-selectall input').on('click', function () { if ($(this).prop('checked')) selectAllCheckboxes(models); else deSelectAllCheckboxes(models); - + renderClientPlatforms(graph, modal) }); modal.find('.platforms-selectall input').on('click', function () { - if ($(this).prop('checked')) + if ($(this).prop('checked')) renderClientPlatforms(graph, modal) else { var enabledPlatforms = modal.find('.platforms-column .checkmark-container'); deSelectCheckbox(enabledPlatforms); }; - + }); modal.find('.models-column input').on('click', function () { @@ -232,38 +418,28 @@ $(document).ready(function () { function preselectDefaultSettings(graph, modal, appConfig) { const defaultSelections = appConfig.DefaultSelections; - selectDefaultPlatformType(defaultSelections.platformTypes, graph, modal); - applyPlatformFilters(defaultSelections.platformFilters, modal, graph); - clearAllSettings(defaultSelections); validateSelections(); validatePrecisionSelection(); } - function selectDefaultPlatformType(platformTypes, graph, modal) { if (!platformTypes) return; - const type = platformTypes.data[0]; $(`input[data-ietype="${type}"]`).prop('checked', true); renderClientPlatforms(graph, modal); } - function applyPlatformFilters(platformFilters, modal, graph) { if (!platformFilters) return; - const filters = modal.find('.selectable-box-container').children('.selectable-box'); filters.removeClass('selected'); - platformFilters.data.forEach(selection => { filters.filter(`[data-${platformFilters.name}="${selection}"]`).addClass('selected'); }); - renderClientPlatforms(graph, modal); } - function clearAllSettings(defaultSelections) { clearAll(); Object.keys(defaultSelections).forEach(setting => { @@ -288,7 +464,7 @@ $(document).ready(function () { $('.platforms-column .checkmark-container').remove(); const clientPlatforms = platformNames.map((platform) => createCheckMark(platform, 'platform', true)); - + var enabledPlatforms = filterPlatforms(graph, getSelectedIeTypes(), getSelectedNetworkModels()); enableCheckBoxes(clientPlatforms, enabledPlatforms); modal.find('.platforms-column').append(clientPlatforms); @@ -330,11 +506,11 @@ $(document).ready(function () { } function enableCheckBoxes(items, enabledItems) { - items.forEach((item) => { + items.forEach((item) => { item.find(':input').prop('disabled', true); enabledItems.forEach((platform) => { var tmp = item.find(':input'); - if(tmp[0].dataset.platform === platform.Platform){ + if (tmp[0].dataset.platform === platform.Platform) { item.find(':input').prop('checked', true); item.find(':input').prop('disabled', false); } @@ -397,13 +573,7 @@ $(document).ready(function () { li.style.marginLeft = '4px'; li.onclick = () => { - const { type } = chart.config; - if (type === 'pie' || type === 'doughnut') { - // Pie and doughnut charts only have a single dataset and visibility is per item - chart.toggleDataVisibility(item.index); - } else { - chart.setDatasetVisibility(item.datasetIndex, !chart.isDatasetVisible(item.datasetIndex)); - } + chart.toggleDataVisibility(item.index); chart.update(); }; @@ -436,6 +606,41 @@ $(document).ready(function () { } }; + function getChartOptionsByEngines(containerId, allowedAxisIDs) { + const axisConfigs = { + x: { + title: { display: true, text: 'Request Rate' } + }, + y: { + type: 'linear', + display: true, + position: 'left', + title: { display: true, text: 'Throughput [tok/s]' }, + grid: { drawOnChartArea: true } + }, + y1: { + type: 'linear', + display: true, + position: 'right', + title: { display: true, text: 'TPOT Mean Latency' }, + grid: { drawOnChartArea: true } + } + }; + + return { + responsive: true, + scales: Object.keys(axisConfigs) + .filter(key => allowedAxisIDs.includes(key)) + .reduce((obj, key) => { + obj[key] = axisConfigs[key]; + return obj; + }, {}), + plugins: { + legend: { display: false }, + htmlLegend: { containerID: containerId } + } + }; + } function getChartOptions(title, containerId) { return { responsive: true, @@ -469,25 +674,9 @@ $(document).ready(function () { } } - function getChartDataNew(labels, datasets) { - return { - labels: labels, - datasets: datasets.map((item) => { - return { - label: item.label, - data: item.data, - backgroundColor: item.color, - borderColor: 'rgba(170,170,170,0)', - barThickness: 10 - } - }) - } - } - - function renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions) { + function renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions, isLLM) { $('.chart-placeholder').empty(); $('.modal-footer').empty(); - const display = new ChartDisplay(getChartsDisplayMode(kpis.length), kpis.length); networkModels.forEach((networkModel) => { @@ -509,8 +698,17 @@ $(document).ready(function () { var filteredIeTypes = Filter.ByIeTypes(filteredNetworkModels, ieTypes); var filteredGraphData = Filter.ByClientPlatforms(filteredIeTypes, platforms); $('.chart-placeholder').append(chartContainer); + var labels = Graph.getPlatformNames(filteredGraphData); if (filteredGraphData.length > 0) { - createChartWithNewData(filteredGraphData, appConfig, chartContainer, kpis, ieTypes, precisions, display); + if (isLLM === true) { + var graphConfigs = setGraphConfigsByEngines(filteredGraphData, appConfig, kpis, precisions) + createChartWithNewDataByEngines(labels, graphConfigs, chartContainer, display); + } + else { + var graphConfigs = setGraphConfigs(filteredGraphData, appConfig, kpis, precisions) + createChartWithNewData(labels, graphConfigs, chartContainer, display); + } + } else { createEmptyChartContainer(chartContainer); } @@ -526,22 +724,56 @@ $(document).ready(function () { // this function should take the final data set and turn it into graphs // params: GraphData, unused, chartContainer - function createChartWithNewData(model, appConfig, chartContainer, parameters, ietype, precisions, display) { + function createChartWithNewDataByEngines(labels, graphConfigs, chartContainer, display) { var chartWrap = $('
'); chartWrap.addClass('chart-wrap'); chartContainer.append(chartWrap); - var graphConfigs = parameters.map((parameter) => { - var groupUnit = model[0]; - var kpiData = Graph.getDatabyParameterOld(model, appConfig.ParametersMap[parameter], precisions); - var config = Graph.getGraphConfigOld(appConfig.ParametersMap[parameter], groupUnit, precisions, JSON.parse(JSON.stringify(appConfig))); - precisions.forEach((precision, index) => { - config.datasets[index].data = kpiData.map(tData => tData[0][precision]); + var labelsContainer = $('
'); + labelsContainer.addClass('chart-labels-container'); + chartWrap.append(labelsContainer); + + // get the kpi title's and create headers for the graphs + var chartGraphsContainer = $('
'); + chartGraphsContainer.addClass('chart-graphs-container'); + chartWrap.append(chartGraphsContainer); + + graphConfigs.forEach((graphConfig, index) => { + const id = getRandomNumber(); + if (graphConfig.unit === undefined) { + graphConfig.unit = 'No unit.'; + } + + var graphItem = $(`
`); + graphItem.addClass('graph-item'); + var columnHeaderContainer = $('
'); + columnHeaderContainer.addClass('chart-column-title'); + var columnIcon = $('
'); + columnIcon.addClass(graphConfig.iconClass); + columnHeaderContainer.append(columnIcon); + var columnHeader = $('
'); + columnHeader.append($('
' + graphConfig.chartTitle + '
')); + columnHeaderContainer.append(columnHeader); + chartGraphsContainer.append(graphItem); + var graphClass = $('
'); + graphClass.addClass('graph-row'); + + graphItem.append(columnHeaderContainer); + graphItem.append(graphClass); + processMetricByEngines(labels, graphConfig, graphClass, 'graph-row-column', id); + window.setTimeout(() => { + var labelsItem = $('
'); + setInitialItemsVisibility(labelsItem, index, display.mode); }); - return config; }); - - // get the client platform labels and create labels for all the graphs - var labels = Graph.getPlatformNames(model); + setChartsDisplayDirection(display.mode); + adjustHeaderIcons(display.mode); + } + function createChartWithNewData(labels, graphConfigs, chartContainer, display) { + + var chartWrap = $('
'); + chartWrap.addClass('chart-wrap'); + chartContainer.append(chartWrap); + var labelsContainer = $('
'); labelsContainer.addClass('chart-labels-container'); chartWrap.append(labelsContainer); @@ -566,7 +798,7 @@ $(document).ready(function () { columnHeaderContainer.append(columnIcon); var columnHeader = $('
'); columnHeader.append($('
' + graphConfig.chartTitle + '
')); - columnHeader.append($('
' + graphConfig.unit + ' ' + appConfig.UnitDescription[graphConfig.unit] + '
')); + // columnHeader.append($('
' + graphConfig.unit + ' ' + appConfig.UnitDescription[graphConfig.unit] + '
')); columnHeaderContainer.append(columnHeader); chartGraphsContainer.append(graphItem); var graphClass = $('
'); @@ -574,7 +806,7 @@ $(document).ready(function () { graphItem.append(columnHeaderContainer); graphItem.append(graphClass); - processMetricNew(labels, graphConfig.datasets, graphConfig.chartTitle, graphClass, 'graph-row-column', id); + processMetric(labels, graphConfig.datasets, graphConfig.chartTitle, graphClass, 'graph-row-column', id); window.setTimeout(() => { const topPadding = getLabelsTopPadding(display.mode); @@ -592,12 +824,39 @@ $(document).ready(function () { setInitialItemsVisibility(labelsItem, index, display.mode); labelsContainer.append(labelsItem); }); - }); - setChartsDisplayDirection(display.mode); - adjustHeaderIcons(display.mode); + }) + } + function setGraphConfigsByEngines(model, appConfig, engines, precisions) { + var graphConfigs = model.map((platform) => { + var kpiData = []; + engines.forEach((engine) => Graph.getDatabyParameter(platform, engine, kpiData)); + var graphConfig = []; + engines.forEach((engine) => { + const engineGraphConfig = { engine: engine, config: Graph.getGraphConfig(engine, precisions, JSON.parse(JSON.stringify(appConfig))) }; + precisions.forEach((precision, index) => { + engineGraphConfig.config.datasets[index].data = kpiData[engine][0].find(el => el[precision])?.[precision]; + }); + graphConfig.push(engineGraphConfig); + graphConfig.chartTitle = platform.Platform; + }); + return graphConfig + }) + return graphConfigs; } - function processMetricNew(labels, datasets, chartTitle, container, widthClass, id) { + function setGraphConfigs(model, appConfig, parameters, precisions) { + var graphConfigs = parameters.map((parameter) => { + var groupUnit = model[0]; + var kpiData = Graph.getDatabyParameterOld(model, appConfig.ParametersMap[parameter], precisions); + var config = Graph.getGraphConfigOld(appConfig.ParametersMap[parameter], groupUnit, precisions, JSON.parse(JSON.stringify(appConfig))); + precisions.forEach((precision, index) => { + config.datasets[index].data = kpiData.map(tData => tData[0][precision]); + }); + return config; + }); + return graphConfigs + } + function processMetric(labels, datasets, chartTitle, container, widthClass, id) { // ratio for consistent chart label height var heightRatio = (30 + (labels.length * 55)); var chart = $('
'); @@ -616,13 +875,90 @@ $(document).ready(function () { window.setTimeout(() => { new Chart(context, { type: 'bar', - data: getChartDataNew(labels, datasets), + data: getChartData(labels, datasets), options: getChartOptions(chartTitle, containerId), plugins: [htmlLegendPlugin] }); }); } + function getChartData(labels, datasets) { + return { + labels: labels, + datasets: datasets.map((item) => { + return { + label: item.label, + data: item.data, + backgroundColor: item.color, + borderColor: 'rgba(170,170,170,0)', + barThickness: 10 + } + }) + } + } + + function processMetricByEngines(labels, datasets, container, widthClass, id) { + // ratio for consistent chart label height + var heightRatio = (80 + (labels.length * 55)); + var chart = $('
'); + const containerId = `legend-container-${id}`; + const legend = $(`
`); + legend.addClass('graph-legend-container'); + chart.addClass('chart'); + chart.addClass(widthClass); + chart.height(heightRatio); + var canvas = $(''); + chart.append(canvas); + container.append(chart); + container.append(legend); + var context = canvas.get(0).getContext('2d'); + context.canvas.height = heightRatio; + window.setTimeout(() => { + var labels = []; + for (const key in datasets[0].config.datasets[0].data) { + labels.push(key); + } + + var graphDatas = []; + datasets.forEach((engineSet) => { + engineSet.config.datasets.forEach((precision) => { + var precData = []; + for (const key in precision.data) { + precData.push(precision.data[key]); + } + graphDatas.push({ + label: engineSet.engine + ' ' + precision.label, + data: precData, + borderColor: precision.color, + backgroundColor: precision.color, + yAxisID: precision.label === "Throughput" ? 'y' : 'y1', + fill: false + } + ) + }) + }) + + let allowedAxisIDs = ['x']; + const newAxisIDs = [...new Set(graphDatas.map(item => item.yAxisID))]; + + newAxisIDs.forEach(id => { + if (!allowedAxisIDs.includes(id)) { + allowedAxisIDs.push(id); + } + }); + + new Chart(context, { + type: 'line', + data: { + labels: labels, + datasets: graphDatas + }, + options: getChartOptionsByEngines(containerId, allowedAxisIDs), + plugins: [htmlLegendPlugin] + }); + }); + } + function getRandomNumber() { return Math.floor(Math.random() * 100000); } diff --git a/docs/sphinx_setup/_static/js/graphsLLM.js b/docs/sphinx_setup/_static/js/graphsLLM.js deleted file mode 100644 index 4dbc0313e2a133..00000000000000 --- a/docs/sphinx_setup/_static/js/graphsLLM.js +++ /dev/null @@ -1,889 +0,0 @@ -// =================== GENERAL OUTPUT CONFIG ========================= - -class Filter { - - // param: GraphData[], networkModels[] - static FilterByNetworkModel(graphDataArr, networkModels) { - const optionMap = new Map(); - networkModels.map((model) => graphDataArr.filter((graphData => graphData.Model === model))) - .flat(1) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); - } - - // param: GraphData[], ieType - static ByIeTypes(graphDataArr, ieTypes) { - const optionMap = new Map(); - graphDataArr - .filter(graphData => ieTypes.includes(graphData.PlatformType)) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); - } - - // param: GraphData[], ieType, networkModels - static ByTypesAndModels(graphDataArr, ieTypes, models) { - const optionMap = new Map(); - graphDataArr - .filter(graphData => ieTypes.includes(graphData.PlatformType)) - .filter(graphData => models.includes(graphData.Model)) - .forEach(item => optionMap.set(item.Platform, item)); - return Array.from(optionMap.values()); - } - - // param: GraphData[], clientPlatforms - static ByIeKpis(graphDataArr, clientPlatforms) { - var kpis = [] - clientPlatforms.forEach((platformName) => { - graphDataArr.filter((data) => { - if (data.Platform.includes(platformName)) { - for (var key in data.Parameters) { - if (!kpis.includes(key)) kpis.push(key) - } - } - }) - }) - return kpis; - } - - // param: GraphData[] - static getParameters(graphDataArr) { - var parameters = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - if (!parameters.includes(Graph.capitalizeFirstLetter(key))) parameters.push(Graph.capitalizeFirstLetter(key)) - } - }) - return parameters; - } - - // param: GraphData[] - static getIeTypes(graphDataArr) { - var kpis = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - if (!kpis.includes(Graph.capitalizeFirstLetter(key))) kpis.push(Graph.capitalizeFirstLetter(key)) - } - }) - return kpis; - } - - // param: GraphData[], clientPlatforms[] - static ByClientPlatforms(graphDataArr, platformsArr) { - return graphDataArr.filter((data) => { - return platformsArr.includes(data.Platform) - }); - } - - // param: GraphData[], coreTypes[] - static FilterByCoreTypes(graphDataArr, coreTypes) { - if (coreTypes) { - return graphDataArr.filter((data) => coreTypes.includes(data.PlatformType)); - } - return graphDataArr; - } -} - -class Modal { - static getPrecisionsLabels(graphDataArr) { - var kpis = [] - graphDataArr.filter((data) => { - for (var key in data.Parameters) { - data.Parameters[key].Precisions.forEach((key) => { - Object.keys(key).forEach((key) => { - if (!kpis.includes(key.toUpperCase())) kpis.push(key.toUpperCase()) - }); - }) - } - }) - return kpis; - } - - static getPrecisions(appConfig, labels) { - return labels.map((label) => { - var prec = appConfig.PrecisionsMap[label]; - if (prec !== undefined) { - return prec; - } - else { - return "no name"; - } - }); - } -} - -class Graph { - // functions to get unique keys - static getNetworkModels(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.Model))); - } - static getIeTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))); - } - static getCoreTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.ieType))); - } - - // param: GraphData[] - static getPlatformNames(graphDataArr) { - return graphDataArr.map((data) => data.Platform); - } - - // param: GraphData[], engine: string, precisions: list - static getDatabyParameter(graphDataArr, engine, array) { - if (!Array.isArray(array[engine])) { - array[engine] = []; - } - array[engine].push(graphDataArr.Parameters[engine].Precisions); - return array; - } - - // this returns an object that is used to ender the chart - static getGraphConfig(engine, precisions, appConfig) { - return { - chartTitle: 'Throughput vs Latency', - iconClass: 'latency-icon', - datasets: precisions.map((precision) => appConfig.PrecisionData[engine][precision]), - unit: "None" - }; - } - // param: GraphData[], parameterName: string, precisions: list - static getDatabyParameterOld(graphDataArr, parameterName, precisions) { - var array = []; - graphDataArr.forEach((item) => { - if (item.Parameters[parameterName] !== undefined) { - array.push(item.Parameters[parameterName].Precisions); - } - else { - var obj = {}; - precisions.forEach((prec) => { - obj[prec] = 0; - }) - array.push([obj]) - } - }) - return array; - - } - - // this returns an object that is used to ender the chart - static getGraphConfigOld(parameterName, item, precisions, appConfig) { - return { - chartTitle: Graph.capitalizeFirstLetter(parameterName), - iconClass: parameterName + '-icon', - unit: item.Parameters[parameterName]?.Unit, - datasets: precisions.map((precision) => appConfig.PrecisionData[precision]), - }; - } - static capitalizeFirstLetter(string) { - return string.charAt(0).toUpperCase() + string.slice(1); - } -} - -class ChartDisplay { - constructor(mode, numberOfCharts) { - this.mode = mode; - this.numberOfChartsInRow = numberOfCharts; - } -} - -$(document).ready(function () { - $('.ovms-toolkit-benchmark-llm-result').on('click', () => showModal("graph-data-ovms-genai.json")); - function clickBuildGraphsLLM(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions) { - renderData(graph, appConfig, networkModels, ieTypes, platforms, kpis, precisions); - $('.modal-footer').show(); - $('#modal-display-graphs').show(); - $('.edit-settings-btn').off('click').on('click', (event) => { - $('#modal-configure-graphs').show(); - $('#modal-display-graphs').hide(); - $('.modal-footer').hide(); - $('.chart-placeholder').empty(); - }); - - $('.graph-chart-title-header').off('click').on('click', (event) => { - var parent = event.target.parentElement; - - if ($(parent).children('.chart-wrap,.empty-chart-container').is(":visible")) { - $(parent).children('.chart-wrap,.empty-chart-container').hide(); - $(parent).children('.chevron-right-btn').show(); - $(parent).children('.chevron-down-btn').hide(); - } else { - $(parent).children('.chart-wrap,.empty-chart-container').show(); - $(parent).children('.chevron-down-btn').show(); - $(parent).children('.chevron-right-btn').hide(); - } - }); - } - - function hideModal() { - $('#graphModal').remove(); - $('body').css('overflow', 'auto'); - } - - function showModal(file) { - $('body').css('overflow', 'hidden'); - - fetch('../_static/benchmarks_files/data/' + file) - .then((response) => response.json()) - .then((jsonData) => { - fetch('../_static/benchmarks_files/graph-config.json') - .then((configResponse) => configResponse.json()) - .then((appConfig) => { - renderModal(jsonData, appConfig) - }) - }); - } - - function getSelectedNetworkModels() { - return $('.models-column input:checked, .platforms-column input:checked').not('[data-networkmodel="Select All"]').map(function () { - return $(this).data('networkmodel'); - }).get(); - } - - function getSelectedIeTypes() { - return $('.ietype-column input:checked').map(function () { - return $(this).data('ietype'); - }).get(); - } - - function getSelectedClientPlatforms() { - return $('.platforms-column input:checked').map(function () { - return $(this).data('platform'); - }).get(); - } - - function getSelectedKpis() { - return $('.kpi-column input:checked').map(function () { - return $(this).data('kpi'); - }).get(); - } - - function getSelectedPrecisions() { - return $('.precisions-column input:checked').map(function () { - return $(this).data('precision'); - }).get(); - } - - function validateSelections() { - if (getSelectedNetworkModels().length > 0 - && getSelectedIeTypes() - && getSelectedClientPlatforms().length > 0 - && getSelectedKpis().length > 0) { - if (getSelectedPrecisions().length > 0) { - $('#build-graphs-btn').prop('disabled', false); - return; - } - $('#build-graphs-btn').prop('disabled', true); - return; - } - $('#build-graphs-btn').prop('disabled', true); - } - - function renderModal(graph, appConfig) { - new Graph(graph); - var networkModels = Graph.getNetworkModels(graph); - var ieTypes = Graph.getIeTypes(graph); - fetch('../_static/html/modalLLM.html').then((response) => response.text()).then((text) => { - - // generate and configure modal container - var modal = $('
'); - modal.attr('id', 'graphModal'); - modal.addClass('modal'); - var modalContent = $(text); - modalContent.attr('id', 'graphModalContent'); - modalContent.addClass('modal-content'); - modal.append(modalContent); - - const models = networkModels.map((networkModel) => createCheckMark(networkModel, 'networkmodel')); - modal.find('.models-column').append(models); - - const selectAllModelsButton = createCheckMark('', 'networkmodel'); - modal.find('.models-selectall').append(selectAllModelsButton); - - const selectAllPlatformsButton = createCheckMark('', 'platform'); - modal.find('.platforms-selectall').append(selectAllPlatformsButton); - - const precisions = Modal.getPrecisionsLabels(graph).map((precision) => createCheckMark(precision, 'precision', false)); - modal.find('.precisions-column').append(precisions); - - selectAllCheckboxes(precisions); - disableAllCheckboxes(precisions); - - const selectAllTypesButton = createCheckMark('', 'ietype') - modal.find('.ietype-selectall').append(selectAllTypesButton); - - const iefilter = ieTypes.map((ieType) => createCheckMark(ieType, 'ietype')); - modal.find('.ietype-column').append(iefilter); - - modal.find('#modal-display-graphs').hide(); - modal.find('.ietype-column input').first().prop('checked', true); - - const kpiLabels = Filter.getParameters(graph).map((parameter) => createCheckMark(parameter, 'kpi', false)); - modal.find('.kpi-column').append(kpiLabels); - - $('body').prepend(modal); - - preselectDefaultSettings(graph, modal, appConfig); - - //is not generic solution :( - if (appConfig.DefaultSelections.platformTypes?.data?.includes('Select All')) { - selectAllCheckboxes(iefilter); - - }; - renderClientPlatforms(graph, modal); - - $('.clear-all-btn').on('click', clearAll); - $('#build-graphs-btn').on('click', () => { - $('#modal-configure-graphs').hide(); - clickBuildGraphsLLM(graph, appConfig, getSelectedNetworkModels(), getSelectedIeTypes(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(appConfig, getSelectedPrecisions())); - }); - $('.modal-close').on('click', hideModal); - $('.close-btn').on('click', hideModal); - - modal.find('.ietype-selectall input').on('click', function () { - if ($(this).prop('checked')) - selectAllCheckboxes(iefilter); - else deSelectAllCheckboxes(iefilter); - }); - - modal.find('.models-selectall input').on('click', function () { - if ($(this).prop('checked')) selectAllCheckboxes(models); - else deSelectAllCheckboxes(models); - - renderClientPlatforms(graph, modal) - }); - - modal.find('.platforms-selectall input').on('click', function () { - if ($(this).prop('checked')) - renderClientPlatforms(graph, modal) - else { - var enabledPlatforms = modal.find('.platforms-column .checkmark-container'); - deSelectCheckbox(enabledPlatforms); - }; - - }); - - modal.find('.models-column input').on('click', function () { - if (!$(this)[0].checked) { - deSelectCheckbox(selectAllModelsButton); - } - }); - - - modal.find('.ietype-column input').on('click', function () { - if (!$(this)[0].checked) { - deSelectCheckbox(selectAllTypesButton); - } - }); - - modal.find('.models-column input').on('click', () => renderClientPlatforms(graph, modal)); - modal.find('.ietype-column input').on('click', () => renderClientPlatforms(graph, modal)); - modal.find('.ietype-selectall input').on('click', () => renderClientPlatforms(graph, modal)); - modal.find('.platforms-column').on('click', () => enableParmeters(graph, getSelectedClientPlatforms())); - - modal.find('.kpi-column input').on('click', validatePrecisionSelection); - modal.find('input').on('click', validateSelections); - - var modalFilters = document.getElementById("modal-filters"); - - var showFiltersButton = document.getElementById("filters"); - showFiltersButton.onclick = function () { - modalFilters.style.display = "block"; - } - - var closeFiltersButton = document.getElementsByClassName("close-filters")[0]; - closeFiltersButton.onclick = function () { - modalFilters.style.display = "none"; - } - - window.onclick = function (event) { - if (event.target == modalFilters) { - modalFilters.style.display = "none"; - } - } - }); - } - - function validatePrecisionSelection() { - const precisions = $('.precisions-column').find('input') - precisions.prop('disabled', false); - } - - function clearAll() { - $('.modal-content-grid-container input:checkbox').each((index, object) => $(object).prop('checked', false)); - validatePrecisionSelection(); - validateSelections(); - } - - function preselectDefaultSettings(graph, modal, appConfig) { - - const defaultSelections = appConfig.DefaultSelections; - - selectDefaultPlatformType(defaultSelections.platformTypes, graph, modal); - - applyPlatformFilters(defaultSelections.platformFilters, modal, graph); - - clearAllSettings(defaultSelections); - - validateSelections(); - validatePrecisionSelection(); - } - - function selectDefaultPlatformType(platformTypes, graph, modal) { - if (!platformTypes) return; - - const type = platformTypes.data[0]; - $(`input[data-ietype="${type}"]`).prop('checked', true); - renderClientPlatforms(graph, modal); - } - - function applyPlatformFilters(platformFilters, modal, graph) { - if (!platformFilters) return; - - const filters = modal.find('.selectable-box-container').children('.selectable-box'); - filters.removeClass('selected'); - - platformFilters.data.forEach(selection => { - filters.filter(`[data-${platformFilters.name}="${selection}"]`).addClass('selected'); - }); - - renderClientPlatforms(graph, modal); - } - - function clearAllSettings(defaultSelections) { - clearAll(); - Object.keys(defaultSelections).forEach(setting => { - const { name, data } = defaultSelections[setting]; - data.forEach(selection => { - $(`input[data-${name}="${selection}"]`).prop('checked', true); - }); - }); - } - - function filterClientPlatforms(graph, ietypes) { - return Filter.ByIeTypes(graph, ietypes); - } - - function filterPlatforms(graph, ietypes, models) { - return Filter.ByTypesAndModels(graph, ietypes, models); - } - - function renderClientPlatforms(graph, modal) { - var fPlatforms = filterClientPlatforms(graph, getSelectedIeTypes()); - var platformNames = Graph.getPlatformNames(fPlatforms); - $('.platforms-column .checkmark-container').remove(); - - const clientPlatforms = platformNames.map((platform) => createCheckMark(platform, 'platform', true)); - - var enabledPlatforms = filterPlatforms(graph, getSelectedIeTypes(), getSelectedNetworkModels()); - enableCheckBoxes(clientPlatforms, enabledPlatforms); - modal.find('.platforms-column').append(clientPlatforms); - - enableParmeters(graph, getSelectedClientPlatforms()); - modal.find('.platforms-column input').on('click', validateSelections); - } - - function enableParmeters(graph, clientPlatforms) { - var allKpis = Filter.getParameters(graph); - - allKpis.forEach((kpi) => { - $(`input[data-kpi="${Graph.capitalizeFirstLetter(kpi)}"]`).prop('disabled', true); - }) - - var kpis = Filter.ByIeKpis(graph, clientPlatforms); - kpis.forEach((kpi) => { - $(`input[data-kpi="${Graph.capitalizeFirstLetter(kpi)}"]`).prop('disabled', false); - }) - } - - function createCheckMark(itemLabel, modelLabel, disabled) { - const item = $('