From d63ef1fbd52649fb5cc3fd1e0124bf4769e2284c Mon Sep 17 00:00:00 2001 From: Ashish Agrawal Date: Tue, 21 Feb 2023 14:30:43 -0800 Subject: [PATCH] Create switch to render line charts using vega-lite (#3106) * initial commit for vega migration Signed-off-by: Ashish Agrawal * Cleanup vega implementation Signed-off-by: Ashish Agrawal * fix minor if statement Signed-off-by: Ashish Agrawal * remove log Signed-off-by: Ashish Agrawal * minor enhancements for robustness Signed-off-by: Ashish Agrawal * Update fixes and more robust Signed-off-by: Ashish Agrawal * fix yarn.lock conflicts Signed-off-by: Ashish Agrawal * Clean up code Signed-off-by: Ashish Agrawal * add additional comments Signed-off-by: Ashish Agrawal * add unit tests Signed-off-by: Ashish Agrawal * add fixes and update tests Signed-off-by: Ashish Agrawal * fix test name Signed-off-by: Ashish Agrawal --------- Signed-off-by: Ashish Agrawal --- .../public/components/vega_vis_editor.tsx | 2 +- .../expressions/line_vega_spec_fn.test.js | 200 ++++++++++++ .../public/expressions/line_vega_spec_fn.ts | 299 ++++++++++++++++++ .../public/{ => expressions}/vega_fn.ts | 30 +- src/plugins/vis_type_vega/public/index.ts | 3 + src/plugins/vis_type_vega/public/plugin.ts | 4 +- .../public/vega_request_handler.ts | 2 +- .../public/vega_view/vega_base_view.js | 6 +- .../opensearch_dashboards.json | 2 +- src/plugins/vis_type_vislib/public/line.ts | 2 + .../public/line_to_expression.ts | 60 ++++ src/plugins/visualizations/public/index.ts | 6 +- .../public/legacy/build_pipeline.ts | 15 +- 13 files changed, 610 insertions(+), 21 deletions(-) create mode 100644 src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.test.js create mode 100644 src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.ts rename src/plugins/vis_type_vega/public/{ => expressions}/vega_fn.ts (83%) create mode 100644 src/plugins/vis_type_vislib/public/line_to_expression.ts diff --git a/src/plugins/vis_type_vega/public/components/vega_vis_editor.tsx b/src/plugins/vis_type_vega/public/components/vega_vis_editor.tsx index 86343ae10b0d..5fe1cbcfa6cd 100644 --- a/src/plugins/vis_type_vega/public/components/vega_vis_editor.tsx +++ b/src/plugins/vis_type_vega/public/components/vega_vis_editor.tsx @@ -37,7 +37,7 @@ import { i18n } from '@osd/i18n'; import { VisOptionsProps } from 'src/plugins/vis_default_editor/public'; import { getNotifications } from '../services'; -import { VisParams } from '../vega_fn'; +import { VisParams } from '../expressions/vega_fn'; import { VegaHelpMenu } from './vega_help_menu'; import { VegaActionsMenu } from './vega_actions_menu'; diff --git a/src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.test.js b/src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.test.js new file mode 100644 index 000000000000..2260c46b841a --- /dev/null +++ b/src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.test.js @@ -0,0 +1,200 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + buildLayerMark, + buildXAxis, + buildYAxis, + cleanString, + createSpecFromDatatable, + formatDataTable, + setupConfig, +} from './line_vega_spec_fn'; + +describe('cleanString()', function () { + it('string should not contain "', function () { + const dirtyString = '"someString"'; + expect(cleanString(dirtyString)).toBe('someString'); + }); +}); + +describe('setupConfig()', function () { + it('check all legend positions', function () { + const baseConfig = { + view: { + stroke: null, + }, + concat: { + spacing: 0, + }, + legend: { + orient: null, + }, + }; + const positions = ['top', 'right', 'left', 'bottom']; + positions.forEach((position) => { + const visParams = { legendPosition: position }; + baseConfig.legend.orient = position; + expect(setupConfig(visParams)).toStrictEqual(baseConfig); + }); + }); +}); + +describe('buildLayerMark()', function () { + const types = ['line', 'area', 'histogram']; + const interpolates = ['linear', 'cardinal', 'step-after']; + const strokeWidths = [-1, 0, 1, 2, 3, 4]; + const showCircles = [false, true]; + + it('check each mark possible value', function () { + const mark = { + type: null, + interpolate: null, + strokeWidth: null, + point: null, + }; + types.forEach((type) => { + mark.type = type; + interpolates.forEach((interpolate) => { + mark.interpolate = interpolate; + strokeWidths.forEach((strokeWidth) => { + mark.strokeWidth = strokeWidth; + showCircles.forEach((showCircle) => { + mark.point = showCircle; + const param = { + type: type, + interpolate: interpolate, + lineWidth: strokeWidth, + showCircles: showCircle, + }; + expect(buildLayerMark(param)).toStrictEqual(mark); + }); + }); + }); + }); + }); +}); + +describe('buildXAxis()', function () { + it('build different XAxis', function () { + const xAxisTitle = 'someTitle'; + const xAxisId = 'someId'; + const startTime = 1676596400; + const endTime = 1676796400; + [true, false].forEach((enableGrid) => { + const visParams = { grid: { categoryLines: enableGrid } }; + const vegaXAxis = { + axis: { + title: xAxisTitle, + grid: enableGrid, + }, + field: xAxisId, + type: 'temporal', + scale: { + domain: [startTime, endTime], + }, + }; + expect(buildXAxis(xAxisTitle, xAxisId, startTime, endTime, visParams)).toStrictEqual( + vegaXAxis + ); + }); + }); +}); + +describe('buildYAxis()', function () { + it('build different YAxis', function () { + const valueAxis = { + id: 'someId', + labels: { + rotate: 75, + show: false, + }, + position: 'left', + title: { + text: 'someText', + }, + }; + const column = { name: 'columnName', id: 'columnId' }; + const visParams = { grid: { valueAxis: true } }; + const vegaYAxis = { + axis: { + title: 'someText', + grid: true, + orient: 'left', + labels: false, + labelAngle: 75, + }, + field: 'columnId', + type: 'quantitative', + }; + expect(buildYAxis(column, valueAxis, visParams)).toStrictEqual(vegaYAxis); + + valueAxis.title.text = '""'; + vegaYAxis.axis.title = 'columnName'; + expect(buildYAxis(column, valueAxis, visParams)).toStrictEqual(vegaYAxis); + }); +}); + +describe('createSpecFromDatatable()', function () { + it('build simple line chart"', function () { + const datatable = + '{"type":"opensearch_dashboards_datatable","rows":[{"col-0-2":1672214400000,"col-1-1":44},{"col-0-2":1672300800000,"col-1-1":150},{"col-0-2":1672387200000,"col-1-1":154},{"col-0-2":1672473600000,"col-1-1":144},{"col-0-2":1672560000000,"col-1-1":133},{"col-0-2":1672646400000,"col-1-1":149},{"col-0-2":1672732800000,"col-1-1":152},{"col-0-2":1672819200000,"col-1-1":144},{"col-0-2":1672905600000,"col-1-1":166},{"col-0-2":1672992000000,"col-1-1":151},{"col-0-2":1673078400000,"col-1-1":143},{"col-0-2":1673164800000,"col-1-1":148},{"col-0-2":1673251200000,"col-1-1":146},{"col-0-2":1673337600000,"col-1-1":137},{"col-0-2":1673424000000,"col-1-1":152},{"col-0-2":1673510400000,"col-1-1":152},{"col-0-2":1673596800000,"col-1-1":151},{"col-0-2":1673683200000,"col-1-1":157},{"col-0-2":1673769600000,"col-1-1":151},{"col-0-2":1673856000000,"col-1-1":152},{"col-0-2":1673942400000,"col-1-1":142},{"col-0-2":1674028800000,"col-1-1":151},{"col-0-2":1674115200000,"col-1-1":163},{"col-0-2":1674201600000,"col-1-1":156},{"col-0-2":1674288000000,"col-1-1":153},{"col-0-2":1674374400000,"col-1-1":162},{"col-0-2":1674460800000,"col-1-1":152},{"col-0-2":1674547200000,"col-1-1":159},{"col-0-2":1674633600000,"col-1-1":165},{"col-0-2":1674720000000,"col-1-1":153},{"col-0-2":1674806400000,"col-1-1":149},{"col-0-2":1674892800000,"col-1-1":94}],"columns":[{"id":"col-0-2","name":"order_date per day","meta":{"type":"date_histogram","indexPatternId":"ff959d40-b880-11e8-a6d9-e546fe2bba5f","aggConfigParams":{"field":"order_date","timeRange":{"from":"now-90d","to":"now"},"useNormalizedOpenSearchInterval":true,"scaleMetricValues":false,"interval":"auto","drop_partials":false,"min_doc_count":1,"extended_bounds":{}}}},{"id":"col-1-1","name":"Count","meta":{"type":"count","indexPatternId":"ff959d40-b880-11e8-a6d9-e546fe2bba5f","aggConfigParams":{}}}]}'; + const visParams = + '{"addLegend":true,"addTimeMarker":false,"addTooltip":true,"categoryAxes":[{"id":"CategoryAxis-1","labels":{"filter":true,"show":true,"truncate":100},"position":"bottom","scale":{"type":"linear"},"show":true,"style":{},"title":{},"type":"category"}],"grid":{"categoryLines":false},"labels":{},"legendPosition":"right","seriesParams":[{"data":{"id":"1","label":"Count"},"drawLinesBetweenPoints":true,"interpolate":"linear","lineWidth":2,"mode":"normal","show":true,"showCircles":true,"type":"line","valueAxis":"ValueAxis-1"}],"thresholdLine":{"color":"#E7664C","show":false,"style":"full","value":10,"width":1},"times":[],"type":"line","valueAxes":[{"id":"ValueAxis-1","labels":{"filter":false,"rotate":0,"show":true,"truncate":100},"name":"LeftAxis-1","position":"left","scale":{"mode":"normal","type":"linear"},"show":true,"style":{},"title":{"text":"Count"},"type":"value"}]}'; + const dimensions = + '{"x":{"accessor":0,"format":{"id":"date","params":{"pattern":"YYYY-MM-DD"}},"params":{"date":true,"interval":"P1D","intervalESValue":1,"intervalESUnit":"d","format":"YYYY-MM-DD","bounds":{"min":"2022-11-18T00:14:09.617Z","max":"2023-02-16T00:14:09.617Z"}},"label":"order_date per day","aggType":"date_histogram"},"y":[{"accessor":1,"format":{"id":"number"},"params":{},"label":"Count","aggType":"count"}]}'; + const spec = + '{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"col-0-2":1672214400000,"col-1-1":44},{"col-0-2":1672300800000,"col-1-1":150},{"col-0-2":1672387200000,"col-1-1":154},{"col-0-2":1672473600000,"col-1-1":144},{"col-0-2":1672560000000,"col-1-1":133},{"col-0-2":1672646400000,"col-1-1":149},{"col-0-2":1672732800000,"col-1-1":152},{"col-0-2":1672819200000,"col-1-1":144},{"col-0-2":1672905600000,"col-1-1":166},{"col-0-2":1672992000000,"col-1-1":151},{"col-0-2":1673078400000,"col-1-1":143},{"col-0-2":1673164800000,"col-1-1":148},{"col-0-2":1673251200000,"col-1-1":146},{"col-0-2":1673337600000,"col-1-1":137},{"col-0-2":1673424000000,"col-1-1":152},{"col-0-2":1673510400000,"col-1-1":152},{"col-0-2":1673596800000,"col-1-1":151},{"col-0-2":1673683200000,"col-1-1":157},{"col-0-2":1673769600000,"col-1-1":151},{"col-0-2":1673856000000,"col-1-1":152},{"col-0-2":1673942400000,"col-1-1":142},{"col-0-2":1674028800000,"col-1-1":151},{"col-0-2":1674115200000,"col-1-1":163},{"col-0-2":1674201600000,"col-1-1":156},{"col-0-2":1674288000000,"col-1-1":153},{"col-0-2":1674374400000,"col-1-1":162},{"col-0-2":1674460800000,"col-1-1":152},{"col-0-2":1674547200000,"col-1-1":159},{"col-0-2":1674633600000,"col-1-1":165},{"col-0-2":1674720000000,"col-1-1":153},{"col-0-2":1674806400000,"col-1-1":149},{"col-0-2":1674892800000,"col-1-1":94}]},"config":{"view":{"stroke":null},"concat":{"spacing":0},"legend":{"orient":"right"}},"layer":[{"mark":{"type":"line","interpolate":"linear","strokeWidth":2,"point":true},"encoding":{"x":{"axis":{"title":"order_date per day","grid":false},"field":"col-0-2","type":"temporal","scale":{"domain":[1668730449617,1676506449617]}},"y":{"axis":{"title":"Count","orient":"left","labels":true,"labelAngle":0},"field":"col-1-1","type":"quantitative"},"tooltip":[{"field":"col-0-2","type":"temporal","title":"order_date per day"},{"field":"col-1-1","type":"quantitative","title":"Count"}],"color":{"datum":"Count"}}}]}'; + expect( + JSON.stringify( + createSpecFromDatatable( + formatDataTable(JSON.parse(datatable)), + JSON.parse(visParams), + JSON.parse(dimensions) + ) + ) + ).toBe(spec); + }); + + it('build empty chart if no x-axis is defined"', function () { + const datatable = + '{"type":"opensearch_dashboards_datatable","rows":[{"col-0-1":4675}],"columns":[{"id":"col-0-1","name":"Count","meta":{"type":"count","indexPatternId":"ff959d40-b880-11e8-a6d9-e546fe2bba5f","aggConfigParams":{}}}]}'; + const visParams = + '{"type":"line","grid":{"categoryLines":false},"categoryAxes":[{"id":"CategoryAxis-1","type":"category","position":"bottom","show":true,"style":{},"scale":{"type":"linear"},"labels":{"show":true,"filter":true,"truncate":100},"title":{}}],"valueAxes":[{"id":"ValueAxis-1","name":"LeftAxis-1","type":"value","position":"left","show":true,"style":{},"scale":{"type":"linear","mode":"normal"},"labels":{"show":true,"rotate":0,"filter":false,"truncate":100},"title":{"text":"Count"}}],"seriesParams":[{"show":true,"type":"line","mode":"normal","data":{"label":"Count","id":"1"},"valueAxis":"ValueAxis-1","drawLinesBetweenPoints":true,"lineWidth":2,"interpolate":"linear","showCircles":true}],"addTooltip":true,"addLegend":true,"legendPosition":"right","times":[],"addTimeMarker":false,"labels":{},"thresholdLine":{"show":false,"value":10,"width":1,"style":"full","color":"#E7664C"}}'; + const dimensions = + '{"x":null,"y":[{"accessor":0,"format":{"id":"number"},"params":{},"label":"Count","aggType":"count"}]}'; + const spec = + '{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"col-0-1":4675}]},"config":{"view":{"stroke":null},"concat":{"spacing":0},"legend":{"orient":"right"}},"layer":[]}'; + expect( + JSON.stringify( + createSpecFromDatatable( + formatDataTable(JSON.parse(datatable)), + JSON.parse(visParams), + JSON.parse(dimensions) + ) + ) + ).toBe(spec); + }); + + it('build complicated line chart"', function () { + const datatable = + '{"type":"opensearch_dashboards_datatable","rows":[{"col-0-2":1672214400000,"col-1-1":44,"col-2-3":60.9375},{"col-0-2":1672300800000,"col-1-1":150,"col-2-3":82.5},{"col-0-2":1672387200000,"col-1-1":154,"col-2-3":79.5},{"col-0-2":1672473600000,"col-1-1":144,"col-2-3":75.875},{"col-0-2":1672560000000,"col-1-1":133,"col-2-3":259.25},{"col-0-2":1672646400000,"col-1-1":149,"col-2-3":90},{"col-0-2":1672732800000,"col-1-1":152,"col-2-3":79.0625},{"col-0-2":1672819200000,"col-1-1":144,"col-2-3":82.5},{"col-0-2":1672905600000,"col-1-1":166,"col-2-3":85.25},{"col-0-2":1672992000000,"col-1-1":151,"col-2-3":92},{"col-0-2":1673078400000,"col-1-1":143,"col-2-3":90.75},{"col-0-2":1673164800000,"col-1-1":148,"col-2-3":92},{"col-0-2":1673251200000,"col-1-1":146,"col-2-3":83.25},{"col-0-2":1673337600000,"col-1-1":137,"col-2-3":98},{"col-0-2":1673424000000,"col-1-1":152,"col-2-3":83.6875},{"col-0-2":1673510400000,"col-1-1":152,"col-2-3":83.6875},{"col-0-2":1673596800000,"col-1-1":151,"col-2-3":87.4375},{"col-0-2":1673683200000,"col-1-1":157,"col-2-3":63.75},{"col-0-2":1673769600000,"col-1-1":151,"col-2-3":81.5625},{"col-0-2":1673856000000,"col-1-1":152,"col-2-3":100.6875},{"col-0-2":1673942400000,"col-1-1":142,"col-2-3":98},{"col-0-2":1674028800000,"col-1-1":151,"col-2-3":100.8125},{"col-0-2":1674115200000,"col-1-1":163,"col-2-3":83.6875},{"col-0-2":1674201600000,"col-1-1":156,"col-2-3":85.8125},{"col-0-2":1674288000000,"col-1-1":153,"col-2-3":98},{"col-0-2":1674374400000,"col-1-1":162,"col-2-3":75.9375},{"col-0-2":1674460800000,"col-1-1":152,"col-2-3":113.375},{"col-0-2":1674547200000,"col-1-1":159,"col-2-3":73.625},{"col-0-2":1674633600000,"col-1-1":165,"col-2-3":72.8125},{"col-0-2":1674720000000,"col-1-1":153,"col-2-3":113.375},{"col-0-2":1674806400000,"col-1-1":149,"col-2-3":82.5},{"col-0-2":1674892800000,"col-1-1":94,"col-2-3":54}],"columns":[{"id":"col-0-2","name":"order_date per day","meta":{"type":"date_histogram","indexPatternId":"ff959d40-b880-11e8-a6d9-e546fe2bba5f","aggConfigParams":{"field":"order_date","timeRange":{"from":"now-90d","to":"now"},"useNormalizedOpenSearchInterval":true,"scaleMetricValues":false,"interval":"auto","drop_partials":false,"min_doc_count":1,"extended_bounds":{}}}},{"id":"col-1-1","name":"Count","meta":{"type":"count","indexPatternId":"ff959d40-b880-11e8-a6d9-e546fe2bba5f","aggConfigParams":{}}},{"id":"col-2-3","name":"Max products.min_price","meta":{"type":"max","indexPatternId":"ff959d40-b880-11e8-a6d9-e546fe2bba5f","aggConfigParams":{"field":"products.min_price"}}}]}'; + const visParams = + '{"addLegend":true,"addTimeMarker":true,"addTooltip":true,"categoryAxes":[{"id":"CategoryAxis-1","labels":{"filter":true,"show":true,"truncate":100},"position":"bottom","scale":{"type":"linear"},"show":true,"style":{},"title":{},"type":"category"}],"grid":{"categoryLines":false,"valueAxis":"ValueAxis-1"},"labels":{},"legendPosition":"bottom","seriesParams":[{"data":{"id":"1","label":"Count"},"drawLinesBetweenPoints":true,"interpolate":"linear","lineWidth":2,"mode":"normal","show":true,"showCircles":true,"type":"line","valueAxis":"ValueAxis-1"},{"data":{"id":"3","label":"Max products.min_price"},"drawLinesBetweenPoints":true,"interpolate":"linear","lineWidth":2,"mode":"normal","show":true,"showCircles":true,"type":"line","valueAxis":"ValueAxis-1"}],"thresholdLine":{"color":"#E7664C","show":true,"style":"dashed","value":100,"width":1},"times":[],"type":"line","valueAxes":[{"id":"ValueAxis-1","labels":{"filter":false,"rotate":75,"show":true,"truncate":100},"name":"RightAxis-1","position":"right","scale":{"mode":"normal","type":"linear"},"show":true,"style":{},"title":{"text":"Count"},"type":"value"}]}'; + const dimensions = + '{"x":{"accessor":0,"format":{"id":"date","params":{"pattern":"YYYY-MM-DD"}},"params":{"date":true,"interval":"P1D","intervalESValue":1,"intervalESUnit":"d","format":"YYYY-MM-DD","bounds":{"min":"2022-11-19T03:26:04.730Z","max":"2023-02-17T03:26:04.730Z"}},"label":"order_date per day","aggType":"date_histogram"},"y":[{"accessor":1,"format":{"id":"number"},"params":{},"label":"Count","aggType":"count"},{"accessor":2,"format":{"id":"number","params":{"parsedUrl":{"origin":"http://localhost:5603","pathname":"/rao/app/visualize","basePath":"/rao"}}},"params":{},"label":"Max products.min_price","aggType":"max"}]}'; + const spec = + '{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"col-0-2":1672214400000,"col-1-1":44,"col-2-3":60.9375},{"col-0-2":1672300800000,"col-1-1":150,"col-2-3":82.5},{"col-0-2":1672387200000,"col-1-1":154,"col-2-3":79.5},{"col-0-2":1672473600000,"col-1-1":144,"col-2-3":75.875},{"col-0-2":1672560000000,"col-1-1":133,"col-2-3":259.25},{"col-0-2":1672646400000,"col-1-1":149,"col-2-3":90},{"col-0-2":1672732800000,"col-1-1":152,"col-2-3":79.0625},{"col-0-2":1672819200000,"col-1-1":144,"col-2-3":82.5},{"col-0-2":1672905600000,"col-1-1":166,"col-2-3":85.25},{"col-0-2":1672992000000,"col-1-1":151,"col-2-3":92},{"col-0-2":1673078400000,"col-1-1":143,"col-2-3":90.75},{"col-0-2":1673164800000,"col-1-1":148,"col-2-3":92},{"col-0-2":1673251200000,"col-1-1":146,"col-2-3":83.25},{"col-0-2":1673337600000,"col-1-1":137,"col-2-3":98},{"col-0-2":1673424000000,"col-1-1":152,"col-2-3":83.6875},{"col-0-2":1673510400000,"col-1-1":152,"col-2-3":83.6875},{"col-0-2":1673596800000,"col-1-1":151,"col-2-3":87.4375},{"col-0-2":1673683200000,"col-1-1":157,"col-2-3":63.75},{"col-0-2":1673769600000,"col-1-1":151,"col-2-3":81.5625},{"col-0-2":1673856000000,"col-1-1":152,"col-2-3":100.6875},{"col-0-2":1673942400000,"col-1-1":142,"col-2-3":98},{"col-0-2":1674028800000,"col-1-1":151,"col-2-3":100.8125},{"col-0-2":1674115200000,"col-1-1":163,"col-2-3":83.6875},{"col-0-2":1674201600000,"col-1-1":156,"col-2-3":85.8125},{"col-0-2":1674288000000,"col-1-1":153,"col-2-3":98},{"col-0-2":1674374400000,"col-1-1":162,"col-2-3":75.9375},{"col-0-2":1674460800000,"col-1-1":152,"col-2-3":113.375},{"col-0-2":1674547200000,"col-1-1":159,"col-2-3":73.625},{"col-0-2":1674633600000,"col-1-1":165,"col-2-3":72.8125},{"col-0-2":1674720000000,"col-1-1":153,"col-2-3":113.375},{"col-0-2":1674806400000,"col-1-1":149,"col-2-3":82.5},{"col-0-2":1674892800000,"col-1-1":94,"col-2-3":54}]},"config":{"view":{"stroke":null},"concat":{"spacing":0},"legend":{"orient":"bottom"}},"layer":[{"mark":{"type":"line","interpolate":"linear","strokeWidth":2,"point":true},"encoding":{"x":{"axis":{"title":"order_date per day","grid":false},"field":"col-0-2","type":"temporal","scale":{"domain":[1668828364730,1676604364730]}},"y":{"axis":{"title":"Count","grid":"ValueAxis-1","orient":"right","labels":true,"labelAngle":75},"field":"col-1-1","type":"quantitative"},"tooltip":[{"field":"col-0-2","type":"temporal","title":"order_date per day"},{"field":"col-1-1","type":"quantitative","title":"Count"}],"color":{"datum":"Count"}}},{"mark":{"type":"line","interpolate":"linear","strokeWidth":2,"point":true},"encoding":{"x":{"axis":{"title":"order_date per day","grid":false},"field":"col-0-2","type":"temporal","scale":{"domain":[1668828364730,1676604364730]}},"y":{"axis":{"title":"Count","grid":"ValueAxis-1","orient":"right","labels":true,"labelAngle":75},"field":"col-2-3","type":"quantitative"},"tooltip":[{"field":"col-0-2","type":"temporal","title":"order_date per day"},{"field":"col-2-3","type":"quantitative","title":"Max products.min_price"}],"color":{"datum":"Max products.min_price"}}},{"mark":"rule","encoding":{"x":{"type":"temporal","field":"now_field"},"color":{"value":"red"},"size":{"value":1}}},{"mark":{"type":"rule","color":"#E7664C","strokeDash":[8,8]},"encoding":{"y":{"datum":100}}}],"transform":[{"calculate":"now()","as":"now_field"}]}'; + expect( + JSON.stringify( + createSpecFromDatatable( + formatDataTable(JSON.parse(datatable)), + JSON.parse(visParams), + JSON.parse(dimensions) + ) + ) + ).toBe(spec); + }); +}); diff --git a/src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.ts b/src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.ts new file mode 100644 index 000000000000..b0f440e59647 --- /dev/null +++ b/src/plugins/vis_type_vega/public/expressions/line_vega_spec_fn.ts @@ -0,0 +1,299 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { cloneDeep } from 'lodash'; +import { i18n } from '@osd/i18n'; +import { + ExpressionFunctionDefinition, + OpenSearchDashboardsDatatable, + OpenSearchDashboardsDatatableColumn, +} from '../../../expressions/public'; +import { VegaVisualizationDependencies } from '../plugin'; +import { VislibDimensions, VisParams } from '../../../visualizations/public'; + +type Input = OpenSearchDashboardsDatatable; +type Output = Promise; + +interface Arguments { + visLayers: string | null; + visParams: string; + dimensions: string; +} + +export type LineVegaSpecExpressionFunctionDefinition = ExpressionFunctionDefinition< + 'line_vega_spec', + Input, + Arguments, + Output +>; + +// TODO: move this to the visualization plugin that has VisParams once all of these parameters have been better defined +interface ValueAxis { + id: string; + labels: { + filter: boolean; + rotate: number; + show: boolean; + truncate: number; + }; + name: string; + position: string; + scale: { + mode: string; + type: string; + }; + show: true; + style: any; + title: { + text: string; + }; + type: string; +} + +// Get the first xaxis field as only 1 setup of X Axis will be supported and +// there won't be support for split series and split chart +const getXAxisId = (dimensions: any, columns: OpenSearchDashboardsDatatableColumn[]): string => { + return columns.filter((column) => column.name === dimensions.x.label)[0].id; +}; + +export const cleanString = (rawString: string): string => { + return rawString.replaceAll('"', ''); +}; + +export const formatDataTable = ( + datatable: OpenSearchDashboardsDatatable +): OpenSearchDashboardsDatatable => { + datatable.columns.forEach((column) => { + // clean quotation marks from names in columns + column.name = cleanString(column.name); + }); + return datatable; +}; + +export const setupConfig = (visParams: VisParams) => { + const legendPosition = visParams.legendPosition; + return { + view: { + stroke: null, + }, + concat: { + spacing: 0, + }, + legend: { + orient: legendPosition, + }, + }; +}; + +export const buildLayerMark = (seriesParams: { + type: string; + interpolate: string; + lineWidth: number; + showCircles: boolean; +}) => { + return { + // Possible types are: line, area, histogram. The eligibility checker will + // prevent area and histogram (though area works in vega-lite) + type: seriesParams.type, + // Possible types: linear, cardinal, step-after. All of these types work in vega-lite + interpolate: seriesParams.interpolate, + // The possible values is any number, which matches what vega-lite supports + strokeWidth: seriesParams.lineWidth, + // this corresponds to showing the dots in the visbuilder for each data point + point: seriesParams.showCircles, + }; +}; + +export const buildXAxis = ( + xAxisTitle: string, + xAxisId: string, + startTime: number, + endTime: number, + visParams: VisParams +) => { + return { + axis: { + title: xAxisTitle, + grid: visParams.grid.categoryLines, + }, + field: xAxisId, + // Right now, the line charts can only set the x-axis value to be a date attribute, so + // this should always be of type temporal + type: 'temporal', + scale: { + domain: [startTime, endTime], + }, + }; +}; + +export const buildYAxis = ( + column: OpenSearchDashboardsDatatableColumn, + valueAxis: ValueAxis, + visParams: VisParams +) => { + return { + axis: { + title: cleanString(valueAxis.title.text) || column.name, + grid: visParams.grid.valueAxis, + orient: valueAxis.position, + labels: valueAxis.labels.show, + labelAngle: valueAxis.labels.rotate, + }, + field: column.id, + type: 'quantitative', + }; +}; + +export const createSpecFromDatatable = ( + datatable: OpenSearchDashboardsDatatable, + visParams: VisParams, + dimensions: VislibDimensions +): object => { + // TODO: we can try to use VegaSpec type but it is currently very outdated, where many + // of the fields and sub-fields don't have other optional params that we want for customizing. + // For now, we make this more loosely-typed by just specifying it as a generic object. + const spec = {} as any; + + spec.$schema = 'https://vega.github.io/schema/vega-lite/v5.json'; + spec.data = { + values: datatable.rows, + }; + spec.config = setupConfig(visParams); + + // Get the valueAxes data and generate a map to easily fetch the different valueAxes data + const valueAxis = new Map(); + visParams?.valueAxes?.forEach((yAxis: ValueAxis) => { + valueAxis.set(yAxis.id, yAxis); + }); + + spec.layer = [] as any[]; + + if (datatable.rows.length > 0 && dimensions.x !== null) { + const xAxisId = getXAxisId(dimensions, datatable.columns); + const xAxisTitle = cleanString(dimensions.x.label); + // get x-axis bounds for the chart + const startTime = new Date(dimensions.x.params.bounds.min).valueOf(); + const endTime = new Date(dimensions.x.params.bounds.max).valueOf(); + let skip = 0; + datatable.columns.forEach((column, index) => { + // Check if it's not xAxis column data + if (column.meta?.aggConfigParams?.interval !== undefined) { + skip++; + } else { + const currentSeriesParams = visParams.seriesParams[index - skip]; + const currentValueAxis = valueAxis.get(currentSeriesParams.valueAxis.toString()); + let tooltip: Array<{ field: string; type: string; title: string }> = []; + if (visParams.addTooltip) { + tooltip = [ + { field: xAxisId, type: 'temporal', title: xAxisTitle }, + { field: column.id, type: 'quantitative', title: column.name }, + ]; + } + spec.layer.push({ + mark: buildLayerMark(currentSeriesParams), + encoding: { + x: buildXAxis(xAxisTitle, xAxisId, startTime, endTime, visParams), + y: buildYAxis(column, currentValueAxis, visParams), + tooltip, + color: { + // This ensures all the different metrics have their own distinct and unique color + datum: column.name, + }, + }, + }); + } + }); + } + + if (visParams.addTimeMarker) { + spec.transform = [ + { + calculate: 'now()', + as: 'now_field', + }, + ]; + + spec.layer.push({ + mark: 'rule', + encoding: { + x: { + type: 'temporal', + field: 'now_field', + }, + // The time marker on vislib is red, so keeping this consistent + color: { + value: 'red', + }, + size: { + value: 1, + }, + }, + }); + } + + if (visParams.thresholdLine.show as boolean) { + const layer = { + mark: { + type: 'rule', + color: visParams.thresholdLine.color, + strokeDash: [1, 0], + }, + encoding: { + y: { + datum: visParams.thresholdLine.value, + }, + }, + }; + + // Can only support making a threshold line with full or dashed style, but not dot-dashed + // due to vega-lite limitations + if (visParams.thresholdLine.style !== 'full') { + layer.mark.strokeDash = [8, 8]; + } + + spec.layer.push(layer); + } + + return spec; +}; + +export const createLineVegaSpecFn = ( + dependencies: VegaVisualizationDependencies +): LineVegaSpecExpressionFunctionDefinition => ({ + name: 'line_vega_spec', + type: 'string', + inputTypes: ['opensearch_dashboards_datatable'], + help: i18n.translate('visTypeVega.function.help', { + defaultMessage: 'Construct line vega spec', + }), + args: { + visLayers: { + types: ['string', 'null'], + default: '', + help: '', + }, + visParams: { + types: ['string'], + default: '""', + help: '', + }, + dimensions: { + types: ['string'], + default: '""', + help: '', + }, + }, + async fn(input, args, context) { + const table = cloneDeep(input); + + // creating initial vega spec from table + const spec = createSpecFromDatatable( + formatDataTable(table), + JSON.parse(args.visParams), + JSON.parse(args.dimensions) + ); + return JSON.stringify(spec); + }, +}); diff --git a/src/plugins/vis_type_vega/public/vega_fn.ts b/src/plugins/vis_type_vega/public/expressions/vega_fn.ts similarity index 83% rename from src/plugins/vis_type_vega/public/vega_fn.ts rename to src/plugins/vis_type_vega/public/expressions/vega_fn.ts index abe2d3665ed3..f5ab178cbd74 100644 --- a/src/plugins/vis_type_vega/public/vega_fn.ts +++ b/src/plugins/vis_type_vega/public/expressions/vega_fn.ts @@ -35,13 +35,13 @@ import { ExpressionFunctionDefinition, OpenSearchDashboardsContext, Render, -} from '../../expressions/public'; -import { VegaVisualizationDependencies } from './plugin'; -import { createVegaRequestHandler } from './vega_request_handler'; -import { VegaInspectorAdapters } from './vega_inspector/index'; -import { TimeRange, Query } from '../../data/public'; -import { VisRenderValue } from '../../visualizations/public'; -import { VegaParser } from './data_model/vega_parser'; +} from '../../../expressions/public'; +import { VegaVisualizationDependencies } from '../plugin'; +import { createVegaRequestHandler } from '../vega_request_handler'; +import { VegaInspectorAdapters } from '../vega_inspector'; +import { TimeRange, Query } from '../../../data/public'; +import { VisRenderValue } from '../../../visualizations/public'; +import { VegaParser } from '../data_model/vega_parser'; type Input = OpenSearchDashboardsContext | null; type Output = Promise>; @@ -52,6 +52,14 @@ interface Arguments { export type VisParams = Required; +export type VegaExpressionFunctionDefinition = ExpressionFunctionDefinition< + 'vega', + Input, + Arguments, + Output, + ExecutionContext +>; + interface RenderValue extends VisRenderValue { visData: VegaParser; visType: 'vega'; @@ -60,13 +68,7 @@ interface RenderValue extends VisRenderValue { export const createVegaFn = ( dependencies: VegaVisualizationDependencies -): ExpressionFunctionDefinition< - 'vega', - Input, - Arguments, - Output, - ExecutionContext -> => ({ +): VegaExpressionFunctionDefinition => ({ name: 'vega', type: 'render', inputTypes: ['opensearch_dashboards_context', 'null'], diff --git a/src/plugins/vis_type_vega/public/index.ts b/src/plugins/vis_type_vega/public/index.ts index c41add63d681..ed0c794837dd 100644 --- a/src/plugins/vis_type_vega/public/index.ts +++ b/src/plugins/vis_type_vega/public/index.ts @@ -35,3 +35,6 @@ import { VegaPlugin as Plugin } from './plugin'; export function plugin(initializerContext: PluginInitializerContext) { return new Plugin(initializerContext); } + +export { VegaExpressionFunctionDefinition } from './expressions/vega_fn'; +export { LineVegaSpecExpressionFunctionDefinition } from './expressions/line_vega_spec_fn'; diff --git a/src/plugins/vis_type_vega/public/plugin.ts b/src/plugins/vis_type_vega/public/plugin.ts index 64ab81eedfd2..9751a73ccf91 100644 --- a/src/plugins/vis_type_vega/public/plugin.ts +++ b/src/plugins/vis_type_vega/public/plugin.ts @@ -43,13 +43,14 @@ import { setInjectedMetadata, } from './services'; -import { createVegaFn } from './vega_fn'; +import { createVegaFn } from './expressions/vega_fn'; import { createVegaTypeDefinition } from './vega_type'; import { IServiceSettings } from '../../maps_legacy/public'; import './index.scss'; import { ConfigSchema } from '../config'; import { getVegaInspectorView } from './vega_inspector'; +import { createLineVegaSpecFn } from './expressions/line_vega_spec_fn'; /** @internal */ export interface VegaVisualizationDependencies { @@ -104,6 +105,7 @@ export class VegaPlugin implements Plugin, void> { inspector.registerView(getVegaInspectorView({ uiSettings: core.uiSettings })); expressions.registerFunction(() => createVegaFn(visualizationDependencies)); + expressions.registerFunction(() => createLineVegaSpecFn(visualizationDependencies)); visualizations.createBaseVisualization(createVegaTypeDefinition(visualizationDependencies)); } diff --git a/src/plugins/vis_type_vega/public/vega_request_handler.ts b/src/plugins/vis_type_vega/public/vega_request_handler.ts index 7878e97d5889..7711f5d0f497 100644 --- a/src/plugins/vis_type_vega/public/vega_request_handler.ts +++ b/src/plugins/vis_type_vega/public/vega_request_handler.ts @@ -34,7 +34,7 @@ import { SearchAPI } from './data_model/search_api'; import { TimeCache } from './data_model/time_cache'; import { VegaVisualizationDependencies } from './plugin'; -import { VisParams } from './vega_fn'; +import { VisParams } from './expressions/vega_fn'; import { getData, getInjectedMetadata } from './services'; import { VegaInspectorAdapters } from './vega_inspector'; diff --git a/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js b/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js index 1286495af901..32dcfe2c026a 100644 --- a/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js +++ b/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js @@ -437,7 +437,11 @@ export class VegaBaseView { * Set global debug variable to simplify vega debugging in console. Show info message first time */ setDebugValues(view, spec, vlspec) { - this._parser.searchAPI.inspectorAdapters?.vega.bindInspectValues({ + // The vega inspector can now be null when rendering line charts using vega for the overlay visualization feature. + // This is because the inspectors get added at bootstrap to the different chart types and visualize embeddable + // thinks the line chart is vislib line chart and uses that inspector adapter and has no way of knowing it's + // actually a vega-lite chart and needs to use the vega inspector adapter without hacky code. + this._parser.searchAPI.inspectorAdapters?.vega?.bindInspectValues({ view, spec: vlspec || spec, }); diff --git a/src/plugins/vis_type_vislib/opensearch_dashboards.json b/src/plugins/vis_type_vislib/opensearch_dashboards.json index b0d9627bb10f..0cef84e463f6 100644 --- a/src/plugins/vis_type_vislib/opensearch_dashboards.json +++ b/src/plugins/vis_type_vislib/opensearch_dashboards.json @@ -3,7 +3,7 @@ "version": "opensearchDashboards", "server": true, "ui": true, - "requiredPlugins": ["charts", "data", "expressions", "visualizations", "opensearchDashboardsLegacy"], + "requiredPlugins": ["charts", "data", "expressions", "visualizations", "opensearchDashboardsLegacy", "visTypeVega"], "optionalPlugins": ["visTypeXy"], "requiredBundles": ["opensearchDashboardsUtils", "visDefaultEditor"] } diff --git a/src/plugins/vis_type_vislib/public/line.ts b/src/plugins/vis_type_vislib/public/line.ts index 06a5be4fe414..04ae732e2903 100644 --- a/src/plugins/vis_type_vislib/public/line.ts +++ b/src/plugins/vis_type_vislib/public/line.ts @@ -50,6 +50,7 @@ import { createVislibVisController } from './vis_controller'; import { VisTypeVislibDependencies } from './plugin'; import { Rotates } from '../../charts/public'; import { VIS_EVENT_TO_TRIGGER } from '../../visualizations/public'; +import { toExpressionAst } from './line_to_expression'; export const createLineVisTypeDefinition = (deps: VisTypeVislibDependencies) => ({ name: 'line', @@ -58,6 +59,7 @@ export const createLineVisTypeDefinition = (deps: VisTypeVislibDependencies) => description: i18n.translate('visTypeVislib.line.lineDescription', { defaultMessage: 'Emphasize trends', }), + toExpressionAst, visualization: createVislibVisController(deps), getSupportedTriggers: () => { return [VIS_EVENT_TO_TRIGGER.filter, VIS_EVENT_TO_TRIGGER.brush]; diff --git a/src/plugins/vis_type_vislib/public/line_to_expression.ts b/src/plugins/vis_type_vislib/public/line_to_expression.ts new file mode 100644 index 000000000000..26797d8348a0 --- /dev/null +++ b/src/plugins/vis_type_vislib/public/line_to_expression.ts @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { buildVislibDimensions, Vis } from '../../visualizations/public'; +import { buildExpression, buildExpressionFunction } from '../../expressions/public'; +import { OpenSearchaggsExpressionFunctionDefinition } from '../../data/common/search/expressions'; +import { + VegaExpressionFunctionDefinition, + LineVegaSpecExpressionFunctionDefinition, +} from '../../vis_type_vega/public'; + +export const toExpressionAst = async (vis: Vis, params: any) => { + // Construct the existing expr fns that are ran for vislib line chart, up until the render fn. + // That way we get the exact same data table of results as if it was a vislib chart. + const opensearchaggsFn = buildExpressionFunction( + 'opensearchaggs', + { + index: vis.data.indexPattern!.id!, + metricsAtAllLevels: vis.isHierarchical(), + partialRows: vis.params.showPartialRows || false, + aggConfigs: JSON.stringify(vis.data.aggs!.aggs), + includeFormatHints: false, + } + ); + + // Checks if there are vislayers to overlay. If not, default to the vislib implementation. + if (params.visLayers == null || Object.keys(params.visLayers).length === 0) { + // This wont work but is needed so then it will default to the original vis lib renderer + const dimensions = await buildVislibDimensions(vis, params); + const visConfig = { ...vis.params, dimensions }; + const vislib = buildExpressionFunction('vislib', { + type: 'line', + visConfig: JSON.stringify(visConfig), + }); + const ast = buildExpression([opensearchaggsFn, vislib]); + return ast.toAst(); + } else { + const dimensions = await buildVislibDimensions(vis, params); + // adding the new expr fn here that takes the datatable and converts to a vega spec + const vegaSpecFn = buildExpressionFunction( + 'line_vega_spec', + { + visLayers: JSON.stringify([]), + visParams: JSON.stringify(vis.params), + dimensions: JSON.stringify(dimensions), + } + ); + const vegaSpecFnExpressionBuilder = buildExpression([vegaSpecFn]); + + // build vega expr fn. use nested expression fn syntax to first construct the + // spec via 'line_vega_spec' fn, then set as the arg for the final 'vega' fn + const vegaFn = buildExpressionFunction('vega', { + spec: vegaSpecFnExpressionBuilder, + }); + const ast = buildExpression([opensearchaggsFn, vegaFn]); + return ast.toAst(); + } +}; diff --git a/src/plugins/visualizations/public/index.ts b/src/plugins/visualizations/public/index.ts index 46d3b3dd7d03..ffc24a81b381 100644 --- a/src/plugins/visualizations/public/index.ts +++ b/src/plugins/visualizations/public/index.ts @@ -43,7 +43,11 @@ export { Vis } from './vis'; export { TypesService } from './vis_types/types_service'; export { VISUALIZE_EMBEDDABLE_TYPE, VIS_EVENT_TO_TRIGGER } from './embeddable'; export { VisualizationContainer, VisualizationNoResults } from './components'; -export { getSchemas as getVisSchemas, buildVislibDimensions } from './legacy/build_pipeline'; +export { + getSchemas as getVisSchemas, + buildVislibDimensions, + VislibDimensions, +} from './legacy/build_pipeline'; /** @public types */ export { VisualizationsSetup, VisualizationsStart }; diff --git a/src/plugins/visualizations/public/legacy/build_pipeline.ts b/src/plugins/visualizations/public/legacy/build_pipeline.ts index 1cbb3bc38879..a2b95afe6756 100644 --- a/src/plugins/visualizations/public/legacy/build_pipeline.ts +++ b/src/plugins/visualizations/public/legacy/build_pipeline.ts @@ -331,7 +331,20 @@ const buildVisConfig: BuildVisConfigFunction = { }, }; -export const buildVislibDimensions = async (vis: any, params: BuildPipelineParams) => { +export interface VislibDimensions { + x: any; + y: SchemaConfig[]; + z?: any[]; + width?: any[]; + series?: any[]; + splitRow?: any[]; + splitColumn?: any[]; +} + +export const buildVislibDimensions = async ( + vis: any, + params: BuildPipelineParams +): Promise => { const schemas = getSchemas(vis, { timeRange: params.timeRange, timefilter: params.timefilter,