diff --git a/changelogs/fragments/7285.yml b/changelogs/fragments/7285.yml new file mode 100644 index 000000000000..4624cbb6eef1 --- /dev/null +++ b/changelogs/fragments/7285.yml @@ -0,0 +1,2 @@ +feat: +- Support PPL in vega visualization ([#7285](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7285)) \ No newline at end of file diff --git a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts index d9bd7721d6cb..9bb01c4c6023 100644 --- a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts +++ b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts @@ -94,7 +94,7 @@ export function getSearchParamsFromRequest( } return { - index: searchRequest.index.title || searchRequest.index, + index: searchRequest.index?.title || searchRequest.index, body: searchRequest.body, ...searchParams, }; diff --git a/src/plugins/data/server/search/routes/search.test.ts b/src/plugins/data/server/search/routes/search.test.ts index 12afc8a577f9..d31d443dcef8 100644 --- a/src/plugins/data/server/search/routes/search.test.ts +++ b/src/plugins/data/server/search/routes/search.test.ts @@ -94,7 +94,10 @@ describe('Search service', () => { await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); expect(mockDataStart.search.search).toBeCalled(); - expect(mockDataStart.search.search.mock.calls[0][1]).toStrictEqual(mockBody); + expect(mockDataStart.search.search.mock.calls[0][1]).toStrictEqual({ + ...mockBody, + rawRequest: mockRequest, + }); expect(mockResponse.ok).toBeCalled(); expect(mockResponse.ok.mock.calls[0][0]).toEqual({ body: response, @@ -125,7 +128,10 @@ describe('Search service', () => { await handler((mockContext as unknown) as RequestHandlerContext, mockRequest, mockResponse); expect(mockDataStart.search.search).toBeCalled(); - expect(mockDataStart.search.search.mock.calls[0][1]).toStrictEqual(mockBody); + expect(mockDataStart.search.search.mock.calls[0][1]).toStrictEqual({ + ...mockBody, + rawRequest: mockRequest, + }); expect(mockResponse.customError).toBeCalled(); const error: any = mockResponse.customError.mock.calls[0][0]; expect(error.body.message).toBe('oh no'); diff --git a/src/plugins/data/server/search/routes/search.ts b/src/plugins/data/server/search/routes/search.ts index d569de539f39..070d392abf63 100644 --- a/src/plugins/data/server/search/routes/search.ts +++ b/src/plugins/data/server/search/routes/search.ts @@ -62,7 +62,7 @@ export function registerSearchRoute( try { const { withLongNumeralsSupport, ...response } = await selfStart.search.search( context, - { ...searchRequest, id }, + { ...searchRequest, id, rawRequest: request }, { abortSignal, strategy, diff --git a/src/plugins/data/server/search/types.ts b/src/plugins/data/server/search/types.ts index 6927d1289673..c50006023727 100644 --- a/src/plugins/data/server/search/types.ts +++ b/src/plugins/data/server/search/types.ts @@ -72,7 +72,9 @@ export interface ISearchSetup { } export interface ISearchStart< - SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, + SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest & { + rawRequest?: OpenSearchDashboardsRequest; + }, SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse > { aggs: AggsStart; diff --git a/src/plugins/query_enhancements/common/constants.ts b/src/plugins/query_enhancements/common/constants.ts index 7e82677407de..358d41745e8e 100644 --- a/src/plugins/query_enhancements/common/constants.ts +++ b/src/plugins/query_enhancements/common/constants.ts @@ -10,6 +10,7 @@ export const BASE_API = '/api/enhancements'; export const SEARCH_STRATEGY = { PPL: 'ppl', + PPL_RAW: 'pplraw', SQL: 'sql', SQL_ASYNC: 'sqlasync', }; diff --git a/src/plugins/query_enhancements/server/plugin.ts b/src/plugins/query_enhancements/server/plugin.ts index db105849694e..6c5357af70f1 100644 --- a/src/plugins/query_enhancements/server/plugin.ts +++ b/src/plugins/query_enhancements/server/plugin.ts @@ -25,8 +25,10 @@ import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, QueryEnhancementsPluginStart, + QueryEnhancementsPluginStartDependencies, } from './types'; import { OpenSearchObservabilityPlugin, OpenSearchPPLPlugin } from './utils'; +import { pplRawSearchStrategyProvider } from './search/ppl_raw_search_strategy'; export class QueryEnhancementsPlugin implements Plugin { @@ -51,6 +53,7 @@ export class QueryEnhancementsPlugin } const pplSearchStrategy = pplSearchStrategyProvider(this.config$, this.logger, client); + const pplRawSearchStrategy = pplRawSearchStrategyProvider(this.config$, this.logger, client); const sqlSearchStrategy = sqlSearchStrategyProvider(this.config$, this.logger, client); const sqlAsyncSearchStrategy = sqlAsyncSearchStrategyProvider( this.config$, @@ -59,6 +62,7 @@ export class QueryEnhancementsPlugin ); data.search.registerSearchStrategy(SEARCH_STRATEGY.PPL, pplSearchStrategy); + data.search.registerSearchStrategy(SEARCH_STRATEGY.PPL_RAW, pplRawSearchStrategy); data.search.registerSearchStrategy(SEARCH_STRATEGY.SQL, sqlSearchStrategy); data.search.registerSearchStrategy(SEARCH_STRATEGY.SQL_ASYNC, sqlAsyncSearchStrategy); diff --git a/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts b/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts new file mode 100644 index 000000000000..72508db5d7f6 --- /dev/null +++ b/src/plugins/query_enhancements/server/search/ppl_raw_search_strategy.ts @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SharedGlobalConfig, Logger, ILegacyClusterClient } from 'opensearch-dashboards/server'; +import { Observable } from 'rxjs'; +import { ISearchStrategy, SearchUsage } from '../../../data/server'; +import { shimSchemaRow } from '../utils'; + +export const pplRawSearchStrategyProvider = ( + config$: Observable, + logger: Logger, + client: ILegacyClusterClient, + usage?: SearchUsage +): ISearchStrategy => { + return { + search: async (context, request: any, options) => { + const runSearch = request.dataSourceId + ? context.dataSource.opensearch.legacy.getClient(request.dataSourceId).callAPI + : client.asScoped(request.rawRequest).callAsCurrentUser; + + try { + const rawResponse: any = await runSearch('ppl.pplQuery', { body: request.params.body }); + const data = shimSchemaRow(rawResponse); + rawResponse.jsonData = data.jsonData; + + return { + rawResponse, + }; + } catch (e) { + logger.error(`pplRawSearchStrategy: ${e.message}`); + if (usage) usage.trackError(); + throw e; + } + }, + }; +}; diff --git a/src/plugins/vis_type_vega/public/data_model/ppl_parser.test.ts b/src/plugins/vis_type_vega/public/data_model/ppl_parser.test.ts new file mode 100644 index 000000000000..f4ee80ecbe14 --- /dev/null +++ b/src/plugins/vis_type_vega/public/data_model/ppl_parser.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { PPLQueryParser } from './ppl_parser'; + +test('it should throw error if with invalid url object', () => { + const searchApiMock = { + search: jest.fn(() => ({ + toPromise: jest.fn(() => Promise.resolve({})), + })), + }; + const parser = new PPLQueryParser(searchApiMock); + expect(() => parser.parseUrl({}, {})).toThrowError(); + expect(() => parser.parseUrl({}, { body: {} })).toThrowError(); + expect(() => parser.parseUrl({}, { body: { query: {} } })).toThrowError(); +}); + +test('it should parse url object', () => { + const searchApiMock = { + search: jest.fn(() => ({ + toPromise: jest.fn(() => Promise.resolve({})), + })), + }; + const parser = new PPLQueryParser(searchApiMock); + const result = parser.parseUrl({}, { body: { query: 'source=test_index' } }); + expect(result.dataObject).toEqual({}); + expect(result.url).toEqual({ body: { query: 'source=test_index' } }); +}); + +it('should populate data to request', async () => { + const searchApiMock = { + search: jest.fn(() => ({ + toPromise: jest.fn(() => + Promise.resolve([{ name: 'request name', rawResponse: { jsonData: [{ id: 'id1' }] } }]) + ), + })), + }; + const parser = new PPLQueryParser(searchApiMock); + const request = { + url: { body: { query: 'source=test_index' } }, + dataObject: { + name: 'request name', + }, + }; + await parser.populateData([request]); + expect(request.dataObject.values).toEqual([{ id: 'id1' }]); +}); diff --git a/src/plugins/vis_type_vega/public/data_model/ppl_parser.ts b/src/plugins/vis_type_vega/public/data_model/ppl_parser.ts new file mode 100644 index 000000000000..8babcfc6e387 --- /dev/null +++ b/src/plugins/vis_type_vega/public/data_model/ppl_parser.ts @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { Data, UrlObject, PPLQueryRequest } from './types'; +import { SearchAPI } from './search_api'; + +const getRequestName = (request: PPLQueryRequest, index: number) => + request.dataObject.name || + i18n.translate('visTypeVega.opensearchQueryParser.unnamedRequest', { + defaultMessage: 'Unnamed request #{index}', + values: { index }, + }); + +export class PPLQueryParser { + searchAPI: SearchAPI; + + constructor(searchAPI: SearchAPI) { + this.searchAPI = searchAPI; + } + + parseUrl(dataObject: Data, url: UrlObject) { + // data.url.body.query must be defined + if (!url.body || !url.body.query || typeof url.body.query !== 'string') { + throw new Error( + i18n.translate('visTypeVega.pplQueryParser.dataUrl.PPL.queryCannotBeEmpty', { + defaultMessage: '{dataUrlParam} must have query specified', + values: { + dataUrlParam: '"data.url"', + }, + }) + ); + } + + return { dataObject, url }; + } + + async populateData(requests: PPLQueryRequest[]) { + const searchRequests = requests.map((r, index) => ({ + ...r.url, + name: getRequestName(r, index), + })); + + const data$ = await this.searchAPI.search(searchRequests, { strategy: 'pplraw' }); + const results = await data$.toPromise(); + results.forEach((data, index) => { + const requestObject = requests.find((item) => getRequestName(item, index) === data.name); + + if (requestObject) { + requestObject.dataObject.url = requestObject.url; + requestObject.dataObject.values = (data.rawResponse as any).jsonData; + } + }); + } +} diff --git a/src/plugins/vis_type_vega/public/data_model/search_api.ts b/src/plugins/vis_type_vega/public/data_model/search_api.ts index 2c0b7cdd614e..f3491063b305 100644 --- a/src/plugins/vis_type_vega/public/data_model/search_api.ts +++ b/src/plugins/vis_type_vega/public/data_model/search_api.ts @@ -28,6 +28,7 @@ * under the License. */ +import { i18n } from '@osd/i18n'; import { combineLatest } from 'rxjs'; import { map, tap } from 'rxjs/operators'; import { CoreStart, IUiSettingsClient } from 'opensearch-dashboards/public'; @@ -38,10 +39,21 @@ import { SearchRequest, DataPublicPluginStart, IOpenSearchSearchResponse, + IOpenSearchSearchRequest, } from '../../../data/public'; import { search as dataPluginSearch } from '../../../data/public'; import { VegaInspectorAdapters } from '../vega_inspector'; -import { RequestResponder } from '../../../inspector/public'; +import { RequestResponder, RequestStatistics } from '../../../inspector/public'; + +interface RawPPLStrategySearchResponse { + rawResponse: { + datarows: any[]; + jsonData: any[]; + schema: any[]; + size: number; + total: number; + }; +} export interface SearchAPIDependencies { uiSettings: IUiSettingsClient; @@ -58,7 +70,7 @@ export class SearchAPI { public readonly inspectorAdapters?: VegaInspectorAdapters ) {} - async search(searchRequests: SearchRequest[]) { + async search(searchRequests: SearchRequest[], options?: { strategy?: string }) { const { search } = this.dependencies.search; const requestResponders: any = {}; @@ -87,7 +99,13 @@ export class SearchAPI { ? { params, dataSourceId } : { params }; - return search(searchApiParams, { abortSignal: this.abortSignal }).pipe( + return search< + IOpenSearchSearchRequest, + IOpenSearchSearchResponse | RawPPLStrategySearchResponse + >(searchApiParams, { + abortSignal: this.abortSignal, + strategy: options?.strategy, + }).pipe( tap((data) => this.inspectSearchResult(data, requestResponders[requestId])), map((data) => ({ name: requestId, @@ -135,14 +153,45 @@ export class SearchAPI { } } + private getPPLRawResponseInspectorStats(response: RawPPLStrategySearchResponse['rawResponse']) { + const stats: RequestStatistics = {}; + stats.hitsTotal = { + label: i18n.translate('data.search.searchSource.hitsTotalLabel', { + defaultMessage: 'Hits (total)', + }), + value: `${response.total}`, + description: i18n.translate('data.search.searchSource.hitsTotalDescription', { + defaultMessage: 'The number of documents that match the query.', + }), + }; + + stats.hits = { + label: i18n.translate('data.search.searchSource.hitsLabel', { + defaultMessage: 'Hits', + }), + value: `${response.size}`, + description: i18n.translate('data.search.searchSource.hitsDescription', { + defaultMessage: 'The number of documents returned by the query.', + }), + }; + return stats; + } + private inspectSearchResult( - response: IOpenSearchSearchResponse, + response: IOpenSearchSearchResponse | RawPPLStrategySearchResponse, requestResponder: RequestResponder ) { if (requestResponder) { - requestResponder - .stats(dataPluginSearch.getResponseInspectorStats(response.rawResponse)) - .ok({ json: response.rawResponse }); + // inspect ppl response + if ('jsonData' in response.rawResponse) { + requestResponder + .stats(this.getPPLRawResponseInspectorStats(response.rawResponse)) + .ok({ json: response.rawResponse }); + } else { + requestResponder + .stats(dataPluginSearch.getResponseInspectorStats(response.rawResponse)) + .ok({ json: response.rawResponse }); + } } } } diff --git a/src/plugins/vis_type_vega/public/data_model/types.ts b/src/plugins/vis_type_vega/public/data_model/types.ts index 4d0b2bb476d4..3bb604c762ae 100644 --- a/src/plugins/vis_type_vega/public/data_model/types.ts +++ b/src/plugins/vis_type_vega/public/data_model/types.ts @@ -40,7 +40,7 @@ import { UrlParser } from './url_parser'; interface Body { aggs?: SearchParams['body']['aggs']; - query?: Query; + query?: Query | string; timeout?: string; } @@ -214,6 +214,7 @@ export type OpenSearchQueryRequest = Requests; export type EmsQueryRequest = Requests & { obj: UrlObject; }; +export type PPLQueryRequest = Requests; export interface ContextVarsObject { [index: string]: any; diff --git a/src/plugins/vis_type_vega/public/data_model/vega_parser.ts b/src/plugins/vis_type_vega/public/data_model/vega_parser.ts index 927fbec6615b..349e771c65b1 100644 --- a/src/plugins/vis_type_vega/public/data_model/vega_parser.ts +++ b/src/plugins/vis_type_vega/public/data_model/vega_parser.ts @@ -37,6 +37,8 @@ import { euiThemeVars } from '@osd/ui-shared-deps/theme'; import { i18n } from '@osd/i18n'; // @ts-ignore import { Signal } from 'vega'; + +import { HttpSetup } from 'opensearch-dashboards/public'; import { vega, vegaLite } from '../lib/vega'; import { OpenSearchQueryParser } from './opensearch_query_parser'; import { Utils } from './utils'; @@ -59,6 +61,7 @@ import { ControlsDirection, OpenSearchDashboards, } from './types'; +import { PPLQueryParser } from './ppl_parser'; // Set default single color to match other OpenSearch Dashboards visualizations const defaultColor: string = euiPaletteColorBlind()[0]; @@ -653,6 +656,7 @@ The URL is an identifier only. OpenSearch Dashboards and your browser will never opensearch: new OpenSearchQueryParser(this.timeCache, this.searchAPI, this.filters, onWarn), emsfile: new EmsFileParser(serviceSettings), url: new UrlParser(onWarn), + ppl: new PPLQueryParser(this.searchAPI), }; } const pending: PendingType = {}; diff --git a/src/plugins/visualizations/public/legacy/build_pipeline.ts b/src/plugins/visualizations/public/legacy/build_pipeline.ts index de41a7a48c02..71a5c2f5ad8e 100644 --- a/src/plugins/visualizations/public/legacy/build_pipeline.ts +++ b/src/plugins/visualizations/public/legacy/build_pipeline.ts @@ -390,8 +390,8 @@ export const buildVislibDimensions = async ( export const buildPipeline = async (vis: Vis, params: BuildPipelineParams) => { const { indexPattern, searchSource } = vis.data; - const query = searchSource!.getField('query'); - const filters = searchSource!.getField('filter'); + const query = searchSource?.getField('query'); + const filters = searchSource?.getField('filter'); const { uiState, title } = vis; // context