From 086a7ef8a75690508d0492e46d1395c417637cc2 Mon Sep 17 00:00:00 2001 From: Paul Sebastian Date: Tue, 3 Oct 2023 14:29:30 -0700 Subject: [PATCH] implementing async query support with polling (#131) * successfully getting async post requests Signed-off-by: Paul Sebastian * get query with hardcoded job id working Signed-off-by: Paul Sebastian * working spark query after constant time wait Signed-off-by: Paul Sebastian * added polling with some hardcoded values Signed-off-by: Paul Sebastian * ability to switch between sources implemented Signed-off-by: Paul Sebastian * implemented basic spinner Signed-off-by: Paul Sebastian * small pr asks and cancellation implemented Signed-off-by: Paul Sebastian * fixing small clear state issues Signed-off-by: Paul Sebastian * reduce route name redundancy Signed-off-by: Paul Sebastian * remove multiple query implementation for async Signed-off-by: Paul Sebastian * needed to modify the endpoint Signed-off-by: Paul Sebastian * default data source being Opensearch and updated snapshots Signed-off-by: Paul Sebastian --------- Signed-off-by: Paul Sebastian --- common/types/index.ts | 2 + .../Main/__snapshots__/main.test.tsx.snap | 262 ++++++++++++-- public/components/Main/main.tsx | 204 ++++++++++- public/components/PPLPage/PPLPage.tsx | 81 ++--- .../QueryResults/AsyncQueryBody.tsx | 32 ++ .../components/QueryResults/QueryResults.tsx | 333 ++++++++++-------- public/components/SQLPage/SQLPage.tsx | 9 +- server/clusters/sql/sqlPlugin.js | 38 +- server/routes/query.ts | 51 +++ server/services/QueryService.ts | 58 ++- server/services/utils/constants.ts | 1 + server/utils/constants.ts | 2 + 12 files changed, 827 insertions(+), 246 deletions(-) create mode 100644 public/components/QueryResults/AsyncQueryBody.tsx diff --git a/common/types/index.ts b/common/types/index.ts index 928d7578..c96bb147 100644 --- a/common/types/index.ts +++ b/common/types/index.ts @@ -76,3 +76,5 @@ export interface CreateAccelerationForm { refreshIntervalOptions: RefreshIntervalType; formErrors: FormErrorsType; } + +export type AsyncQueryLoadingStatus = "SUCCESS" | "FAILED" | "RUNNING" | "SCHEDULED" | "CANCELED" \ No newline at end of file diff --git a/public/components/Main/__snapshots__/main.test.tsx.snap b/public/components/Main/__snapshots__/main.test.tsx.snap index 2f967b2b..d77c51ba 100644 --- a/public/components/Main/__snapshots__/main.test.tsx.snap +++ b/public/components/Main/__snapshots__/main.test.tsx.snap @@ -30,11 +30,20 @@ exports[`
spec click clear button 1`] = ` data-test-subj="comboBoxInput" tabindex="-1" > -

- Connection Name -

+ + + Opensearch + + +
spec click clear button 1`] = `
+
); @@ -801,6 +966,9 @@ export class Main extends React.Component { getText={this.getText} isResultFullScreen={this.state.isResultFullScreen} setIsResultFullScreen={this.setIsResultFullScreen} + asyncLoading={this.state.asyncLoading} + asyncLoadingStatus={this.state.asyncLoadingStatus} + cancelAsyncQuery={this.cancelAsyncQuery} />
diff --git a/public/components/PPLPage/PPLPage.tsx b/public/components/PPLPage/PPLPage.tsx index d80143a0..69530e8f 100644 --- a/public/components/PPLPage/PPLPage.tsx +++ b/public/components/PPLPage/PPLPage.tsx @@ -3,8 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ - -import React from "react"; +import React from 'react'; import { EuiPanel, EuiButton, @@ -20,23 +19,24 @@ import { EuiModalHeader, EuiModalHeaderTitle, EuiOverlayMask, -} from "@elastic/eui"; +} from '@elastic/eui'; import { ResponseDetail, TranslateResult } from '../Main/main'; import _ from 'lodash'; interface PPLPageProps { - onRun: (query: string) => void, - onTranslate: (query: string) => void, - onClear: () => void, - updatePPLQueries: (query: string) => void, - pplQuery: string, - pplTranslations: ResponseDetail[] + onRun: (query: string) => void; + onTranslate: (query: string) => void; + onClear: () => void; + updatePPLQueries: (query: string) => void; + pplQuery: string; + pplTranslations: ResponseDetail[]; + asyncLoading: boolean; } interface PPLPageState { - pplQuery: string, - translation: string, - isModalVisible: boolean + pplQuery: string; + translation: string; + isModalVisible: boolean; } export class PPLPage extends React.Component { @@ -44,19 +44,18 @@ export class PPLPage extends React.Component { super(props); this.state = { pplQuery: this.props.pplQuery, - translation: "", - isModalVisible: false + translation: '', + isModalVisible: false, }; } setIsModalVisible(visible: boolean): void { this.setState({ - isModalVisible: visible - }) + isModalVisible: visible, + }); } render() { - const closeModal = () => this.setIsModalVisible(false); const showModal = () => this.setIsModalVisible(true); @@ -65,10 +64,12 @@ export class PPLPage extends React.Component { return this.props.pplTranslations[0].fulfilled; } return false; - } + }; const explainContent = pplTranslationsNotEmpty() - ? this.props.pplTranslations.map((queryTranslation: any) => JSON.stringify(queryTranslation.data, null, 2)).join("\n") + ? this.props.pplTranslations + .map((queryTranslation: any) => JSON.stringify(queryTranslation.data, null, 2)) + .join('\n') : 'This query is not explainable.'; let modal; @@ -82,11 +83,7 @@ export class PPLPage extends React.Component { - + {explainContent} @@ -94,7 +91,7 @@ export class PPLPage extends React.Component { Close - + @@ -103,7 +100,9 @@ export class PPLPage extends React.Component { return ( -

Query editor

 + +

Query editor

+ { onChange={this.props.updatePPLQueries} showPrintMargin={false} setOptions={{ - fontSize: "14px", + fontSize: '14px', showLineNumbers: false, showGutter: false, }} @@ -121,38 +120,36 @@ export class PPLPage extends React.Component { /> - this.props.onRun(this.props.pplQuery)} > - - Run + + {this.props.asyncLoading ? 'Running' : 'Run'} { - this.props.updatePPLQueries(""); + this.props.updatePPLQueries(''); this.props.onClear(); }} > - - Clear - + Clear - - this.props.onTranslate(this.props.pplQuery) - } - > + this.props.onTranslate(this.props.pplQuery)}> Explain {modal} - - ) + + ); } } diff --git a/public/components/QueryResults/AsyncQueryBody.tsx b/public/components/QueryResults/AsyncQueryBody.tsx new file mode 100644 index 00000000..eb28ab85 --- /dev/null +++ b/public/components/QueryResults/AsyncQueryBody.tsx @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiFlexGroup, EuiText, EuiLoadingSpinner, EuiButton } from '@elastic/eui'; +import { AsyncQueryLoadingStatus } from '../../../common/types'; +import React from 'react'; + +interface AsyncQueryBodyProps { + asyncLoading: boolean; + asyncLoadingStatus: AsyncQueryLoadingStatus; + cancelAsyncQuery: () => void; +} + +export function AsyncQueryBody(props: AsyncQueryBodyProps) { + const { asyncLoading, asyncLoadingStatus, cancelAsyncQuery } = props; + + // TODO: implement query failure display + // TODO: implement query cancellation + + return ( + + + +

Query running

+ + status: {asyncLoadingStatus} + Cancel + + ); +} diff --git a/public/components/QueryResults/QueryResults.tsx b/public/components/QueryResults/QueryResults.tsx index 5acccac3..1002e378 100644 --- a/public/components/QueryResults/QueryResults.tsx +++ b/public/components/QueryResults/QueryResults.tsx @@ -3,18 +3,49 @@ * SPDX-License-Identifier: Apache-2.0 */ - -import React from "react"; +import React from 'react'; // @ts-ignore -import { SortableProperties, SortableProperty } from "@elastic/eui/lib/services"; +import { SortableProperties, SortableProperty } from '@elastic/eui/lib/services'; // @ts-ignore -import { EuiPanel, EuiFlexGroup, EuiFlexItem, EuiTab, EuiTabs, EuiPopover, EuiContextMenuItem, EuiContextMenuPanel, EuiHorizontalRule, EuiSearchBar, Pager, EuiIcon, EuiText, EuiSpacer, EuiTextAlign, EuiButton, EuiButtonIcon, Comparators } from "@elastic/eui"; -import { QueryResult, QueryMessage, Tab, ResponseDetail, ItemIdToExpandedRowMap, DataRow } from "../Main/main"; -import QueryResultsBody from "./QueryResultsBody"; -import { getQueryIndex, needsScrolling, getSelectedResults } from "../../utils/utils"; -import { DEFAULT_NUM_RECORDS_PER_PAGE, MESSAGE_TAB_LABEL, TAB_CONTAINER_ID } from "../../utils/constants"; +import { + EuiPanel, + EuiFlexGroup, + EuiFlexItem, + EuiTab, + EuiTabs, + EuiPopover, + EuiContextMenuItem, + EuiContextMenuPanel, + EuiHorizontalRule, + EuiSearchBar, + Pager, + EuiIcon, + EuiText, + EuiSpacer, + EuiTextAlign, + EuiButton, + EuiButtonIcon, + Comparators, +} from '@elastic/eui'; +import { + QueryResult, + QueryMessage, + Tab, + ResponseDetail, + ItemIdToExpandedRowMap, + DataRow, +} from '../Main/main'; +import QueryResultsBody from './QueryResultsBody'; +import { getQueryIndex, needsScrolling, getSelectedResults } from '../../utils/utils'; +import { + DEFAULT_NUM_RECORDS_PER_PAGE, + MESSAGE_TAB_LABEL, + TAB_CONTAINER_ID, +} from '../../utils/constants'; import { PanelWrapper } from '../../utils/PanelWrapper'; import _ from 'lodash'; +import { AsyncQueryBody } from './AsyncQueryBody'; +import { AsyncQueryLoadingStatus } from '../../../common/types'; interface QueryResultsProps { language: string; @@ -39,6 +70,9 @@ interface QueryResultsProps { getText: (queries: string[]) => void; isResultFullScreen: boolean; setIsResultFullScreen: (isFullScreen: boolean) => void; + asyncLoading: boolean; + asyncLoadingStatus: AsyncQueryLoadingStatus; + cancelAsyncQuery: () => void; } interface QueryResultsState { @@ -60,20 +94,20 @@ class QueryResults extends React.Component this.state = { isPopoverOpen: false, tabsOverflow: this.props.tabsOverflow ? this.props.tabsOverflow : false, - itemsPerPage: DEFAULT_NUM_RECORDS_PER_PAGE + itemsPerPage: DEFAULT_NUM_RECORDS_PER_PAGE, }; this.sortableColumns = []; - this.sortedColumn = ""; + this.sortedColumn = ''; this.sortableProperties = new SortableProperties( [ { - name: "", - getValue: (item: any) => "", - isAscending: true - } + name: '', + getValue: (item: any) => '', + isAscending: true, + }, ], - "" + '' ); this.tabNames = []; @@ -81,15 +115,15 @@ class QueryResults extends React.Component } componentDidUpdate() { - const showArrow = needsScrolling("tabsContainer"); + const showArrow = needsScrolling('tabsContainer'); if (showArrow !== this.state.tabsOverflow) { this.setState({ tabsOverflow: showArrow }); } } // Actions for Tabs Button showTabsMenu = (): void => { - this.setState(prevState => ({ - isPopoverOpen: !prevState.isPopoverOpen + this.setState((prevState) => ({ + isPopoverOpen: !prevState.isPopoverOpen, })); }; @@ -107,14 +141,14 @@ class QueryResults extends React.Component closePopover = (): void => { this.setState({ - isPopoverOpen: false + isPopoverOpen: false, }); }; onChangeItemsPerPage = (itemsPerPage: number) => { this.pager.setItemsPerPage(itemsPerPage); this.setState({ - itemsPerPage + itemsPerPage, }); }; @@ -136,21 +170,17 @@ class QueryResults extends React.Component this.sortableColumns.push({ name: field, getValue: (item: DataRow) => item.data[field], - isAscending: true + isAscending: true, }); }); - this.sortedColumn = - this.sortableColumns.length > 0 ? this.sortableColumns[0].name : ""; - this.sortableProperties = new SortableProperties( - this.sortableColumns, - this.sortedColumn - ); - } + this.sortedColumn = this.sortableColumns.length > 0 ? this.sortableColumns[0].name : ''; + this.sortableProperties = new SortableProperties(this.sortableColumns, this.sortedColumn); + }; searchItems(dataRows: DataRow[], searchQuery: string): DataRow[] { let rows: { [key: string]: any }[] = []; for (const row of dataRows) { - rows.push(row.data) + rows.push(row.data); } const searchResult = EuiSearchBar.Query.execute(searchQuery, rows); let result: DataRow[] = []; @@ -158,25 +188,25 @@ class QueryResults extends React.Component let dataRow: DataRow = { // rowId does not matter here since the data rows would be sorted later rowId: 0, - data: row - } - result.push(dataRow) + data: row, + }; + result.push(dataRow); } return result; } onSort = (prop: string, items: DataRow[]): DataRow[] => { let sortedRows = this.sortDataRows(items, prop); - this.sortableProperties.sortOn(prop) + this.sortableProperties.sortOn(prop); this.sortedColumn = prop; return sortedRows; - } + }; sortDataRows(dataRows: DataRow[], field: string): DataRow[] { const property = this.sortableProperties.getSortablePropertyByName(field); const copy = [...dataRows]; let comparator = (a: DataRow, b: DataRow) => { - if (typeof property === "undefined") { + if (typeof property === 'undefined') { return 0; } let dataA = a.data; @@ -190,7 +220,7 @@ class QueryResults extends React.Component } } return 0; - } + }; if (!this.sortableProperties.isAscendingByName(field)) { Comparators.reverse(comparator); } @@ -202,20 +232,21 @@ class QueryResults extends React.Component { id: MESSAGE_TAB_LABEL, name: _.truncate(MESSAGE_TAB_LABEL, { length: 17 }), - disabled: false - } + disabled: false, + }, ]; this.tabNames = []; if (this.props.queryResults) { for (let i = 0; i < this.props.queryResults.length; i += 1) { - const tabName = this.props.language === "SQL" ? getQueryIndex(this.props.queries[i]) : "Events"; + const tabName = + this.props.language === 'SQL' ? getQueryIndex(this.props.queries[i]) : 'Events'; this.tabNames.push(tabName); if (this.props.queryResults[i].fulfilled) { tabs.push({ id: i.toString(), name: tabName, - disabled: false + disabled: false, }); } } @@ -225,7 +256,10 @@ class QueryResults extends React.Component render() { // Update PAGINATION and SORTABLE columns - const queryResultSelected = getSelectedResults(this.props.queryResults, this.props.selectedTabId); + const queryResultSelected = getSelectedResults( + this.props.queryResults, + this.props.selectedTabId + ); if (queryResultSelected) { const matchingItems: object[] = this.props.searchQuery @@ -236,12 +270,7 @@ class QueryResults extends React.Component } // Action button with list of tabs, TODO: disable tabArrowRight and tabArrowLeft when no more scrolling is possible - const tabArrowDown = ( - - ); + const tabArrowDown = ; const tabs: Tab[] = this.renderTabs(); const tabsItems = tabs.map((tab, index) => ( - {this.props.queryResults.length > 0 && ( - this.props.isResultFullScreen ? + {this.props.queryResults.length > 0 && + (this.props.isResultFullScreen ? ( this.props.setIsResultFullScreen(false)} /> - : + ) : ( this.props.setIsResultFullScreen(true)} > Full screen view - - )} + + ))} - {this.props.queryResults.length === 0 ? ( - // show no results message instead of the results table when there are no results + {!this.props.asyncLoading ? ( <> - - - - -

No result

- - -

Enter a query in the query editor above to see results.

- - - - - - ) : ( - <> - - {/*TABS*/} + {this.props.queryResults.length === 0 ? ( + // show no results message instead of the results table when there are no results + <> + + + + +

No result

+ + +

Enter a query in the query editor above to see results.

+ + + + + + ) : ( + <> - - {tabsButtons} - - + + {tabsButtons} + + - {/*ARROW DOWN*/} - {this.state.tabsOverflow && ( -
- - - - - - - -
- )} - - + {/*ARROW DOWN*/} + {this.state.tabsOverflow && ( +
+ + + + + + + +
+ )} + + - {/*RESULTS TABLE*/} - - - - - )} + {/*RESULTS TABLE*/} + + + + + )} + + ) : ( + <> + + + + + + + + + )} ); } diff --git a/public/components/SQLPage/SQLPage.tsx b/public/components/SQLPage/SQLPage.tsx index 0cc0bac4..b0f1eb5c 100644 --- a/public/components/SQLPage/SQLPage.tsx +++ b/public/components/SQLPage/SQLPage.tsx @@ -33,6 +33,7 @@ interface SQLPageProps { sqlQuery: string; sqlTranslations: ResponseDetail[]; selectedDatasource: string; + asyncLoading: boolean; } interface SQLPageState { @@ -142,8 +143,12 @@ export class SQLPage extends React.Component { this.props.onRun(this.props.sqlQuery)}> - - Run + + {this.props.asyncLoading ? 'Running' : 'Run'} `, + req: { + jobId: { + type: 'string', + required: true, + }, + }, + }, + needBody: true, + method: 'GET', + }); + + sql.asyncDeleteQuery = ca({ + url: { + fmt: `${SPARK_SQL_QUERY_ROUTE}/<%=jobId%>`, + req: { + jobId: { + type: 'string', + required: true, + }, + }, + }, + needBody: true, + method: 'DELETE', + }); } diff --git a/server/routes/query.ts b/server/routes/query.ts index 02526bb5..a4b0b376 100644 --- a/server/routes/query.ts +++ b/server/routes/query.ts @@ -16,6 +16,8 @@ import { ROUTE_PATH_PPL_CSV, ROUTE_PATH_PPL_JSON, ROUTE_PATH_PPL_TEXT, + ROUTE_PATH_SPARK_SQL_QUERY, + ROUTE_PATH_SPARK_SQL_JOB_QUERY, } from '../utils/constants'; export default function query(server: IRouter, service: QueryService) { @@ -138,4 +140,53 @@ export default function query(server: IRouter, service: QueryService) { }); } ); + + server.post( + { + path: ROUTE_PATH_SPARK_SQL_QUERY, + validate: { + body: schema.any(), + }, + }, + async (context, request, response): Promise> => { + const retVal = await service.describeSQLAsyncQuery(request); + return response.ok({ + body: retVal, + }); + } + ) + + server.get( + { + path: ROUTE_PATH_SPARK_SQL_JOB_QUERY + "/{id}", + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, request, response): Promise> => { + const retVal = await service.describeSQLAsyncGetQuery(request, request.params.id); + return response.ok({ + body: retVal, + }); + } + ) + + server.delete( + { + path: ROUTE_PATH_SPARK_SQL_JOB_QUERY + "/{id}", + validate: { + params: schema.object({ + id: schema.string(), + }), + }, + }, + async (context, request, response): Promise> => { + const retVal = await service.describeAsyncDeleteQuery(request, request.params.id); + return response.ok({ + body: retVal, + }); + } + ) } diff --git a/server/services/QueryService.ts b/server/services/QueryService.ts index 33425766..9aa927ce 100644 --- a/server/services/QueryService.ts +++ b/server/services/QueryService.ts @@ -14,13 +14,10 @@ export default class QueryService { this.client = client; } - describeQueryInternal = async (request: any, format: string, responseFormat: string) => { + describeQueryPostInternal = async (request: any, format: string, responseFormat: string, body: any) => { try { - const queryRequest = { - query: request.body.query, - }; const params = { - body: JSON.stringify(queryRequest), + body: JSON.stringify(body), }; const queryResponse = await this.client.asScoped(request).callAsCurrentUser(format, params); @@ -42,35 +39,70 @@ export default class QueryService { } }; + describeQueryJobIdInternal = async (request: any, format: string, jobId: string, responseFormat: string) => { + try { + const queryResponse = await this.client.asScoped(request).callAsCurrentUser(format, { + jobId: jobId, + }); + return { + data: { + ok: true, + resp: _.isEqual(responseFormat, 'json') ? JSON.stringify(queryResponse) : queryResponse, + }, + }; + } catch (err) { + console.log(err); + return { + data: { + ok: false, + resp: err.message, + body: err.body + }, + }; + } + }; + describeSQLQuery = async (request: any) => { - return this.describeQueryInternal(request, 'sql.sqlQuery', 'json'); + return this.describeQueryPostInternal(request, 'sql.sqlQuery', 'json', request.body); }; describePPLQuery = async (request: any) => { - return this.describeQueryInternal(request, 'sql.pplQuery', 'json'); + return this.describeQueryPostInternal(request, 'sql.pplQuery', 'json', request.body); }; describeSQLCsv = async (request: any) => { - return this.describeQueryInternal(request, 'sql.sqlCsv', null); + return this.describeQueryPostInternal(request, 'sql.sqlCsv', null, request.body); }; describePPLCsv = async (request: any) => { - return this.describeQueryInternal(request, 'sql.pplCsv', null); + return this.describeQueryPostInternal(request, 'sql.pplCsv', null, request.body); }; describeSQLJson = async (request: any) => { - return this.describeQueryInternal(request, 'sql.sqlJson', 'json'); + return this.describeQueryPostInternal(request, 'sql.sqlJson', 'json', request.body); }; describePPLJson = async (request: any) => { - return this.describeQueryInternal(request, 'sql.pplJson', 'json'); + return this.describeQueryPostInternal(request, 'sql.pplJson', 'json', request.body); }; describeSQLText = async (request: any) => { - return this.describeQueryInternal(request, 'sql.sqlText', null); + return this.describeQueryPostInternal(request, 'sql.sqlText', null, request.body); }; describePPLText = async (request: any) => { - return this.describeQueryInternal(request, 'sql.pplText', null); + return this.describeQueryPostInternal(request, 'sql.pplText', null, request.body); + }; + + describeSQLAsyncQuery = async (request: any) => { + return this.describeQueryPostInternal(request, 'sql.sparkSqlQuery', null, request.body); + }; + + describeSQLAsyncGetQuery = async (request: any, jobId: string) => { + return this.describeQueryJobIdInternal(request, 'sql.sparkSqlGetQuery', jobId, null); + }; + + describeAsyncDeleteQuery = async (request: any, jobId: string) => { + return this.describeQueryJobIdInternal(request, 'sql.asyncDeleteQuery', jobId, null); }; } diff --git a/server/services/utils/constants.ts b/server/services/utils/constants.ts index 043d84b5..2d103254 100644 --- a/server/services/utils/constants.ts +++ b/server/services/utils/constants.ts @@ -13,6 +13,7 @@ export const PPL_QUERY_ROUTE = `/_plugins/_ppl`; export const FORMAT_CSV = `format=csv`; export const FORMAT_JSON = `format=json`; export const FORMAT_TEXT = `format=raw`; +export const SPARK_SQL_QUERY_ROUTE = `/_plugins/_async_query`; export const DEFAULT_HEADERS = { 'Content-Type': 'application/json', diff --git a/server/utils/constants.ts b/server/utils/constants.ts index fab457eb..da6b6522 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -13,3 +13,5 @@ export const ROUTE_PATH_SQL_JSON = '/api/sql_console/sqljson'; export const ROUTE_PATH_PPL_JSON = '/api/sql_console/ppljson'; export const ROUTE_PATH_SQL_TEXT = '/api/sql_console/sqltext'; export const ROUTE_PATH_PPL_TEXT = '/api/sql_console/ppltext'; +export const ROUTE_PATH_SPARK_SQL_QUERY = '/api/spark_sql_console'; +export const ROUTE_PATH_SPARK_SQL_JOB_QUERY = '/api/spark_sql_console/job'; \ No newline at end of file