From c8067fcf6c7ed0e30cad4854869286c8ca7f61be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ida=20=C5=A0tambuk?= Date: Tue, 4 Jun 2024 18:26:23 +0200 Subject: [PATCH] Refactor ad hoc variable processing (#399) --- cspell.config.json | 4 +- package.json | 2 + src/datasource.test.ts | 342 +++++++++++++++++++------------------ src/datasource.ts | 235 +++++++++++++------------- src/modifyQuery.test.ts | 260 +++++++++++++++++++++++++++++ src/modifyQuery.ts | 362 ++++++++++++++++++++++++++++++++++++++++ yarn.lock | 10 ++ 7 files changed, 929 insertions(+), 286 deletions(-) create mode 100644 src/modifyQuery.test.ts create mode 100644 src/modifyQuery.ts diff --git a/cspell.config.json b/cspell.config.json index 90f6f3f9..8e90ddcc 100644 --- a/cspell.config.json +++ b/cspell.config.json @@ -114,6 +114,8 @@ "Throughputs", "mainstat", "secondarystat", - "x-ndjson" + "x-ndjson", + "Xtorm", + "syslogd" ] } diff --git a/package.json b/package.json index 316725ad..cc6fd4c7 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@grafana/runtime": "10.2.0", "@grafana/ui": "10.2.0", "lodash": "4.17.21", + "lucene": "^2.1.1", "react": "17.0.2", "react-dom": "17.0.2", "react-use": "17.4.0", @@ -48,6 +49,7 @@ "@testing-library/user-event": "14.2.0", "@types/jest": "^29.5.0", "@types/lodash": "^4.14.194", + "@types/lucene": "^2.1.7", "@types/node": "^20.8.7", "@types/prismjs": "1.26.0", "@types/react": "^17.0.14", diff --git a/src/datasource.test.ts b/src/datasource.test.ts index e92d7f01..770787bd 100644 --- a/src/datasource.test.ts +++ b/src/datasource.test.ts @@ -1,4 +1,5 @@ import { + AdHocVariableFilter, ArrayVector, CoreApp, DataFrame, @@ -28,7 +29,7 @@ import { OpenSearchQuery, QueryType, } from './types'; -import { Filters } from './components/QueryEditor/BucketAggregationsEditor/aggregations'; +import { DateHistogram, Filters } from './components/QueryEditor/BucketAggregationsEditor/aggregations'; import { matchers } from './dependencies/matchers'; import { MetricAggregation } from 'components/QueryEditor/MetricAggregationsEditor/aggregations'; import { firstValueFrom, lastValueFrom, of } from 'rxjs'; @@ -48,7 +49,7 @@ jest.mock('./tracking.ts', () => ({ trackQuery: jest.fn(), })); -const getAdHocFiltersMock = jest.fn, any>(() => []); +const getAdHocFiltersMock = jest.fn(() => []); jest.mock('@grafana/runtime', () => ({ ...(jest.requireActual('@grafana/runtime') as unknown as object), @@ -61,12 +62,10 @@ jest.mock('@grafana/runtime', () => ({ }; }, getTemplateSrv: () => ({ - replace: jest.fn((text) => { - if (text.startsWith('$')) { - return `resolvedVariable`; - } else { - return text; - } + replace: jest.fn((text: string) => { + // Replace all $ words, except global variables ($__interval, $__interval_ms, etc.) - they get interpolated on the BE + const resolved = text.replace(/\$(?!__)\w+/g, "resolvedVariable"); + return resolved; }), getAdhocFilters: getAdHocFiltersMock, }), @@ -1435,116 +1434,6 @@ describe('OpenSearchDatasource', function (this: any) { expect(mockedSuperQuery).toHaveBeenCalled(); }); - it('should send interpolated query to backend', () => { - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const query: OpenSearchQuery = { - refId: 'A', - query: '$someVariable', - metrics: [ - { - id: '1', - type: 'raw_data', - settings: { - size: '500', - order: 'desc', - useTimeRange: true, - }, - }, - ], - bucketAggs: [], - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [query], - }; - ctx.ds.query(request); - const expectedRequest = { ...request, targets: [{ ...query, query: 'resolvedVariable' }] }; - expect(mockedSuperQuery).toHaveBeenCalledWith(expectedRequest); - }); - - it('should send ad hoc filtered query to backend', () => { - getAdHocFiltersMock.mockImplementation(() => [{ key: 'bar', operator: '=', value: 'test' }]); - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const query: OpenSearchQuery = { - refId: 'A', - query: '', - metrics: [ - { - id: '1', - type: 'raw_data', - settings: { - size: '500', - order: 'desc', - useTimeRange: true, - }, - }, - ], - bucketAggs: [], - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [query], - }; - ctx.ds.query(request); - const expectedRequest = { ...request, targets: [{ ...query, query: '* AND bar:"test"' }] }; - expect(mockedSuperQuery).toHaveBeenCalledWith(expectedRequest); - }); - - it('should send interpolated and ad hoc filtered query to backend', () => { - getAdHocFiltersMock.mockImplementation(() => [{ key: 'bar', operator: '=', value: 'test' }]); - const mockedSuperQuery = jest - .spyOn(DataSourceWithBackend.prototype, 'query') - .mockImplementation((request: DataQueryRequest) => of()); - const query: OpenSearchQuery = { - refId: 'A', - query: '$someVariable', - metrics: [ - { - id: '1', - type: 'raw_data', - settings: { - size: '500', - order: 'desc', - useTimeRange: true, - }, - }, - ], - bucketAggs: [], - }; - const request: DataQueryRequest = { - requestId: '', - interval: '', - intervalMs: 1, - scopedVars: {}, - timezone: '', - app: CoreApp.Dashboard, - startTime: 0, - range: createTimeRange(toUtc([2015, 4, 30, 10]), toUtc([2015, 5, 1, 10])), - targets: [query], - }; - ctx.ds.query(request); - const expectedRequest = { ...request, targets: [{ ...query, query: 'resolvedVariable AND bar:"test"' }] }; - expect(mockedSuperQuery).toHaveBeenCalledWith(expectedRequest); - }); - it('does not send logs queries in Dashboard to backend', () => { const mockedSuperQuery = jest .spyOn(DataSourceWithBackend.prototype, 'query') @@ -1781,35 +1670,7 @@ describe('OpenSearchDatasource', function (this: any) { expect(datasourceRequestMock).toHaveBeenCalled(); }); }); - - it('should correctly interpolate variables in query', () => { - const query: OpenSearchQuery = { - refId: 'A', - bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }], - metrics: [{ type: 'count', id: '1' }], - query: '$var', - }; - - const interpolatedQuery = ctx.ds.interpolateVariablesInQueries([query], {})[0]; - - expect(interpolatedQuery.query).toBe('resolvedVariable'); - expect((interpolatedQuery.bucketAggs![0] as Filters).settings!.filters![0].query).toBe('resolvedVariable'); - }); - - it('should correctly handle empty query strings', () => { - const query: OpenSearchQuery = { - refId: 'A', - bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '', label: '' }] }, id: '1' }], - metrics: [{ type: 'count', id: '1' }], - query: '', - }; - - const interpolatedQuery = ctx.ds.interpolateVariablesInQueries([query], {})[0]; - - expect(interpolatedQuery.query).toBe('*'); - expect((interpolatedQuery.bucketAggs![0] as Filters).settings!.filters![0].query).toBe('*'); - }); - + describe('getSupportedQueryTypes', () => { it('should return Lucene when no other types are set', () => { const instanceSettings = { @@ -1866,6 +1727,7 @@ describe('OpenSearchDatasource', function (this: any) { ); }); }); + describe('#executeLuceneQueries', () => { beforeEach(() => { createDatasource({ @@ -1992,6 +1854,7 @@ describe('OpenSearchDatasource', function (this: any) { }); describe('addAdHocFilters', () => { + const adHocFilters = [{ key: 'test', operator: '=', value: 'test1', condition: '' }]; describe('with invalid filters', () => { describe('Lucene queries', () => { it('should filter out ad hoc filter without key', () => { @@ -2001,6 +1864,13 @@ describe('OpenSearchDatasource', function (this: any) { expect(query).toBe('foo:"bar"'); }); + it('should filter out ad hoc filter without key when query is empty', () => { + const query = ctx.ds.addAdHocFilters({ refId: 'A', query: '', queryType: QueryType.Lucene }, [ + { key: '', operator: '=', value: 'a', condition: '' }, + ]); + expect(query).toBe('*'); + }); + it('should filter out ad hoc filter without value', () => { const query = ctx.ds.addAdHocFilters({ refId: 'A', query: 'foo:"bar"', queryType: QueryType.Lucene }, [ { key: 'a', operator: '=', value: '', condition: '' }, @@ -2031,20 +1901,18 @@ describe('OpenSearchDatasource', function (this: any) { expect(query).toBe('source = test-index'); }); - it('should filter out filter ad hoc filter with invalid operators', () => { + it('should filter out ad hoc filter with invalid operators', () => { const query = ctx.ds.addAdHocFilters({ refId: 'A', query: 'source = test-index', queryType: QueryType.PPL }, [ - { key: 'a', operator: '=~', value: '', condition: '' }, - { key: 'a', operator: '!~', value: '', condition: '' }, + { key: 'a', operator: '=~', value: 'test', condition: '' }, + { key: 'a', operator: '!~', value: 'test', condition: '' }, ]); expect(query).toBe('source = test-index'); }); }); }); - describe('with 1 ad hoc filter', () => { - const adHocFilters = [{ key: 'test', operator: '=', value: 'test1', condition: '' }]; - - it('should correctly add 1 ad hoc filter when query is not empty', () => { + describe('queries with 1 ad hoc filter', () => { + it('should correctly add 1 ad hoc filter when Lucene query is not empty', () => { const query = ctx.ds.addAdHocFilters( { refId: 'A', query: 'foo:"bar"', queryType: QueryType.Lucene }, adHocFilters @@ -2052,13 +1920,31 @@ describe('OpenSearchDatasource', function (this: any) { expect(query).toBe('foo:"bar" AND test:"test1"'); }); - it('should correctly add 1 ad hoc filter when query is empty', () => { + it('should correctly add 1 ad hoc filter when PPL query is not empty', () => { + const query = ctx.ds.addAdHocFilters( + { refId: 'A', query: 'foo="bar"', queryType: QueryType.PPL }, + adHocFilters + ); + expect(query).toBe('foo="bar" | where `test` = \'test1\''); + }); + }); + + describe('Empty queries with 1 ad hoc filter', () => { + it('Lucene queries should correctly add 1 ad hoc filter when query is empty', () => { // an empty string query is transformed to '*' but this can be refactored to have the same behavior as Elasticsearch const query = ctx.ds.addAdHocFilters({ refId: 'A', query: '', queryType: QueryType.Lucene }, adHocFilters); expect(query).toBe('test:"test1"'); }); - it('should escape characters in filter keys', () => { + it('PPL queries should correctly add 1 ad hoc filter when query is empty', () => { + // an empty string query is transformed to '*' but this can be refactored to have the same behavior as Elasticsearch + const query = ctx.ds.addAdHocFilters({ refId: 'A', query: '', queryType: QueryType.PPL }, adHocFilters); + expect(query).toBe("`test` = 'test1'"); + }); + }); + + describe('Escaping characters in adhoc filter', () => { + it('should escape characters in filter keys in Lucene queries', () => { const query = ctx.ds.addAdHocFilters({ refId: 'A', query: '', queryType: QueryType.Lucene }, [ { key: 'field:name', operator: '=', value: 'field:value', condition: '' }, ]); @@ -2092,14 +1978,14 @@ describe('OpenSearchDatasource', function (this: any) { }); describe('PPL queries', () => { - const adHocFilters = [ - { key: 'bar', operator: '=', value: 'baz', condition: '' }, - { key: 'job', operator: '!=', value: 'grafana', condition: '' }, - { key: 'bytes', operator: '>', value: 50, condition: '' }, - { key: 'count', operator: '<', value: 100, condition: '' }, - { key: 'timestamp', operator: '=', value: '2020-11-22 16:40:43', condition: '' }, - ]; it('should return query with ad-hoc filters applied', () => { + const adHocFilters: AdHocVariableFilter[] = [ + { key: 'bar', operator: '=', value: 'baz', condition: '' }, + { key: 'job', operator: '!=', value: 'grafana', condition: '' }, + { key: 'bytes', operator: '>', value: '50', condition: '' }, + { key: 'count', operator: '<', value: '100', condition: '' }, + { key: 'timestamp', operator: '=', value: '2020-11-22 16:40:43', condition: '' }, + ]; const query = ctx.ds.addAdHocFilters( { refId: 'A', query: 'source = test-index', queryType: QueryType.PPL }, adHocFilters @@ -2111,6 +1997,136 @@ describe('OpenSearchDatasource', function (this: any) { }); }); }); + + describe('interpolateQueries for Explore', () => { + const adHocFilters = [ + { key: 'bar', operator: '=', value: 'baz', condition: '' }, + { key: 'job', operator: '!=', value: 'grafana', condition: '' }, + { key: 'bytes', operator: '>', value: '50', condition: '' }, + { key: 'count', operator: '<', value: '100', condition: '' }, + { key: 'timestamp', operator: '=', value: '2020-11-22 16:40:43', condition: '' }, + ]; + it('correctly applies template variables and adhoc filters to Lucene queries', () => { + const query: OpenSearchQuery = { + refId: 'A', + queryType: QueryType.Lucene, + bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }], + metrics: [{ type: 'count', id: '1' }], + query: '$var', + }; + + const interpolatedQueries = ctx.ds.interpolateVariablesInQueries([query], {}, adHocFilters); + expect(interpolatedQueries[0].query).toBe( + 'resolvedVariable AND bar:"baz" AND -job:"grafana" AND bytes:>50 AND count:<100 AND timestamp:"2020-11-22 16:40:43"' + ); + }); + + it('should correctly apply template variables and adhoc filters to PPL queries', () => { + const query: OpenSearchQuery = { + refId: 'A', + queryType: QueryType.PPL, + bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }], + metrics: [{ type: 'count', id: '1' }], + query: '$var', + }; + + const interpolatedQueries = ctx.ds.interpolateVariablesInQueries([query], {}, adHocFilters); + expect(interpolatedQueries[0].query).toBe( + "resolvedVariable | where `bar` = 'baz' and `job` != 'grafana' and `bytes` > 50 and `count` < 100 and `timestamp` = timestamp('2020-11-22 16:40:43.000000')" + ); + }); + }); + + describe('applyTemplateVariables', () => { + it('should correctly handle empty query strings in Lucene queries', () => { + const query: OpenSearchQuery = { + refId: 'A', + bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '', label: '' }] }, id: '1' }], + metrics: [{ type: 'count', id: '1' }], + query: '', + }; + + const interpolatedQuery = ctx.ds.applyTemplateVariables(query, {}); + + expect(interpolatedQuery.query).toBe('*'); + expect((interpolatedQuery.bucketAggs![0] as Filters).settings!.filters![0].query).toBe('*'); + }); + + it('should correctly interpolate variables in Lucene query', () => { + const query: OpenSearchQuery = { + refId: 'A', + bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }], + metrics: [{ type: 'count', id: '1' }], + query: '$var AND foo:bar', + }; + + const interpolatedQuery = ctx.ds.applyTemplateVariables(query, {}); + + expect(interpolatedQuery.query).toBe('resolvedVariable AND foo:bar'); + expect((interpolatedQuery.bucketAggs![0] as Filters).settings!.filters![0].query).toBe('resolvedVariable'); + }); + + it('should correctly interpolate variables in nested fields in Lucene query', () => { + const query: OpenSearchQuery = { + refId: 'A', + bucketAggs: [{field: 'avgPrice', settings:{interval: "$var", min_doc_count: "$var", trimEdges: "$var"}, type: 'date_histogram', id: '1'}], + metrics: [{ type: 'count', id: '1' }], + query: '$var AND foo:bar', + }; + + const interpolatedQuery = ctx.ds.applyTemplateVariables(query, {}); + + expect((interpolatedQuery.bucketAggs![0] as DateHistogram).settings!.interval).toBe('resolvedVariable'); + expect((interpolatedQuery.bucketAggs![0] as DateHistogram).settings!.min_doc_count).toBe('resolvedVariable'); + expect((interpolatedQuery.bucketAggs![0] as DateHistogram).settings!.trimEdges).toBe('resolvedVariable'); + }) + + it('correctly applies template variables and adhoc filters to Lucene queries', () => { + const adHocFilters: AdHocVariableFilter[] = [ + { key: 'bar', operator: '=', value: 'baz', condition: '' }, + { key: 'job', operator: '!=', value: 'grafana', condition: '' }, + { key: 'bytes', operator: '>', value: '50', condition: '' }, + { key: 'count', operator: '<', value: '100', condition: '' }, + { key: 'timestamp', operator: '=', value: '2020-11-22 16:40:43', condition: '' }, + ]; + const query: OpenSearchQuery = { + refId: 'A', + queryType: QueryType.Lucene, + bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }], + metrics: [{ type: 'count', id: '1' }], + query: '$var', + }; + + // called from grafana runtime + const interpolatedQuery = ctx.ds.applyTemplateVariables(query, {}, adHocFilters); + expect(interpolatedQuery.query).toBe( + 'resolvedVariable AND bar:"baz" AND -job:"grafana" AND bytes:>50 AND count:<100 AND timestamp:"2020-11-22 16:40:43"' + ); + }); + + it('correctly applies template variables and adhoc filters to PPL queries', () => { + const adHocFilters: AdHocVariableFilter[] = [ + { key: 'bar', operator: '=', value: 'baz', condition: '' }, + { key: 'job', operator: '!=', value: 'grafana', condition: '' }, + { key: 'bytes', operator: '>', value: '50', condition: '' }, + { key: 'count', operator: '<', value: '100', condition: '' }, + { key: 'timestamp', operator: '=', value: '2020-11-22 16:40:43', condition: '' }, + ]; + const query: OpenSearchQuery = { + refId: 'A', + queryType: QueryType.PPL, + bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }], + metrics: [{ type: 'count', id: '1' }], + query: '$var', + }; + + const interpolatedQuery = ctx.ds.applyTemplateVariables(query, {}, adHocFilters); + expect(interpolatedQuery.query).toBe( + "resolvedVariable | where `bar` = 'baz' and `job` != 'grafana' and `bytes` > 50 and `count` < 100 and `timestamp` = timestamp('2020-11-22 16:40:43.000000')" + ); + }); + }); + describe('Data links', () => { it('should add links to dataframe for logs queries in the backend flow', async () => { createDatasource({ diff --git a/src/datasource.ts b/src/datasource.ts index e40727a9..46f6a698 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -9,12 +9,14 @@ import { ScopedVars, DataLink, MetricFindValue, - dateMath, dateTime, TimeRange, LoadingState, toUtc, getDefaultTimeRange, + ToggleFilterAction, + QueryFilterOptions, + AdHocVariableFilter, CoreApp, } from '@grafana/data'; import { OpenSearchResponse } from './OpenSearchResponse'; @@ -25,6 +27,7 @@ import { BackendSrvRequest, DataSourceWithBackend, FetchError, + TemplateSrv, getBackendSrv, getDataSourceSrv, getTemplateSrv, @@ -44,6 +47,14 @@ import { enhanceDataFramesWithDataLinks, sha256 } from 'utils'; import { Version } from 'configuration/utils'; import { createTraceDataFrame, createListTracesDataFrame } from 'traces/formatTraces'; import { createLuceneTraceQuery, getTraceIdFromLuceneQueryString } from 'traces/queryTraces'; +import { + PPLQueryHasFilter, + addAdhocFilterToPPLQuery, + addLuceneAdHocFilter, + luceneQueryHasFilter, + toggleQueryFilterForLucene, + toggleQueryFilterForPPL, +} from 'modifyQuery'; // Those are metadata fields as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_identity_metadata_fields. // custom fields can start with underscores, therefore is not safe to exclude anything that starts with one. @@ -70,7 +81,10 @@ export class OpenSearchDatasource extends DataSourceWithBackend) { + constructor( + instanceSettings: DataSourceInstanceSettings, + private readonly templateSrv: TemplateSrv = getTemplateSrv() + ) { super(instanceSettings); this.basicAuth = instanceSettings.basicAuth; this.withCredentials = instanceSettings.withCredentials; @@ -332,110 +346,91 @@ export class OpenSearchDatasource extends DataSourceWithBackend 0) { - expandedQueries = queries.map((query) => { - let interpolatedQuery; - if (query.queryType === QueryType.PPL) { - interpolatedQuery = this.interpolatePPLQuery(query.query || '', scopedVars); - } else { - interpolatedQuery = this.interpolateLuceneQuery(query.query || '', scopedVars); - } - const expandedQuery = { - ...query, - query: interpolatedQuery, - }; + private interpolateLuceneQuery(queryString: string, scopedVars: ScopedVars) { + return this.templateSrv.replace(queryString, scopedVars, 'lucene'); + } - for (let bucketAgg of query.bucketAggs || []) { - if (bucketAgg.type === 'filters') { - for (let filter of bucketAgg.settings?.filters || []) { - filter.query = this.interpolateLuceneQuery(filter.query, scopedVars); - } - } - } - return expandedQuery; - }); - } - return expandedQueries; + private interpolatePPLQuery(queryString: string, scopedVars: ScopedVars) { + return this.templateSrv.replace(queryString, scopedVars, 'pipe'); } - addAdHocFilters(target: OpenSearchQuery, adHocFilters: any) { - if (target.queryType === QueryType.PPL) { - return this.addPPLAdHocFilters(target.query || '', adHocFilters); - } - return this.addLuceneAdHocFilters(target.query || '', adHocFilters); + // called from Explore + interpolateVariablesInQueries( + queries: OpenSearchQuery[], + scopedVars: ScopedVars | {}, + filters?: AdHocVariableFilter[] + ): OpenSearchQuery[] { + return queries.map((q) => this.applyTemplateVariables(q, scopedVars, filters)); } - addLuceneAdHocFilters(query: string, adHocFilters: Array<{ key: string; operator: string; value: string }>) { - if (adHocFilters.length === 0) { - return query; + applyTemplateVariables( + query: OpenSearchQuery, + scopedVars: ScopedVars, + adHocFilters?: AdHocVariableFilter[] + ): OpenSearchQuery { + let interpolatedQuery: string; + if (query.queryType === QueryType.PPL) { + interpolatedQuery = this.interpolatePPLQuery(query.query || '', scopedVars); + } else { + interpolatedQuery = this.interpolateLuceneQuery(query.query || '*', scopedVars); } - const osFilters = adHocFilters.map((filter) => { - let { key, operator, value } = filter; - if (!key || !value) { - return ''; - } - /** - * Keys and values in ad hoc filters may contain characters such as - * colons, which needs to be escaped. - */ - key = key.replace(/:/g, '\\:'); - switch (operator) { - case '=': - return `${key}:"${value}"`; - case '!=': - return `-${key}:"${value}"`; - case '=~': - return `${key}:/${value}/`; - case '!~': - return `-${key}:/${value}/`; - case '>': - return `${key}:>${value}`; - case '<': - return `${key}:<${value}`; + // We need a separate interpolation format for lucene queries, therefore we first interpolate any + // lucene query string and then everything else + for (let bucketAgg of query.bucketAggs || []) { + if (bucketAgg.type === 'filters') { + for (let filter of bucketAgg.settings?.filters || []) { + filter.query = this.interpolateLuceneQuery(filter.query, scopedVars) || '*'; + } } - return ''; - }); - - if (osFilters.length < 0) { - return query; } - return [query, ...osFilters].filter((f) => f).join(' AND '); - } - - addPPLAdHocFilters(queryString: string, adHocFilters: any) { - for (let i = 0; i < adHocFilters.length; i++) { - const { key, operator } = adHocFilters[i]; - let { value } = adHocFilters[i]; + const queryWithAppliedAdHocFilters = adHocFilters?.length + ? this.addAdHocFilters({ ...query, query: interpolatedQuery }, adHocFilters) + : interpolatedQuery; - if ('=~' === operator || '!~' === operator || !key || !value) { - continue; - } - - if (dateMath.isValid(value)) { - const ts = dateTime(value).utc().format('YYYY-MM-DD HH:mm:ss.SSSSSS'); - value = `timestamp('${ts}')`; - } else if (typeof value === 'string') { - value = `'${value}'`; - } + // interpolate any nested fields (interval etc.) + const finalQuery = JSON.parse( + this.templateSrv.replace(JSON.stringify({ ...query, query: queryWithAppliedAdHocFilters }), scopedVars) + ); + return finalQuery; + } - const expression = `\`${key}\` ${operator} ${value}`; - const command = i > 0 ? ' and ' : ' | where '; - queryString += `${command}${expression}`; + addAdHocFilters(target: OpenSearchQuery, adHocFilters: AdHocVariableFilter[]): string { + if (target.queryType === QueryType.PPL) { + let finalQuery: string = target.query || ''; + adHocFilters.forEach((filter, i) => { + finalQuery = addAdhocFilterToPPLQuery(finalQuery, filter, i); + }); + return finalQuery; } - return queryString; + let finalQuery: string = target.query ?? ''; + adHocFilters.forEach((filter) => { + finalQuery = addLuceneAdHocFilter(finalQuery, filter); + }); + return finalQuery || '*'; } testDatasource() { @@ -536,39 +531,25 @@ export class OpenSearchDatasource extends DataSourceWithBackend): Observable { - const targetsWithInterpolatedVariables = this.interpolateVariablesInQueries( - _.cloneDeep(request.targets), - request.scopedVars - ); - // Gradually migrate queries to the backend in this condition - if ( - request.targets.every( - (target) => - target.metrics?.every( - (metric) => - metric.type === 'raw_data' || - metric.type === 'raw_document' || - (request.app === CoreApp.Explore && target.queryType === QueryType.Lucene) - ) || - (request.app === CoreApp.Explore && target.queryType === QueryType.PPL) || - target.luceneQueryType === LuceneQueryType.Traces - ) - ) { - // @ts-ignore - const adHocFilters = getTemplateSrv().getAdhocFilters(this.name); - const queriesWithAdHocAndInterpolatedVariables = targetsWithInterpolatedVariables.map((t) => ({ - ...t, - query: this.addAdHocFilters(t, adHocFilters), - })); - const adHocAndInterpolatedRequest = { ...request, targets: queriesWithAdHocAndInterpolatedVariables }; - return super.query(adHocAndInterpolatedRequest).pipe( + if (request.targets.every( + (target) => + target.metrics?.every( + (metric) => + metric.type === 'raw_data' || + metric.type === 'raw_document' || + (request.app === CoreApp.Explore && target.queryType === QueryType.Lucene) + ) || + (request.app === CoreApp.Explore && target.queryType === QueryType.PPL) || + target.luceneQueryType === LuceneQueryType.Traces + )) { + return super.query(request).pipe( tap({ next: (response) => { - trackQuery(response, targetsWithInterpolatedVariables, adHocAndInterpolatedRequest.app); + trackQuery(response, request.targets, request.app); }, error: (error) => { - trackQuery({ error, data: [] }, targetsWithInterpolatedVariables, adHocAndInterpolatedRequest.app); + trackQuery({ error, data: [] }, request.targets, request.app); }, }), map((response) => { @@ -578,6 +559,20 @@ export class OpenSearchDatasource extends DataSourceWithBackend { + if (target.queryType === QueryType.PPL) { + return { + ...target, + query: this.templateSrv.replace(target.query, request.scopedVars, 'pipe'), + }; + } else { + return { + ...target, + query: this.templateSrv.replace(target.query, request.scopedVars, 'lucene') || '*', + }; + } + }); + const luceneTargets: OpenSearchQuery[] = []; const pplTargets: OpenSearchQuery[] = []; for (const target of targetsWithInterpolatedVariables) { @@ -746,10 +741,6 @@ export class OpenSearchDatasource extends DataSourceWithBackend { + it('should return true if the query contains the positive filter', () => { + expect(luceneQueryHasFilter('label:"value"', 'label', 'value')).toBe(true); + expect(luceneQueryHasFilter('label: "value"', 'label', 'value')).toBe(true); + expect(luceneQueryHasFilter('label : "value"', 'label', 'value')).toBe(true); + expect(luceneQueryHasFilter('label:value', 'label', 'value')).toBe(true); + expect(luceneQueryHasFilter('this:"that" AND label:value', 'label', 'value')).toBe(true); + expect(luceneQueryHasFilter('this:"that" OR (test:test AND label:value)', 'label', 'value')).toBe(true); + expect(luceneQueryHasFilter('this:"that" OR (test:test AND label:value)', 'test', 'test')).toBe(true); + expect(luceneQueryHasFilter('(this:"that" OR test:test) AND label:value', 'this', 'that')).toBe(true); + expect(luceneQueryHasFilter('(this:"that" OR test:test) AND label:value', 'test', 'test')).toBe(true); + expect(luceneQueryHasFilter('(this:"that" OR test :test) AND label:value', 'test', 'test')).toBe(true); + expect( + luceneQueryHasFilter( + 'message:"Jun 20 17:19:47 Xtorm syslogd[348]: ASL Sender Statistics"', + 'message', + 'Jun 20 17:19:47 Xtorm syslogd[348]: ASL Sender Statistics' + ) + ).toBe(true); + }); + it('should return false if the query does not contain the positive filter', () => { + expect(luceneQueryHasFilter('label:"value"', 'label', 'otherValue')).toBe(false); + expect(luceneQueryHasFilter('-label:"value"', 'label', 'value')).toBe(false); + expect(luceneQueryHasFilter('-this:"that" AND these:"those"', 'this', 'those')).toBe(false); + }); + it('should return true if the query contains the negative filter', () => { + expect(luceneQueryHasFilter('-label:"value"', 'label', 'value', '-')).toBe(true); + expect(luceneQueryHasFilter('-label: "value"', 'label', 'value', '-')).toBe(true); + expect(luceneQueryHasFilter('-label : "value"', 'label', 'value', '-')).toBe(true); + expect(luceneQueryHasFilter('-label:value', 'label', 'value', '-')).toBe(true); + expect(luceneQueryHasFilter('this:"that" AND -label:value', 'label', 'value', '-')).toBe(true); + }); + it('should return false if the query does not contain the negative filter', () => { + expect(luceneQueryHasFilter('label:"value"', 'label', 'otherValue', '-')).toBe(false); + expect(luceneQueryHasFilter('label:"value"', 'label', 'value', '-')).toBe(false); + }); + it('should support filters containing colons', () => { + expect(luceneQueryHasFilter('label\\:name:"value"', 'label:name', 'value')).toBe(true); + expect(luceneQueryHasFilter('-label\\:name:"value"', 'label:name', 'value', '-')).toBe(true); + }); + it('should support filters containing quotes', () => { + expect(luceneQueryHasFilter('label\\:name:"some \\"value\\""', 'label:name', 'some "value"')).toBe(true); + expect(luceneQueryHasFilter('-label\\:name:"some \\"value\\""', 'label:name', 'some "value"', '-')).toBe(true); + }); +}); + +describe('addFilterToQuery', () => { + it('should add a positive filter to the query', () => { + expect(addAdHocFilterToLuceneQuery('', 'label', 'value')).toBe('label:"value"'); + }); + it('should add a positive filter to the query with other filters', () => { + expect(addAdHocFilterToLuceneQuery('label2:"value2"', 'label', 'value')).toBe('label2:"value2" AND label:"value"'); + }); + it('should add a negative filter to the query', () => { + expect(addAdHocFilterToLuceneQuery('', 'label', 'value', '-')).toBe('-label:"value"'); + }); + it('should add a negative filter to the query with other filters', () => { + expect(addAdHocFilterToLuceneQuery('label2:"value2"', 'label', 'value', '-')).toBe( + 'label2:"value2" AND -label:"value"' + ); + }); + it('should support filters with colons', () => { + expect(addAdHocFilterToLuceneQuery('', 'label:name', 'value')).toBe('label\\:name:"value"'); + }); + it('should support filters with quotes', () => { + expect(addAdHocFilterToLuceneQuery('', 'label:name', 'the "value"')).toBe('label\\:name:"the \\"value\\""'); + }); +}); + +describe('removeFilterFromLucene Query', () => { + it('should remove filter from query', () => { + expect(removeFilterFromLuceneQuery('label:"value"', 'label', 'value')).toBe(''); + }); + it('should remove filter from query with other filters', () => { + expect(removeFilterFromLuceneQuery('label:"value" AND label2:"value2"', 'label', 'value')).toBe('label2:"value2"'); + expect(removeFilterFromLuceneQuery('label:value AND label2:"value2"', 'label', 'value')).toBe('label2:"value2"'); + expect(removeFilterFromLuceneQuery('label : "value" OR label2:"value2"', 'label', 'value')).toBe('label2:"value2"'); + expect(removeFilterFromLuceneQuery('test:"test" OR label:"value" AND label2:"value2"', 'label', 'value')).toBe( + 'test:"test" OR label2:"value2"' + ); + expect(removeFilterFromLuceneQuery('test:"test" OR (label:"value" AND label2:"value2")', 'label', 'value')).toBe( + 'test:"test" OR label2:"value2"' + ); + expect(removeFilterFromLuceneQuery('(test:"test" OR label:"value") AND label2:"value2"', 'label', 'value')).toBe( + '(test:"test") AND label2:"value2"' + ); + expect(removeFilterFromLuceneQuery('(test:"test" OR label:"value") AND label2:"value2"', 'test', 'test')).toBe( + 'label:"value" AND label2:"value2"' + ); + expect(removeFilterFromLuceneQuery('test:"test" OR (label:"value" AND label2:"value2")', 'label2', 'value2')).toBe( + 'test:"test" OR (label:"value")' + ); + }); + it('should not remove the wrong filter', () => { + expect(removeFilterFromLuceneQuery('-label:"value" AND label2:"value2"', 'label', 'value')).toBe( + '-label:"value" AND label2:"value2"' + ); + expect(removeFilterFromLuceneQuery('label2:"value2" OR -label:value', 'label', 'value')).toBe( + 'label2:"value2" OR -label:value' + ); + expect(removeFilterFromLuceneQuery('-label : "value" OR label2:"value2"', 'label', 'value')).toBe( + '-label : "value" OR label2:"value2"' + ); + }); + it('should support filters with colons', () => { + expect(removeFilterFromLuceneQuery('label\\:name:"value"', 'label:name', 'value')).toBe(''); + }); + it('should support filters with quotes', () => { + expect(removeFilterFromLuceneQuery('label\\:name:"the \\"value\\""', 'label:name', 'the "value"')).toBe(''); + }); +}); + +describe('addStringFilterToQuery', () => { + it('should add a positive filter to a query', () => { + expect(addStringFilterToQuery('label:"value"', 'filter')).toBe('label:"value" AND "filter"'); + expect(addStringFilterToQuery('', 'filter')).toBe('"filter"'); + expect(addStringFilterToQuery(' ', 'filter')).toBe('"filter"'); + }); + + it('should add a negative filter to a query', () => { + expect(addStringFilterToQuery('label:"value"', 'filter', false)).toBe('label:"value" NOT "filter"'); + expect(addStringFilterToQuery('', 'filter', false)).toBe('NOT "filter"'); + expect(addStringFilterToQuery(' ', 'filter', false)).toBe('NOT "filter"'); + }); + + it('should escape filter values', () => { + expect(addStringFilterToQuery('label:"value"', '"filter"')).toBe('label:"value" AND "\\"filter\\""'); + expect(addStringFilterToQuery('label:"value"', '"filter"', false)).toBe('label:"value" NOT "\\"filter\\""'); + }); + + it('should escape filter values with backslashes', () => { + expect(addStringFilterToQuery('label:"value"', '"filter with \\"')).toBe( + 'label:"value" AND "\\"filter with \\\\\\""' + ); + expect(addStringFilterToQuery('label:"value"', '"filter with \\"', false)).toBe( + 'label:"value" NOT "\\"filter with \\\\\\""' + ); + }); +}); +describe('PPLQueryHasFilter', () => { + const adHocFilter: AdHocVariableFilter = { + key: 'AvgTicketPrice', + value: '904', + operator: '!=', + }; + it('should return true if the query contains the positive filter', () => { + expect(PPLQueryHasFilter('search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` != 904', adHocFilter)).toBe(true); + expect(PPLQueryHasFilter('search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` > 904', {...adHocFilter, operator: '>'})).toBe(true); + }); + it('should return false if the query does not contain the positive filter', () => { + expect(PPLQueryHasFilter('search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904', adHocFilter)).toBe(false); + }); +}); +describe('toggleQueryFilterForLucene', () => { + describe('FILTER_FOR', () => { + it('should add a positive filter to the query', () => { + const queryString = "AvgTicketPrice:400"; + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_FOR' + }; + expect(toggleQueryFilterForLucene(queryString, filter)).toBe("AvgTicketPrice:400 AND DestAirport:\"Paris\""); + }); + it('should add a negative filter to the query', () => { + const queryString = "AvgTicketPrice:400"; + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_OUT' + }; + expect(toggleQueryFilterForLucene(queryString, filter)).toBe("AvgTicketPrice:400 AND -DestAirport:\"Paris\""); + }); + it('should remove a positive filter if the query already contains it', () => { + const queryString = "AvgTicketPrice:400 AND DestAirport:\"Paris\""; + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_FOR' + }; + expect(toggleQueryFilterForLucene(queryString, filter)).toBe("AvgTicketPrice:400"); + }); + it('should remove a positive filter if a negative filter is passed, then add the negative filter', () => { + const queryString = "AvgTicketPrice:400 AND DestAirport:\"Paris\""; + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_OUT' + }; + expect(toggleQueryFilterForLucene(queryString, filter)).toBe("AvgTicketPrice:400 AND -DestAirport:\"Paris\""); + }); + }); +}) +describe('toggleQueryFilterForPPL', () => { + describe('FILTER_FOR', () => { + it('should add a positive filter to the query', () => { + const queryString = 'search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904'; + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_FOR' + }; + expect(toggleQueryFilterForPPL(queryString, filter)).toBe("search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904 | where `DestAirport` = 'Paris'"); + }); + it('should add a negative filter to the query', () => { + const queryString = 'search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904'; + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_OUT' + }; + expect(toggleQueryFilterForPPL(queryString, filter)).toBe("search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904 | where `DestAirport` != 'Paris'"); + }); + it('should remove a positive filter if the query already contains it', () => { + const queryString = "search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904 | where `DestAirport` = 'Paris'" + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_FOR' + }; + expect(toggleQueryFilterForPPL(queryString, filter)).toBe("search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904"); + }); + it('should remove a positive filter if a negative filter is passed, then add the negative filter', () => { + const queryString = "search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904 | where `DestAirport` = 'Paris'" + const filter: ToggleFilterAction = { + options: { + key: 'DestAirport', + value: 'Paris' + }, + type: 'FILTER_OUT' + }; + expect(toggleQueryFilterForPPL(queryString, filter)).toBe("search source=opensearch_dashboards_sample_data_flights | where `AvgTicketPrice` = 904 | where `DestAirport` != 'Paris'"); + }); + }); +}) diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts new file mode 100644 index 00000000..c6a349a0 --- /dev/null +++ b/src/modifyQuery.ts @@ -0,0 +1,362 @@ +import { isEqual } from 'lodash'; +import lucene, { AST, BinaryAST, LeftOnlyAST, NodeTerm } from 'lucene'; + +import { AdHocVariableFilter, ToggleFilterAction, dateMath, dateTime } from '@grafana/data'; + +type ModifierType = '' | '-'; + +/** + * Adds a label:"value" expression to the query. + */ +export function addLuceneAdHocFilter(query: string, filter: AdHocVariableFilter): string { + if (!filter.key || !filter.value) { + return query; + } + + filter = { + ...filter, + // Type is defined as string, but it can be a number. + value: filter.value.toString(), + }; + + const equalityFilters = ['=', '!=']; + if (equalityFilters.includes(filter.operator)) { + return addAdHocFilterToLuceneQuery(query, filter.key, filter.value, filter.operator === '=' ? '' : '-'); + } + /** + * Keys and values in ad hoc filters may contain characters such as + * colons, which needs to be escaped. + */ + const key = escapeFilter(filter.key); + const value = escapeFilterValue(filter.value); + let addHocFilter = ''; + switch (filter.operator) { + case '=~': + addHocFilter = `${key}:/${value}/`; + break; + case '!~': + addHocFilter = `-${key}:/${value}/`; + break; + case '>': + addHocFilter = `${key}:>${value}`; + break; + case '<': + addHocFilter = `${key}:<${value}`; + break; + } + return concatenate(query, addHocFilter); +} + +/** + * Adds a label:"value" expression to the query. + */ +export function addAdHocFilterToLuceneQuery( + query: string, + key: string, + value: string, + modifier: ModifierType = '' +): string { + if (luceneQueryHasFilter(query, key, value, modifier)) { + return query; + } + + key = escapeFilter(key); + value = escapeFilterValue(value); + const filter = `${modifier}${key}:"${value}"`; + return concatenate(query, filter); +} + +/** + * Checks for the presence of a given label:"value" filter in the query. + */ +export function luceneQueryHasFilter(query: string, key: string, value: string, modifier: ModifierType = ''): boolean { + return findFilterNode(query, key, value, modifier) !== null; +} + +/** + * Merge a query with a filter. + */ +function concatenate(query: string, filter: string, condition = 'AND'): string { + if (!filter) { + return query; + } + return query.trim() === '' ? filter : `${query} ${condition} ${filter}`; +} + +/** + * Removes a label:"value" expression from the query. + */ +export function removeFilterFromLuceneQuery( + query: string, + key: string, + value: string, + modifier: ModifierType = '' +): string { + const node = findFilterNode(query, key, value, modifier); + const ast = parseQuery(query); + if (!node || !ast) { + return query; + } + + return lucene.toString(removeNodeFromTree(ast, node)); +} + +/** + * Given a query, find the NodeTerm that matches the given field and value. + */ +export function findFilterNode( + query: string, + key: string, + value: string, + modifier: ModifierType = '' +): NodeTerm | null { + const field = `${modifier}${lucene.term.escape(key)}`; + value = lucene.phrase.escape(value); + let ast: AST | null = parseQuery(query); + if (!ast) { + return null; + } + + return findNodeInTree(ast, field, value); +} + +function findNodeInTree(ast: AST, field: string, value: string): NodeTerm | null { + // {} + if (Object.keys(ast).length === 0) { + return null; + } + // { left: {}, right: {} } or { left: {} } + if (isAST(ast.left)) { + return findNodeInTree(ast.left, field, value); + } + if (isNodeTerm(ast.left) && ast.left.field === field && ast.left.term === value) { + return ast.left; + } + if (isLeftOnlyAST(ast)) { + return null; + } + if (isNodeTerm(ast.right) && ast.right.field === field && ast.right.term === value) { + return ast.right; + } + if (isBinaryAST(ast.right)) { + return findNodeInTree(ast.right, field, value); + } + return null; +} + +function removeNodeFromTree(ast: AST, node: NodeTerm): AST { + // {} + if (Object.keys(ast).length === 0) { + return ast; + } + // { left: {}, right: {} } or { left: {} } + if (isAST(ast.left)) { + ast.left = removeNodeFromTree(ast.left, node); + return ast; + } + if (isNodeTerm(ast.left) && isEqual(ast.left, node)) { + Object.assign( + ast, + { + left: undefined, + operator: undefined, + right: undefined, + }, + 'right' in ast ? ast.right : {} + ); + return ast; + } + if (isLeftOnlyAST(ast)) { + return ast; + } + if (isNodeTerm(ast.right) && isEqual(ast.right, node)) { + Object.assign(ast, { + right: undefined, + operator: undefined, + }); + return ast; + } + if (isBinaryAST(ast.right)) { + ast.right = removeNodeFromTree(ast.right, node); + return ast; + } + return ast; +} + +/** + * Filters can possibly reserved characters such as colons which are part of the Lucene syntax. + * Use this function to escape filter keys. + */ +export function escapeFilter(value: string) { + return lucene.term.escape(value); +} + +/** + * Values can possibly reserved special characters such as quotes. + * Use this function to escape filter values. + */ +export function escapeFilterValue(value: string) { + value = value.replace(/\\/g, '\\\\'); + return lucene.phrase.escape(value); +} + +/** + * Normalizes the query by removing whitespace around colons, which breaks parsing. + */ +function normalizeQuery(query: string) { + return query.replace(/(\w+)\s(:)/gi, '$1$2'); +} + +function isLeftOnlyAST(ast: unknown): ast is LeftOnlyAST { + if (!ast || typeof ast !== 'object') { + return false; + } + + if ('left' in ast && !('right' in ast)) { + return true; + } + + return false; +} + +function isBinaryAST(ast: unknown): ast is BinaryAST { + if (!ast || typeof ast !== 'object') { + return false; + } + + if ('left' in ast && 'right' in ast) { + return true; + } + return false; +} + +function isAST(ast: unknown): ast is AST { + return isLeftOnlyAST(ast) || isBinaryAST(ast); +} + +function isNodeTerm(ast: unknown): ast is NodeTerm { + if (ast && typeof ast === 'object' && 'term' in ast) { + return true; + } + + return false; +} + +function parseQuery(query: string) { + try { + return lucene.parse(normalizeQuery(query)); + } catch (e) { + return null; + } +} + +export function addStringFilterToQuery(query: string, filter: string, contains = true) { + const expression = `"${escapeFilterValue(filter)}"`; + return query.trim() ? `${query} ${contains ? 'AND' : 'NOT'} ${expression}` : `${contains ? '' : 'NOT '}${expression}`; +} + +function isNotANumber(value: string) { + return isNaN(Number(value)); +} + +function getAdHocPPLQuery(filter: AdHocVariableFilter): string { + let value = ''; + + if ('=~' === filter.operator || '!~' === filter.operator) { + return ''; + } + if (dateMath.isValid(filter.value)) { + const validTime = dateTime(filter.value).utc().format('YYYY-MM-DD HH:mm:ss.SSSSSS'); + value = `timestamp('${validTime}')`; + } else if (typeof filter.value === 'string' && isNotANumber(filter.value)) { + value = `'${filter.value}'`; + } else { + value = filter.value; + } + return `\`${filter.key}\` ${filter.operator} ${value}`; +} +export function addAdhocFilterToPPLQuery(queryString: any, filter: AdHocVariableFilter, i?: number): string { + if (!filter.key || !filter.value) { + return queryString; + } + const adHocQuery: string = getAdHocPPLQuery(filter); + + if (adHocQuery !== '') { + if (queryString === '') { + return adHocQuery; + } + // originally, the query string added '| where' to the query if the filter was the first filter; + // however we have no way of knowing this since toggleQueryFilter called from Explore just passes the current filter + // this should still work even though the filtering is a but different than the original implementation + if (i && i > 0) { + queryString += ' and ' + adHocQuery; + } else { + queryString += ' | where ' + adHocQuery; + } + } + return queryString; +} +export function removeFilterFromPPLQuery(query: string, filter: AdHocVariableFilter): string { + const adHocQuery: string = getAdHocPPLQuery(filter); + return query.replace(` | where ${adHocQuery}`, ''); +} +export function PPLQueryHasFilter(query: string, filter: AdHocVariableFilter): boolean { + const adHocQuery: string = getAdHocPPLQuery(filter); + return query.includes(adHocQuery); +} +export function toggleQueryFilterForLucene(queryString: string, filter: ToggleFilterAction): string { + let expression = queryString; + switch (filter.type) { + case 'FILTER_FOR': { + // This gives the user the ability to toggle a filter on and off. + expression = luceneQueryHasFilter(expression, filter.options.key, filter.options.value) + ? removeFilterFromLuceneQuery(expression, filter.options.key, filter.options.value) + : addAdHocFilterToLuceneQuery(expression, filter.options.key, filter.options.value); + break; + } + case 'FILTER_OUT': { + // If the opposite filter is present, remove it before adding the new one. + if (luceneQueryHasFilter(expression, filter.options.key, filter.options.value)) { + expression = removeFilterFromLuceneQuery(expression, filter.options.key, filter.options.value); + } + expression = addAdHocFilterToLuceneQuery(expression, filter.options.key, filter.options.value, '-'); + break; + } + } + return expression; +} +export function toggleQueryFilterForPPL(queryString: string, filter: ToggleFilterAction): string { + let expression = queryString; + switch (filter.type) { + case 'FILTER_FOR': { + const adHocFilter: AdHocVariableFilter = { + key: filter.options.key, + value: filter.options.value, + operator: '=', + }; + // This gives the user the ability to toggle a filter on and off. + expression = PPLQueryHasFilter(expression, adHocFilter) + ? removeFilterFromPPLQuery(expression, adHocFilter) + : addAdhocFilterToPPLQuery(expression, adHocFilter); + break; + } + case 'FILTER_OUT': { + const adHocFilter: AdHocVariableFilter = { + key: filter.options.key, + value: filter.options.value, + operator: '!=', + }; + // If the opposite filter is present, remove it before adding the new one. + const oppositeFilter: AdHocVariableFilter = { + ...adHocFilter, + operator: '=', + }; + if (PPLQueryHasFilter(expression, oppositeFilter)) { + expression = removeFilterFromPPLQuery(expression, oppositeFilter); + } + expression = addAdhocFilterToPPLQuery(expression, adHocFilter); + break; + } + } + return expression; +} diff --git a/yarn.lock b/yarn.lock index d51f5578..cc51702e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2234,6 +2234,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.202.tgz#f09dbd2fb082d507178b2f2a5c7e74bd72ff98f8" integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ== +"@types/lucene@^2.1.7": + version "2.1.7" + resolved "https://registry.yarnpkg.com/@types/lucene/-/lucene-2.1.7.tgz#fbdea914c5b7d91fd164664ccc6019ed210e729b" + integrity sha512-i3J0OV0RoJSskOJUa76Hgz09deabWwfJajsUxc1M05HryjPpPEKqtRklKe0+O0XVhdrFIiFO1/SInXpDCacfNA== + "@types/node@*", "@types/node@^20.8.7": version "20.10.4" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.4.tgz#b246fd84d55d5b1b71bf51f964bd514409347198" @@ -6349,6 +6354,11 @@ lru-cache@^6.0.0: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.1.0.tgz#2098d41c2dc56500e6c88584aa656c84de7d0484" integrity sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag== +lucene@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/lucene/-/lucene-2.1.1.tgz#e710cc123b214eaf72a4c5f1da06943c0af44d86" + integrity sha512-l0qCX+pgXEZh/7sYQNG+vzhOIFRPjlJJkQ/irk9n7Ak3d+1MrU6F7IV31KILwFkUn153oLK8a2AIt48DzLdVPg== + lz-string@^1.4.4, lz-string@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941"