Skip to content

Commit

Permalink
[ML] Fix custom URLs processing for security app (elastic#76957)
Browse files Browse the repository at this point in the history 
* [ML] fix custom urls processing for security app

* [ML] improve query string parsing

* [ML] remove escaping with !, adjust a unit test for security app

* [ML] unit test

* [ML] unit test
  • Loading branch information
@darnautov
darnautov committed Sep 14, 2020
1 parent 0e12028 commit 20648c2
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 60 deletions.
109 changes: 106 additions & 3 deletions x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,13 @@ describe('ML - custom URL utils', () => {
influencer_field_name: 'airline',
influencer_field_values: ['<>:;[}")'],
},
{
influencer_field_name: 'odd:field,name',
influencer_field_values: [">:&12<'"],
},
],
airline: ['<>:;[}")'],
'odd:field,name': [">:&12<'"],
};

const TEST_RECORD_MULTIPLE_INFLUENCER_VALUES: CustomUrlAnomalyRecordDoc = {
Expand Down Expand Up @@ -97,7 +102,7 @@ describe('ML - custom URL utils', () => {
url_name: 'Raw data',
time_range: 'auto',
url_value:
"discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"$airline$\"'))",
"discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"$airline$\" and odd:field,name : $odd:field,name$'))",
};

const TEST_DASHBOARD_LUCENE_URL: KibanaUrlConfig = {
Expand Down Expand Up @@ -263,9 +268,55 @@ describe('ML - custom URL utils', () => {
);
});

test('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => {
test.skip('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => {
expect(getUrlForRecord(TEST_DISCOVER_URL, TEST_RECORD_SPECIAL_CHARS)).toBe(
"discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\"'))"
"discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\" and odd:field,name:>:&12<''))"
);
});

test('correctly encodes special characters inside of a query string', () => {
const testUrl = {
url_name: 'Show dashboard',
time_range: 'auto',
url_value: `dashboards#/view/351de820-f2bb-11ea-ab06-cb93221707e9?_a=(filters:!(),query:(language:kuery,query:'at@name:"$at@name$" and singlequote!'name:"$singlequote!'name$"'))&_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))`,
};

const testRecord = {
job_id: 'spec-char',
result_type: 'record',
probability: 0.0028099428534745633,
multi_bucket_impact: 5,
record_score: 49.00785814424704,
initial_record_score: 49.00785814424704,
bucket_span: 900,
detector_index: 0,
is_interim: false,
timestamp: 1549593000000,
partition_field_name: 'at@name',
partition_field_value: "contains a ' quote",
function: 'mean',
function_description: 'mean',
typical: [1993.2657340111837],
actual: [1808.3334418402778],
field_name: 'metric%$£&!{(]field',
influencers: [
{
influencer_field_name: "singlequote'name",
influencer_field_values: ["contains a ' quote"],
},
{
influencer_field_name: 'at@name',
influencer_field_values: ["contains a ' quote"],
},
],
"singlequote'name": ["contains a ' quote"],
'at@name': ["contains a ' quote"],
earliest: '2019-02-08T00:00:00.000Z',
latest: '2019-02-08T23:59:59.999Z',
};

expect(getUrlForRecord(testUrl, testRecord)).toBe(
`dashboards#/view/351de820-f2bb-11ea-ab06-cb93221707e9?_a=(filters:!(),query:(language:kuery,query:'at@name:"contains%20a%20!'%20quote" AND singlequote!'name:"contains%20a%20!'%20quote"'))&_g=(filters:!(),time:(from:'2019-02-08T00:00:00.000Z',mode:absolute,to:'2019-02-08T23:59:59.999Z'))`
);
});

Expand Down Expand Up @@ -405,6 +456,58 @@ describe('ML - custom URL utils', () => {
);
});

test('return expected url for Security app', () => {
const urlConfig = {
url_name: 'Hosts Details by process name',
url_value:
"security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))",
};

const testRecords = {
job_id: 'rare_process_by_host_linux_ecs',
result_type: 'record',
probability: 0.018122957282324745,
multi_bucket_impact: 0,
record_score: 20.513469583273547,
initial_record_score: 20.513469583273547,
bucket_span: 900,
detector_index: 0,
is_interim: false,
timestamp: 1549043100000,
by_field_name: 'process.name',
by_field_value: 'seq',
partition_field_name: 'host.name',
partition_field_value: 'showcase',
function: 'rare',
function_description: 'rare',
typical: [0.018122957282324745],
actual: [1],
influencers: [
{
influencer_field_name: 'user.name',
influencer_field_values: ['sophie'],
},
{
influencer_field_name: 'process.name',
influencer_field_values: ['seq'],
},
{
influencer_field_name: 'host.name',
influencer_field_values: ['showcase'],
},
],
'process.name': ['seq'],
'user.name': ['sophie'],
'host.name': ['showcase'],
earliest: '2019-02-01T16:00:00.000Z',
latest: '2019-02-01T18:59:59.999Z',
};

expect(getUrlForRecord(urlConfig, testRecords)).toBe(
"security/hosts/ml-hosts/showcase?_g=()&query=(language:kuery,query:'process.name:\"seq\"')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))"
);
});

test('removes an empty path component with a trailing slash', () => {
const urlConfig = {
url_name: 'APM',
Expand Down
137 changes: 80 additions & 57 deletions x-pack/plugins/ml/public/application/util/custom_url_utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import { get, flow } from 'lodash';
import moment from 'moment';
import rison, { RisonObject, RisonValue } from 'rison-node';

import { parseInterval } from '../../../common/util/parse_interval';
import { escapeForElasticsearchQuery, replaceStringTokens } from './string_utils';
Expand Down Expand Up @@ -113,25 +114,78 @@ function escapeForKQL(value: string | number): string {

type GetResultTokenValue = (v: string) => string;

export const isRisonObject = (value: RisonValue): value is RisonObject => {
return value !== null && typeof value === 'object';
};

const getQueryStringResultProvider = (
record: CustomUrlAnomalyRecordDoc,
getResultTokenValue: GetResultTokenValue
) => (resultPrefix: string, queryString: string, resultPostfix: string): string => {
const URL_LENGTH_LIMIT = 2000;

let availableCharactersLeft = URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length;

// URL template might contain encoded characters
const queryFields = queryString
// Split query string by AND operator.
.split(/\sand\s/i)
// Get property name from `influencerField:$influencerField$` string.
.map((v) => String(v.split(/:(.+)?\$/)[0]).trim());

const queryParts: string[] = [];
const joinOperator = ' AND ';

fieldsLoop: for (let i = 0; i < queryFields.length; i++) {
const field = queryFields[i];
// Use lodash get to allow nested JSON fields to be retrieved.
let tokenValues: string[] | string | null = get(record, field) || null;
if (tokenValues === null) {
continue;
}
tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues];

// Create a pair `influencerField:value`.
// In cases where there are multiple influencer field values for an anomaly
// combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`.
let result = '';
for (let j = 0; j < tokenValues.length; j++) {
const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue(tokenValues[j])}"`;

// Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query.
if (availableCharactersLeft < part.length) {
if (result.length > 0) {
queryParts.push(j > 0 ? `(${result})` : result);
}
break fieldsLoop;
}

result += part;

availableCharactersLeft -= result.length;
}

if (result.length > 0) {
queryParts.push(tokenValues.length > 1 ? `(${result})` : result);
}
}
return queryParts.join(joinOperator);
};

/**
* Builds a Kibana dashboard or Discover URL from the supplied config, with any
* dollar delimited tokens substituted from the supplied anomaly record.
*/
function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) {
const urlValue = urlConfig.url_value;
const URL_LENGTH_LIMIT = 2000;

const isLuceneQueryLanguage = urlValue.includes('language:lucene');

const queryLanguageEscapeCallback = isLuceneQueryLanguage
? escapeForElasticsearchQuery
: escapeForKQL;

const commonEscapeCallback = flow(
// Kibana URLs used rison encoding, so escape with ! any ! or ' characters
(v: string): string => v.replace(/[!']/g, '!$&'),
encodeURIComponent
);
const commonEscapeCallback = flow(encodeURIComponent);

const replaceSingleTokenValues = (str: string) => {
const getResultTokenValue: GetResultTokenValue = flow(
Expand All @@ -154,65 +208,34 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
return flow(
(str: string) => str.replace('$earliest$', record.earliest).replace('$latest$', record.latest),
// Process query string content of the URL
decodeURIComponent,
(str: string) => {
const getResultTokenValue: GetResultTokenValue = flow(
queryLanguageEscapeCallback,
commonEscapeCallback
);

const getQueryStringResult = getQueryStringResultProvider(record, getResultTokenValue);

const match = str.match(/(.+)(\(.*\blanguage:(?:lucene|kuery)\b.*?\))(.+)/);

if (match !== null && match[2] !== undefined) {
const [, prefix, queryDef, postfix] = match;

const q = rison.decode(queryDef);

if (isRisonObject(q) && q.hasOwnProperty('query')) {
const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues);
const resultQuery = getQueryStringResult(resultPrefix, q.query as string, resultPostfix);
return `${resultPrefix}${rison.encode({ ...q, query: resultQuery })}${resultPostfix}`;
}
}

return str.replace(
/(.+query:'|.+&kuery=)([^']*)(['&].+)/,
/(.+&kuery=)(.*?)[^!](&.+)/,
(fullMatch, prefix: string, queryString: string, postfix: string) => {
const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues);

let availableCharactersLeft =
URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length;
const queryFields = queryString
// Split query string by AND operator.
.split(/\sand\s/i)
// Get property name from `influencerField:$influencerField$` string.
.map((v) => v.split(':')[0]);

const queryParts: string[] = [];
const joinOperator = ' AND ';

fieldsLoop: for (let i = 0; i < queryFields.length; i++) {
const field = queryFields[i];
// Use lodash get to allow nested JSON fields to be retrieved.
let tokenValues: string[] | string | null = get(record, field) || null;
if (tokenValues === null) {
continue;
}
tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues];

// Create a pair `influencerField:value`.
// In cases where there are multiple influencer field values for an anomaly
// combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`.
let result = '';
for (let j = 0; j < tokenValues.length; j++) {
const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue(
tokenValues[j]
)}"`;

// Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query.
if (availableCharactersLeft < part.length) {
if (result.length > 0) {
queryParts.push(j > 0 ? `(${result})` : result);
}
break fieldsLoop;
}

result += part;

availableCharactersLeft -= result.length;
}

if (result.length > 0) {
queryParts.push(tokenValues.length > 1 ? `(${result})` : result);
}
}

const resultQuery = queryParts.join(joinOperator);

const resultQuery = getQueryStringResult(resultPrefix, queryString, resultPostfix);
return `${resultPrefix}${resultQuery}${resultPostfix}`;
}
);
Expand Down

0 comments on commit 20648c2

Please sign in to comment.