Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Lens] Adds using queries/filters for field existence endpoint #59033

Merged
merged 17 commits into from
Mar 12, 2020
Merged

[Lens] Adds using queries/filters for field existence endpoint #59033

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
967504e
fix: respect filters and query for displaying field existence
mbondyra Mar 2, 2020
b04f250
test: added dslQuery to test
mbondyra Mar 2, 2020
be49d62
test: fix tests
mbondyra Mar 3, 2020
21429ea
fix: fix types
mbondyra Mar 3, 2020
6b692ab
fix: integration tests
mbondyra Mar 3, 2020
f2aca43
test: add integration test
mbondyra Mar 4, 2020
97e560b
refactor: add default
mbondyra Mar 5, 2020
836d1e1
test: corrected id for new entry
mbondyra Mar 5, 2020
770d241
refactor: change to post
mbondyra Mar 5, 2020
61f2b35
types
mbondyra Mar 5, 2020
57cea75
eslint
mbondyra Mar 5, 2020
c1043c2
test: fixes
mbondyra Mar 5, 2020
a496d75
Merge branch 'master' into IS-52826_field-existence-endpoint-respects…
mbondyra Mar 5, 2020
beb0a7a
test
mbondyra Mar 6, 2020
4afd03f
fix test
mbondyra Mar 6, 2020
7f3a136
Merge branch 'master' into IS-52826_field-existence-endpoint-respects…
elasticmachine Mar 9, 2020
7993eef
Merge remote-tracking branch 'origin/master' into HEAD
wylieconlon Mar 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,9 @@ const initialState: IndexPatternPrivateState = {
},
},
};

const dslQuery = '{"bool":{"must":[{"match_all":{}}],"filter":[],"should":[],"must_not":[]}}';

describe('IndexPattern Data Panel', () => {
let defaultProps: Parameters<typeof InnerIndexPatternDataPanel>[0];
let core: ReturnType<typeof coreMock['createSetup']>;
Expand Down Expand Up @@ -400,6 +403,7 @@ describe('IndexPattern Data Panel', () => {
expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/a',
query: {
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-01',
timeFieldName: 'atime',
Expand All @@ -409,6 +413,7 @@ describe('IndexPattern Data Panel', () => {
expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/a',
query: {
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-02',
timeFieldName: 'atime',
Expand Down Expand Up @@ -441,6 +446,7 @@ describe('IndexPattern Data Panel', () => {
expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/a',
query: {
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-01',
timeFieldName: 'atime',
Expand All @@ -450,6 +456,7 @@ describe('IndexPattern Data Panel', () => {
expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/b',
query: {
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-01',
timeFieldName: 'btime',
Expand Down
13 changes: 13 additions & 0 deletions x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import { trackUiEvent } from '../lens_ui_telemetry';
import { syncExistingFields } from './loader';
import { fieldExists } from './pure_helpers';
import { Loader } from '../loader';
import { esQuery, IIndexPattern } from '../../../../../../src/plugins/data/public';

export type Props = DatasourceDataPanelProps<IndexPatternPrivateState> & {
changeIndexPattern: (
Expand Down Expand Up @@ -113,6 +114,15 @@ export function IndexPatternDataPanel({
timeFieldName: indexPatterns[id].timeFieldName,
}));

const dslQuery = JSON.stringify(
esQuery.buildEsQuery(
indexPatterns[currentIndexPatternId] as IIndexPattern,
query,
filters,
esQuery.getEsQueryConfig(core.uiSettings)
)
);

return (
<>
<Loader
Expand All @@ -122,9 +132,12 @@ export function IndexPatternDataPanel({
setState,
indexPatterns: indexPatternList,
fetchJson: core.http.get,
dslQuery,
})
}
loadDeps={[
query,
filters,
dateRange.fromDate,
dateRange.toDate,
indexPatternList.map(x => `${x.title}:${x.timeFieldName}`).join(','),
Expand Down
10 changes: 10 additions & 0 deletions x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,15 @@ describe('loader', () => {
});

describe('syncExistingFields', () => {
const dslQuery = JSON.stringify({
bool: {
must: [],
filter: [{ match_all: {} }],
should: [],
must_not: [],
},
});

it('should call once for each index pattern', async () => {
const setState = jest.fn();
const fetchJson = jest.fn(({ path }: { path: string }) => {
Expand All @@ -553,6 +562,7 @@ describe('loader', () => {
fetchJson: fetchJson as any,
indexPatterns: [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
setState,
dslQuery,
});

expect(fetchJson).toHaveBeenCalledTimes(3);
Expand Down
3 changes: 3 additions & 0 deletions x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,15 +215,18 @@ export async function syncExistingFields({
dateRange,
fetchJson,
setState,
dslQuery,
}: {
dateRange: DateRange;
indexPatterns: Array<{ id: string; timeFieldName?: string | null }>;
fetchJson: HttpSetup['get'];
setState: SetState;
dslQuery: string;
}) {
const emptinessInfo = await Promise.all(
indexPatterns.map(pattern => {
const query: Record<string, string> = {
dslQuery,
fromDate: dateRange.fromDate,
toDate: dateRange.toDate,
};
Expand Down
33 changes: 19 additions & 14 deletions x-pack/plugins/lens/server/routes/existing_fields.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export async function existingFieldsRoute(setup: CoreSetup) {
indexPatternId: schema.string(),
}),
query: schema.object({
dslQuery: schema.string(),
fromDate: schema.maybe(schema.string()),
toDate: schema.maybe(schema.string()),
timeFieldName: schema.maybe(schema.string()),
Expand Down Expand Up @@ -91,12 +92,14 @@ export async function existingFieldsRoute(setup: CoreSetup) {
async function fetchFieldExistence({
context,
indexPatternId,
dslQuery,
fromDate,
toDate,
timeFieldName,
}: {
indexPatternId: string;
context: RequestHandlerContext;
dslQuery: string;
fromDate?: string;
toDate?: string;
timeFieldName?: string;
Expand All @@ -113,6 +116,7 @@ async function fetchFieldExistence({
const docs = await fetchIndexPatternStats({
fromDate,
toDate,
dslQuery: JSON.parse(dslQuery),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: You've chosen to keep this as a GET request, so the query is passed as a string. Would it make sense to turn this into a POST request with a query body?

client: context.core.elasticsearch.dataClient,
index: indexPatternTitle,
timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName,
Expand Down Expand Up @@ -197,24 +201,23 @@ export function buildFieldList(
async function fetchIndexPatternStats({
client,
index,
dslQuery,
timeFieldName,
fromDate,
toDate,
fields,
}: {
client: IScopedClusterClient;
index: string;
dslQuery: string;
timeFieldName?: string;
fromDate?: string;
toDate?: string;
fields: Field[];
}) {
let query;

if (timeFieldName && fromDate && toDate) {
query = {
bool: {
filter: [
const filter =
timeFieldName && fromDate && toDate
? [
{
range: {
[timeFieldName]: {
Expand All @@ -223,14 +226,16 @@ async function fetchIndexPatternStats({
},
},
},
],
},
};
} else {
query = {
match_all: {},
};
}
dslQuery,
]
: [dslQuery];

const query = {
bool: {
filter,
},
};

const scriptedFields = fields.filter(f => f.isScript);

const result = await client.callAsCurrentUser('search', {
Expand Down
54 changes: 48 additions & 6 deletions x-pack/test/api_integration/apis/lens/existing_fields.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';

const TEST_START_TIME = encodeURIComponent('2015-09-19T06:31:44.000');
const TEST_END_TIME = encodeURIComponent('2015-09-23T18:31:44.000');
const DSL_QUERY = JSON.stringify({ match_all: {} });
const COMMON_HEADERS = {
'kbn-xsrf': 'some-xsrf-token',
};
Expand Down Expand Up @@ -150,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => {
.get(
`/api/lens/existing_fields/${encodeURIComponent(
'logstash-*'
)}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
)}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}&dslQuery=${DSL_QUERY}`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: dslQuery is now required, but could we provide a default match_all query? But since this is only used by our client, this is probably fine.

)
.set(COMMON_HEADERS)
.expect(200);
Expand All @@ -164,7 +165,7 @@ export default ({ getService }: FtrProviderContext) => {
.get(
`/api/lens/existing_fields/${encodeURIComponent(
'metricbeat-*'
)}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
)}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}&dslQuery=${DSL_QUERY}`
)
.set(COMMON_HEADERS)
.expect(200);
Expand All @@ -173,13 +174,54 @@ export default ({ getService }: FtrProviderContext) => {
expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort());
});

it('should throw a 404 for a non-existent index', async () => {
await supertest
it('should return fields filtered by query and filters', async () => {
const expectedFieldNames = [
'@message',
'@message.raw',
'@tags',
'@tags.raw',
'@timestamp',
'agent',
'agent.raw',
'bytes',
'clientip',
'extension',
'extension.raw',
'headings',
'headings.raw',
'host',
'host.raw',
'index',
'index.raw',
'referer',
'request',
'request.raw',
'response',
'response.raw',
'spaces',
'spaces.raw',
'type',
'url',
'url.raw',
'utc_time',
'xss',
'xss.raw',
];

const dslQueryFiltered = JSON.stringify({
bool: {
filter: [{ match: { referer: 'https://www.taylorswift.com/' } }],
},
});
const { body } = await supertest
.get(
`/api/lens/existing_fields/nadachance?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
`/api/lens/existing_fields/${encodeURIComponent(
'logstash-*'
)}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}&dslQuery=${dslQueryFiltered}`
)
.set(COMMON_HEADERS)
.expect(404);
.expect(200);
expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort());
});
});
});
Expand Down
Binary file modified x-pack/test/functional/es_archives/logstash_functional/data.json.gz
View file
Open in desktop
Binary file not shown.