Skip to content

Commit

Permalink
[Lens] Adds using queries/filters for field existence endpoint (elast…
Browse files Browse the repository at this point in the history 
…ic#59033)
  • Loading branch information
@mbondyra
mbondyra committed Mar 12, 2020
1 parent c6b7964 commit b61c309
Show file tree
Hide file tree
Showing 8 changed files with 157 additions and 81 deletions.
47 changes: 25 additions & 22 deletions x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,9 @@ const initialState: IndexPatternPrivateState = {
},
},
};

const dslQuery = { bool: { must: [{ match_all: {} }], filter: [], should: [], must_not: [] } };

describe('IndexPattern Data Panel', () => {
let defaultProps: Parameters<typeof InnerIndexPatternDataPanel>[0];
let core: ReturnType<typeof coreMock['createSetup']>;
Expand Down Expand Up @@ -271,8 +274,8 @@ describe('IndexPattern Data Panel', () => {
describe('loading existence data', () => {
function testProps() {
const setState = jest.fn();
core.http.get.mockImplementation(async ({ path }) => {
const parts = path.split('/');
core.http.post.mockImplementation(async path => {
const parts = ((path as unknown) as string).split('/');
const indexPatternTitle = parts[parts.length - 1];
return {
indexPatternTitle: `${indexPatternTitle}_testtitle`,
Expand Down Expand Up @@ -385,24 +388,24 @@ describe('IndexPattern Data Panel', () => {
});

expect(setState).toHaveBeenCalledTimes(2);
expect(core.http.get).toHaveBeenCalledTimes(2);
expect(core.http.post).toHaveBeenCalledTimes(2);

expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/a',
query: {
expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
body: JSON.stringify({
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-01',
timeFieldName: 'atime',
},
}),
});

expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/a',
query: {
expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
body: JSON.stringify({
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-02',
timeFieldName: 'atime',
},
}),
});

const nextState = setState.mock.calls[1][0]({
Expand All @@ -428,22 +431,22 @@ describe('IndexPattern Data Panel', () => {

expect(setState).toHaveBeenCalledTimes(2);

expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/a',
query: {
expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
body: JSON.stringify({
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-01',
timeFieldName: 'atime',
},
}),
});

expect(core.http.get).toHaveBeenCalledWith({
path: '/api/lens/existing_fields/b',
query: {
expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/b', {
body: JSON.stringify({
dslQuery,
fromDate: '2019-01-01',
toDate: '2020-01-01',
timeFieldName: 'btime',
},
}),
});

const nextState = setState.mock.calls[1][0]({
Expand Down Expand Up @@ -476,13 +479,13 @@ describe('IndexPattern Data Panel', () => {
let overlapCount = 0;
const props = testProps();

core.http.get.mockImplementation(({ path }) => {
core.http.post.mockImplementation(path => {
if (queryCount) {
++overlapCount;
}
++queryCount;

const parts = path.split('/');
const parts = ((path as unknown) as string).split('/');
const indexPatternTitle = parts[parts.length - 1];
const result = Promise.resolve({
indexPatternTitle,
Expand Down Expand Up @@ -516,7 +519,7 @@ describe('IndexPattern Data Panel', () => {
inst.update();
});

expect(core.http.get).toHaveBeenCalledTimes(2);
expect(core.http.post).toHaveBeenCalledTimes(2);
expect(overlapCount).toEqual(0);
});
});
Expand Down
13 changes: 12 additions & 1 deletion x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import { trackUiEvent } from '../lens_ui_telemetry';
import { syncExistingFields } from './loader';
import { fieldExists } from './pure_helpers';
import { Loader } from '../loader';
import { esQuery, IIndexPattern } from '../../../../../../src/plugins/data/public';

export type Props = DatasourceDataPanelProps<IndexPatternPrivateState> & {
changeIndexPattern: (
Expand Down Expand Up @@ -113,6 +114,13 @@ export function IndexPatternDataPanel({
timeFieldName: indexPatterns[id].timeFieldName,
}));

const dslQuery = esQuery.buildEsQuery(
indexPatterns[currentIndexPatternId] as IIndexPattern,
query,
filters,
esQuery.getEsQueryConfig(core.uiSettings)
);

return (
<>
<Loader
Expand All @@ -121,10 +129,13 @@ export function IndexPatternDataPanel({
dateRange,
setState,
indexPatterns: indexPatternList,
fetchJson: core.http.get,
fetchJson: core.http.post,
dslQuery,
})
}
loadDeps={[
query,
filters,
dateRange.fromDate,
dateRange.toDate,
indexPatternList.map(x => `${x.title}:${x.timeFieldName}`).join(','),
Expand Down
12 changes: 11 additions & 1 deletion x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,9 +535,18 @@ describe('loader', () => {
});

describe('syncExistingFields', () => {
const dslQuery = {
bool: {
must: [],
filter: [{ match_all: {} }],
should: [],
must_not: [],
},
};

it('should call once for each index pattern', async () => {
const setState = jest.fn();
const fetchJson = jest.fn(({ path }: { path: string }) => {
const fetchJson = jest.fn((path: string) => {
const indexPatternTitle = _.last(path.split('/'));
return {
indexPatternTitle,
Expand All @@ -553,6 +562,7 @@ describe('loader', () => {
fetchJson: fetchJson as any,
indexPatterns: [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
setState,
dslQuery,
});

expect(fetchJson).toHaveBeenCalledTimes(3);
Expand Down
14 changes: 8 additions & 6 deletions x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,26 +215,28 @@ export async function syncExistingFields({
dateRange,
fetchJson,
setState,
dslQuery,
}: {
dateRange: DateRange;
indexPatterns: Array<{ id: string; timeFieldName?: string | null }>;
fetchJson: HttpSetup['get'];
fetchJson: HttpSetup['post'];
setState: SetState;
dslQuery: object;
}) {
const emptinessInfo = await Promise.all(
indexPatterns.map(pattern => {
const query: Record<string, string> = {
const body: Record<string, string | object> = {
dslQuery,
fromDate: dateRange.fromDate,
toDate: dateRange.toDate,
};

if (pattern.timeFieldName) {
query.timeFieldName = pattern.timeFieldName;
body.timeFieldName = pattern.timeFieldName;
}

return fetchJson({
path: `${BASE_API_URL}/existing_fields/${pattern.id}`,
query,
return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, {
body: JSON.stringify(body),
}) as Promise<ExistingFields>;
})
);
Expand Down
43 changes: 23 additions & 20 deletions x-pack/plugins/lens/server/routes/existing_fields.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,16 @@ const metaFields = ['_source', '_id', '_type', '_index', '_score'];

export async function existingFieldsRoute(setup: CoreSetup) {
const router = setup.http.createRouter();
router.get(

router.post(
{
path: `${BASE_API_URL}/existing_fields/{indexPatternId}`,
validate: {
params: schema.object({
indexPatternId: schema.string(),
}),
query: schema.object({
body: schema.object({
dslQuery: schema.object({}, { allowUnknowns: true }),
fromDate: schema.maybe(schema.string()),
toDate: schema.maybe(schema.string()),
timeFieldName: schema.maybe(schema.string()),
Expand All @@ -64,8 +66,8 @@ export async function existingFieldsRoute(setup: CoreSetup) {
try {
return res.ok({
body: await fetchFieldExistence({
...req.query,
...req.params,
...req.body,
context,
}),
});
Expand All @@ -91,12 +93,14 @@ export async function existingFieldsRoute(setup: CoreSetup) {
async function fetchFieldExistence({
context,
indexPatternId,
dslQuery = { match_all: {} },
fromDate,
toDate,
timeFieldName,
}: {
indexPatternId: string;
context: RequestHandlerContext;
dslQuery: object;
fromDate?: string;
toDate?: string;
timeFieldName?: string;
Expand All @@ -109,10 +113,10 @@ async function fetchFieldExistence({
} = await fetchIndexPatternDefinition(indexPatternId, context);

const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);

const docs = await fetchIndexPatternStats({
fromDate,
toDate,
dslQuery,
client: context.core.elasticsearch.dataClient,
index: indexPatternTitle,
timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName,
Expand Down Expand Up @@ -197,24 +201,23 @@ export function buildFieldList(
async function fetchIndexPatternStats({
client,
index,
dslQuery,
timeFieldName,
fromDate,
toDate,
fields,
}: {
client: IScopedClusterClient;
index: string;
dslQuery: object;
timeFieldName?: string;
fromDate?: string;
toDate?: string;
fields: Field[];
}) {
let query;

if (timeFieldName && fromDate && toDate) {
query = {
bool: {
filter: [
const filter =
timeFieldName && fromDate && toDate
? [
{
range: {
[timeFieldName]: {
Expand All @@ -223,16 +226,17 @@ async function fetchIndexPatternStats({
},
},
},
],
},
};
} else {
query = {
match_all: {},
};
}
const scriptedFields = fields.filter(f => f.isScript);
dslQuery,
]
: [dslQuery];

const query = {
bool: {
filter,
},
};

const scriptedFields = fields.filter(f => f.isScript);
const result = await client.callAsCurrentUser('search', {
index,
body: {
Expand All @@ -251,7 +255,6 @@ async function fetchIndexPatternStats({
}, {} as Record<string, unknown>),
},
});

return result.hits.hits;
}

Expand Down
Loading

0 comments on commit b61c309

Please sign in to comment.