Skip to content

Commit

Permalink
[UnifiedFieldList] Categorize fields as empty that never had a value …
Browse files Browse the repository at this point in the history
…in matching indices (#174063)

With this commit the UnifiedFieldList used in Discover and Lens will categorize fields that never had any value in matching indices as "Empty fields". This was built by using the new `include_fields_with_no_value` parameter of Elasticsearch field_caps API.

Co-authored-by: Matt Kime <[email protected]>
  • Loading branch information
kertal and mattkime authored Feb 13, 2024
1 parent 15c6153 commit fde955d
Show file tree
Hide file tree
Showing 13 changed files with 115 additions and 38 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ export async function fetchFieldExistence({
// filled in by data views service
pattern: '',
indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
includeEmptyFields: false,
});

// take care of fields of existingFieldList, that are not yet available
Expand Down
1 change: 1 addition & 0 deletions src/plugins/data_views/common/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ export interface GetFieldsOptions {
fields?: string[];
allowHidden?: boolean;
forceRefresh?: boolean;
includeEmptyFields?: boolean;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
fields,
forceRefresh,
allowHidden,
includeEmptyFields,
} = options;
const path = indexFilter ? FIELDS_FOR_WILDCARD_PATH : FIELDS_PATH;
const versionQueryParam = indexFilter ? {} : { apiVersion: version };
Expand All @@ -111,6 +112,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
fields,
// default to undefined to keep value out of URL params and improve caching
allow_hidden: allowHidden || undefined,
include_empty_fields: includeEmptyFields,
...versionQueryParam,
},
indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ export class IndexPatternsFetcher {
indexFilter?: QueryDslQueryContainer;
fields?: string[];
allowHidden?: boolean;
includeEmptyFields?: boolean;
}): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
const {
pattern,
Expand All @@ -82,6 +83,7 @@ export class IndexPatternsFetcher {
rollupIndex,
indexFilter,
allowHidden,
includeEmptyFields,
} = options;
const allowNoIndices = fieldCapsOptions
? fieldCapsOptions.allow_no_indices
Expand All @@ -100,6 +102,7 @@ export class IndexPatternsFetcher {
indexFilter,
fields: options.fields || ['*'],
expandWildcards,
includeEmptyFields,
});

if (this.rollupsEnabled && type === 'rollup' && rollupIndex) {
Expand Down
4 changes: 4 additions & 0 deletions src/plugins/data_views/server/fetcher/lib/es_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ interface FieldCapsApiParams {
indexFilter?: QueryDslQueryContainer;
fields?: string[];
expandWildcards?: ExpandWildcard;
includeEmptyFields?: boolean;
}

/**
Expand All @@ -72,6 +73,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
},
fields = ['*'],
expandWildcards,
includeEmptyFields,
} = params;
try {
return await callCluster.fieldCaps(
Expand All @@ -81,6 +83,8 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
ignore_unavailable: true,
index_filter: indexFilter,
expand_wildcards: expandWildcards,
// @ts-expect-error
include_empty_fields: includeEmptyFields ?? true,
...fieldCapsOptions,
},
{ meta: true }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldCapsOptions: undefined,
indexFilter: undefined,
fields: undefined,
includeEmptyFields: undefined,
...args,
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ interface FieldCapabilitiesParams {
indexFilter?: QueryDslQueryContainer;
fields?: string[];
expandWildcards?: ExpandWildcard;
includeEmptyFields?: boolean;
}

/**
Expand All @@ -45,6 +46,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
metaFields = [],
fields,
expandWildcards,
includeEmptyFields,
} = params;
const esFieldCaps = await callFieldCapsApi({
callCluster,
Expand All @@ -53,6 +55,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
indexFilter,
fields,
expandWildcards,
includeEmptyFields,
});
const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ export interface IQuery {
include_unmapped?: boolean;
fields?: string[];
allow_hidden?: boolean;
include_empty_fields?: boolean;
}

export const querySchema = schema.object({
Expand All @@ -68,6 +69,7 @@ export const querySchema = schema.object({
include_unmapped: schema.maybe(schema.boolean()),
fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
allow_hidden: schema.maybe(schema.boolean()),
include_empty_fields: schema.maybe(schema.boolean()),
});

const fieldSubTypeSchema = schema.object({
Expand Down Expand Up @@ -135,6 +137,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
allow_no_index: allowNoIndex,
include_unmapped: includeUnmapped,
allow_hidden: allowHidden,
include_empty_fields: includeEmptyFields,
} = request.query;

// not available to get request
Expand All @@ -161,6 +164,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
},
indexFilter: getIndexFilterDsl({ indexFilter, excludedTiers }),
allowHidden,
includeEmptyFields,
...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
});

Expand Down
4 changes: 2 additions & 2 deletions test/functional/apps/discover/group3/_drag_drop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'53 available fields. 3 meta fields.'
'48 available fields. 5 empty fields. 3 meta fields.'
);
expect((await PageObjects.discover.getColumnHeaders()).join(', ')).to.be(
'@timestamp, Document'
Expand All @@ -72,7 +72,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'53 available fields. 3 meta fields.'
'48 available fields. 5 empty fields. 3 meta fields.'
);
expect((await PageObjects.discover.getColumnHeaders()).join(', ')).to.be(
'@timestamp, Document'
Expand Down
34 changes: 16 additions & 18 deletions test/functional/apps/discover/group3/_sidebar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const fieldEditor = getService('fieldEditor');
const retry = getService('retry');
const dataGrid = getService('dataGrid');
const INITIAL_FIELD_LIST_SUMMARY = '53 available fields. 3 meta fields.';
const INITIAL_FIELD_LIST_SUMMARY = '48 available fields. 5 empty fields. 3 meta fields.';

describe('discover sidebar', function describeIndexTests() {
before(async function () {
Expand Down Expand Up @@ -72,7 +72,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.waitFor('first updates', async () => {
return (
(await PageObjects.unifiedFieldList.getSidebarAriaDescription()) ===
'7 available fields. 2 meta fields.'
'6 available fields. 1 empty field. 2 meta fields.'
);
});

Expand All @@ -81,7 +81,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.waitFor('second updates', async () => {
return (
(await PageObjects.unifiedFieldList.getSidebarAriaDescription()) ===
'13 available fields. 3 meta fields.'
'10 available fields. 3 empty fields. 3 meta fields.'
);
});

Expand Down Expand Up @@ -178,7 +178,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.waitFor('first updates', async () => {
return (
(await PageObjects.unifiedFieldList.getSidebarAriaDescription()) ===
'30 available fields. 2 meta fields.'
'28 available fields. 2 empty fields. 2 meta fields.'
);
});

Expand Down Expand Up @@ -315,11 +315,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {

// Initial Available fields
const expectedInitialAvailableFields =
'@message, @tags, @timestamp, agent, bytes, clientip, extension, geo.coordinates, geo.dest, geo.src, geo.srcdest, headings, host, id, index, ip, links, machine.os, machine.ram, machine.ram_range, memory, meta.char, meta.related, meta.user.firstname, meta.user.lastname, nestedField.child, phpmemory, referer, relatedContent.article:modified_time, relatedContent.article:published_time, relatedContent.article:section, relatedContent.article:tag, relatedContent.og:description, relatedContent.og:image, relatedContent.og:image:height, relatedContent.og:image:width, relatedContent.og:site_name, relatedContent.og:title, relatedContent.og:type, relatedContent.og:url, relatedContent.twitter:card, relatedContent.twitter:description, relatedContent.twitter:image, relatedContent.twitter:site, relatedContent.twitter:title, relatedContent.url, request, response, spaces, type';
'@message, @tags, @timestamp, agent, bytes, clientip, extension, geo.coordinates, geo.dest, geo.src, geo.srcdest, headings, host, index, ip, links, machine.os, machine.ram, machine.ram_range, memory, nestedField.child, phpmemory, referer, relatedContent.article:modified_time, relatedContent.article:published_time, relatedContent.article:section, relatedContent.article:tag, relatedContent.og:description, relatedContent.og:image, relatedContent.og:image:height, relatedContent.og:image:width, relatedContent.og:site_name, relatedContent.og:title, relatedContent.og:type, relatedContent.og:url, relatedContent.twitter:card, relatedContent.twitter:description, relatedContent.twitter:image, relatedContent.twitter:site, relatedContent.twitter:title, relatedContent.url, request, response, spaces, type, url, utc_time, xss';
let availableFields = await PageObjects.unifiedFieldList.getSidebarSectionFieldNames(
'available'
);
expect(availableFields.length).to.be(50);
expect(availableFields.length).to.be(48);
expect(availableFields.join(', ')).to.be(expectedInitialAvailableFields);

// Available fields after scrolling down
Expand All @@ -332,12 +332,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
availableFields = await PageObjects.unifiedFieldList.getSidebarSectionFieldNames(
'available'
);
return availableFields.length === 53;
return availableFields.length === 48;
});

expect(availableFields.join(', ')).to.be(
`${expectedInitialAvailableFields}, url, utc_time, xss`
);
expect(availableFields.join(', ')).to.be(`${expectedInitialAvailableFields}`);

// Expand Meta section
await PageObjects.unifiedFieldList.toggleSidebarSection('meta');
Expand All @@ -361,7 +359,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const availableFields = await PageObjects.unifiedFieldList.getSidebarSectionFieldNames(
'available'
);
expect(availableFields.length).to.be(50);
expect(availableFields.length).to.be(48);
expect(
availableFields
.join(', ')
Expand Down Expand Up @@ -389,7 +387,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
).to.be('relatedContent');

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'53 available fields. 1 unmapped field. 3 meta fields.'
'48 available fields. 1 unmapped field. 5 empty fields. 3 meta fields.'
);
});

Expand All @@ -410,7 +408,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
expect(availableFields.includes('@message')).to.be(true);

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'2 selected fields. 2 popular fields. 53 available fields. 3 meta fields.'
'2 selected fields. 2 popular fields. 48 available fields. 5 empty fields. 3 meta fields.'
);

await PageObjects.unifiedFieldList.clickFieldListItemRemove('@message');
Expand All @@ -430,7 +428,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
).to.be('@message, _id, extension');

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'3 selected fields. 3 popular fields. 53 available fields. 3 meta fields.'
'3 selected fields. 3 popular fields. 48 available fields. 5 empty fields. 3 meta fields.'
);
});

Expand Down Expand Up @@ -485,7 +483,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'53 available fields. 3 meta fields.'
'48 available fields. 5 empty fields. 3 meta fields.'
);
});

Expand Down Expand Up @@ -670,7 +668,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'54 available fields. 3 meta fields.'
'49 available fields. 5 empty fields. 3 meta fields.'
);

let allFields = await PageObjects.unifiedFieldList.getAllFieldNames();
Expand All @@ -689,7 +687,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();

expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'54 available fields. 3 meta fields.'
'49 available fields. 5 empty fields. 3 meta fields.'
);

allFields = await PageObjects.unifiedFieldList.getAllFieldNames();
Expand Down Expand Up @@ -726,7 +724,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {

// check that the sidebar is rendered
expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be(
'54 available fields. 3 meta fields.'
'49 available fields. 5 empty fields. 3 meta fields.'
);
let allFields = await PageObjects.unifiedFieldList.getAllFieldNames();
expect(allFields.includes('_invalid-runtimefield')).to.be(true);
Expand Down
62 changes: 62 additions & 0 deletions test/functional/apps/discover/group4/_field_list_new_fields.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
path: '/my-index-000001',
method: 'DELETE',
});
await es.transport.request({
path: '/my-index-000002',
method: 'DELETE',
});
});

it('Check that new ingested fields are added to the available fields section', async function () {
Expand Down Expand Up @@ -82,5 +86,63 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
'b',
]);
});

it("Mapped fields without values aren't shown", async function () {
const initialPattern = 'my-index-000002';
await es.transport.request({
path: '/my-index-000002/_doc',
method: 'POST',
body: {
'@timestamp': new Date().toISOString(),
a: 'GET /search HTTP/1.1 200 1070000',
},
});

await PageObjects.discover.createAdHocDataView(initialPattern, true);

await retry.waitFor('current data view to get updated', async () => {
return (await PageObjects.discover.getCurrentlySelectedDataView()) === `${initialPattern}*`;
});
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();

expect(await PageObjects.discover.getHitCountInt()).to.be(1);
expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([
'@timestamp',
'a',
]);

await es.transport.request({
path: '/my-index-000002/_mapping',
method: 'PUT',
body: {
properties: {
b: {
type: 'keyword',
},
},
},
});

// add new doc and check for it to make sure we're looking at fresh results
await es.transport.request({
path: '/my-index-000002/_doc',
method: 'POST',
body: {
'@timestamp': new Date().toISOString(),
a: 'GET /search HTTP/1.1 200 1070000',
},
});

await retry.waitFor('the new record was found', async () => {
await queryBar.submitQuery();
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
return (await PageObjects.discover.getHitCountInt()) === 2;
});

expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([
'@timestamp',
'a',
]);
});
});
}
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/lens/group2/fields_list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
it('should show all fields as available', async () => {
expect(
await (await testSubjects.find('lnsIndexPatternAvailableFields-count')).getVisibleText()
).to.eql(53);
).to.eql(50);
});

it('should show a histogram and top values popover for numeric field', async () => {
Expand Down
Loading

0 comments on commit fde955d

Please sign in to comment.