diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx
index fca958a39b086..0871ef4749496 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx
@@ -116,27 +116,6 @@ describe('IndexPattern Field Item', () => {
);
});
- it('should request field stats without a time field, if the index pattern has none', async () => {
- indexPattern.timeFieldName = undefined;
- core.http.post.mockImplementationOnce(() => {
- return Promise.resolve({});
- });
- const wrapper = mountWithIntl();
-
- await act(async () => {
- clickField(wrapper, 'bytes');
- });
-
- expect(core.http.post).toHaveBeenCalledWith(
- '/api/lens/index_stats/my-fake-index-pattern/field',
- expect.anything()
- );
- // Function argument types not detected correctly (https://github.com/microsoft/TypeScript/issues/26591)
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- const { body } = (core.http.post.mock.calls[0] as any)[1];
- expect(JSON.parse(body)).not.toHaveProperty('timeFieldName');
- });
-
it('should request field stats every time the button is clicked', async () => {
let resolveFunction: (arg: unknown) => void;
@@ -150,31 +129,21 @@ describe('IndexPattern Field Item', () => {
clickField(wrapper, 'bytes');
- expect(core.http.post).toHaveBeenCalledWith(
- `/api/lens/index_stats/my-fake-index-pattern/field`,
- {
- body: JSON.stringify({
- dslQuery: {
- bool: {
- must: [{ match_all: {} }],
- filter: [],
- should: [],
- must_not: [],
- },
+ expect(core.http.post).toHaveBeenCalledWith(`/api/lens/index_stats/1/field`, {
+ body: JSON.stringify({
+ dslQuery: {
+ bool: {
+ must: [{ match_all: {} }],
+ filter: [],
+ should: [],
+ must_not: [],
},
- fromDate: 'now-7d',
- toDate: 'now',
- timeFieldName: 'timestamp',
- field: {
- name: 'bytes',
- displayName: 'bytesLabel',
- type: 'number',
- aggregatable: true,
- searchable: true,
- },
- }),
- }
- );
+ },
+ fromDate: 'now-7d',
+ toDate: 'now',
+ fieldName: 'bytes',
+ }),
+ });
expect(wrapper.find(EuiPopover).prop('isOpen')).toEqual(true);
@@ -227,40 +196,30 @@ describe('IndexPattern Field Item', () => {
clickField(wrapper, 'bytes');
expect(core.http.post).toHaveBeenCalledTimes(2);
- expect(core.http.post).toHaveBeenLastCalledWith(
- `/api/lens/index_stats/my-fake-index-pattern/field`,
- {
- body: JSON.stringify({
- dslQuery: {
- bool: {
- must: [],
- filter: [
- {
- bool: {
- should: [{ match_phrase: { 'geo.src': 'US' } }],
- minimum_should_match: 1,
- },
- },
- {
- match: { phrase: { 'geo.dest': 'US' } },
+ expect(core.http.post).toHaveBeenLastCalledWith(`/api/lens/index_stats/1/field`, {
+ body: JSON.stringify({
+ dslQuery: {
+ bool: {
+ must: [],
+ filter: [
+ {
+ bool: {
+ should: [{ match_phrase: { 'geo.src': 'US' } }],
+ minimum_should_match: 1,
},
- ],
- should: [],
- must_not: [],
- },
+ },
+ {
+ match: { phrase: { 'geo.dest': 'US' } },
+ },
+ ],
+ should: [],
+ must_not: [],
},
- fromDate: 'now-14d',
- toDate: 'now-7d',
- timeFieldName: 'timestamp',
- field: {
- name: 'bytes',
- displayName: 'bytesLabel',
- type: 'number',
- aggregatable: true,
- searchable: true,
- },
- }),
- }
- );
+ },
+ fromDate: 'now-14d',
+ toDate: 'now-7d',
+ fieldName: 'bytes',
+ }),
+ });
});
});
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx
index e0198d6d7903e..e5d46b4a7a073 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx
@@ -129,7 +129,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) {
setState((s) => ({ ...s, isLoading: true }));
core.http
- .post(`/api/lens/index_stats/${indexPattern.title}/field`, {
+ .post(`/api/lens/index_stats/${indexPattern.id}/field`, {
body: JSON.stringify({
dslQuery: esQuery.buildEsQuery(
indexPattern as IIndexPattern,
@@ -139,8 +139,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) {
),
fromDate: dateRange.fromDate,
toDate: dateRange.toDate,
- timeFieldName: indexPattern.timeFieldName,
- field,
+ fieldName: field.name,
}),
})
.then((results: FieldStatsResponse) => {
diff --git a/x-pack/plugins/lens/server/routes/existing_fields.test.ts b/x-pack/plugins/lens/server/routes/existing_fields.test.ts
index c6364eca0ff49..3f3e94099f666 100644
--- a/x-pack/plugins/lens/server/routes/existing_fields.test.ts
+++ b/x-pack/plugins/lens/server/routes/existing_fields.test.ts
@@ -61,6 +61,20 @@ describe('existingFields', () => {
expect(result).toEqual(['bar']);
});
+ it('supports runtime fields', () => {
+ const result = existingFields(
+ [searchResults({ runtime_foo: ['scriptvalue'] })],
+ [
+ field({
+ name: 'runtime_foo',
+ runtimeField: { type: 'long', script: { source: '2+2' } },
+ }),
+ ]
+ );
+
+ expect(result).toEqual(['runtime_foo']);
+ });
+
it('supports meta fields', () => {
const result = existingFields(
[{ _mymeta: 'abc', ...searchResults({ bar: ['scriptvalue'] }) }],
@@ -78,6 +92,11 @@ describe('buildFieldList', () => {
typeMeta: 'typemeta',
fields: [
{ name: 'foo', scripted: true, lang: 'painless', script: '2+2' },
+ {
+ name: 'runtime_foo',
+ isMapped: false,
+ runtimeField: { type: 'long', script: { source: '2+2' } },
+ },
{ name: 'bar' },
{ name: '@bar' },
{ name: 'baz' },
@@ -95,6 +114,14 @@ describe('buildFieldList', () => {
});
});
+ it('supports runtime fields', () => {
+ const fields = buildFieldList((indexPattern as unknown) as IndexPattern, []);
+ expect(fields.find((f) => f.runtimeField)).toMatchObject({
+ name: 'runtime_foo',
+ runtimeField: { type: 'long', script: { source: '2+2' } },
+ });
+ });
+
it('supports meta fields', () => {
const fields = buildFieldList((indexPattern as unknown) as IndexPattern, ['_mymeta']);
expect(fields.find((f) => f.isMeta)).toMatchObject({
diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts
index e76abf4598efa..11db9360749ea 100644
--- a/x-pack/plugins/lens/server/routes/existing_fields.ts
+++ b/x-pack/plugins/lens/server/routes/existing_fields.ts
@@ -10,7 +10,7 @@ import { errors } from '@elastic/elasticsearch';
import { schema } from '@kbn/config-schema';
import { RequestHandlerContext, ElasticsearchClient } from 'src/core/server';
import { CoreSetup, Logger } from 'src/core/server';
-import { IndexPattern, IndexPatternsService } from 'src/plugins/data/common';
+import { IndexPattern, IndexPatternsService, RuntimeField } from 'src/plugins/data/common';
import { BASE_API_URL } from '../../common';
import { UI_SETTINGS } from '../../../../../src/plugins/data/server';
import { PluginStartContract } from '../plugin';
@@ -30,6 +30,7 @@ export interface Field {
isMeta: boolean;
lang?: string;
script?: string;
+ runtimeField?: RuntimeField;
}
export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) {
@@ -138,6 +139,7 @@ export function buildFieldList(indexPattern: IndexPattern, metaFields: string[])
// id is a special case - it doesn't show up in the meta field list,
// but as it's not part of source, it has to be handled separately.
isMeta: metaFields.includes(field.name) || field.name === '_id',
+ runtimeField: !field.isMapped ? field.runtimeField : undefined,
};
});
}
@@ -181,6 +183,7 @@ async function fetchIndexPatternStats({
};
const scriptedFields = fields.filter((f) => f.isScript);
+ const runtimeFields = fields.filter((f) => f.runtimeField);
const { body: result } = await client.search({
index,
body: {
@@ -189,6 +192,11 @@ async function fetchIndexPatternStats({
sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [],
fields: ['*'],
_source: false,
+ runtime_mappings: runtimeFields.reduce((acc, field) => {
+ if (!field.runtimeField) return acc;
+ acc[field.name] = field.runtimeField;
+ return acc;
+ }, {} as Record),
script_fields: scriptedFields.reduce((acc, field) => {
acc[field.name] = {
script: {
diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts
index 7fd884755d86d..9094e5442dc51 100644
--- a/x-pack/plugins/lens/server/routes/field_stats.ts
+++ b/x-pack/plugins/lens/server/routes/field_stats.ts
@@ -11,6 +11,7 @@ import DateMath from '@elastic/datemath';
import { schema } from '@kbn/config-schema';
import { CoreSetup } from 'src/core/server';
import { IFieldType } from 'src/plugins/data/common';
+import { SavedObjectNotFound } from '../../../../../src/plugins/kibana_utils/common';
import { ESSearchResponse } from '../../../../typings/elasticsearch';
import { FieldStatsResponse, BASE_API_URL } from '../../common';
import { PluginStartContract } from '../plugin';
@@ -21,28 +22,17 @@ export async function initFieldsRoute(setup: CoreSetup) {
const router = setup.http.createRouter();
router.post(
{
- path: `${BASE_API_URL}/index_stats/{indexPatternTitle}/field`,
+ path: `${BASE_API_URL}/index_stats/{indexPatternId}/field`,
validate: {
params: schema.object({
- indexPatternTitle: schema.string(),
+ indexPatternId: schema.string(),
}),
body: schema.object(
{
dslQuery: schema.object({}, { unknowns: 'allow' }),
fromDate: schema.string(),
toDate: schema.string(),
- timeFieldName: schema.maybe(schema.string()),
- field: schema.object(
- {
- name: schema.string(),
- type: schema.string(),
- esTypes: schema.maybe(schema.arrayOf(schema.string())),
- scripted: schema.maybe(schema.boolean()),
- lang: schema.maybe(schema.string()),
- script: schema.maybe(schema.string()),
- },
- { unknowns: 'allow' }
- ),
+ fieldName: schema.string(),
},
{ unknowns: 'allow' }
),
@@ -50,9 +40,26 @@ export async function initFieldsRoute(setup: CoreSetup) {
},
async (context, req, res) => {
const requestClient = context.core.elasticsearch.client.asCurrentUser;
- const { fromDate, toDate, timeFieldName, field, dslQuery } = req.body;
+ const { fromDate, toDate, fieldName, dslQuery } = req.body;
+
+ const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices();
+ const savedObjectsClient = savedObjects.getScopedClient(req);
+ const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
+ const indexPatternsService = await data.indexPatterns.indexPatternsServiceFactory(
+ savedObjectsClient,
+ esClient
+ );
try {
+ const indexPattern = await indexPatternsService.get(req.params.indexPatternId);
+
+ const timeFieldName = indexPattern.timeFieldName;
+ const field = indexPattern.fields.find((f) => f.name === fieldName);
+
+ if (!field) {
+ throw new Error(`Field {fieldName} not found in index pattern ${indexPattern.title}`);
+ }
+
const filter = timeFieldName
? [
{
@@ -75,11 +82,12 @@ export async function initFieldsRoute(setup: CoreSetup) {
const search = async (aggs: unknown) => {
const { body: result } = await requestClient.search({
- index: req.params.indexPatternTitle,
+ index: indexPattern.title,
track_total_hits: true,
body: {
query,
aggs,
+ runtime_mappings: field.runtimeField ? { [fieldName]: field.runtimeField } : {},
},
size: 0,
});
@@ -104,6 +112,9 @@ export async function initFieldsRoute(setup: CoreSetup) {
body: await getStringSamples(search, field),
});
} catch (e) {
+ if (e instanceof SavedObjectNotFound) {
+ return res.notFound();
+ }
if (e instanceof errors.ResponseError && e.statusCode === 404) {
return res.notFound();
}
diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts
index ac4ebb4e5b02c..94960b9859121 100644
--- a/x-pack/test/api_integration/apis/lens/field_stats.ts
+++ b/x-pack/test/api_integration/apis/lens/field_stats.ts
@@ -22,29 +22,29 @@ export default ({ getService }: FtrProviderContext) => {
describe('index stats apis', () => {
before(async () => {
await esArchiver.loadIfNeeded('logstash_functional');
- await esArchiver.loadIfNeeded('visualize/default');
- await esArchiver.loadIfNeeded('pre_calculated_histogram');
});
after(async () => {
await esArchiver.unload('logstash_functional');
- await esArchiver.unload('visualize/default');
- await esArchiver.unload('pre_calculated_histogram');
});
describe('field distribution', () => {
+ before(async () => {
+ await esArchiver.loadIfNeeded('visualize/default');
+ });
+ after(async () => {
+ await esArchiver.unload('visualize/default');
+ });
+
it('should return a 404 for missing index patterns', async () => {
await supertest
- .post('/api/lens/index_stats/logstash/field')
+ .post('/api/lens/index_stats/123/field')
.set(COMMON_HEADERS)
.send({
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
timeFieldName: '@timestamp',
- field: {
- name: 'bytes',
- type: 'number',
- },
+ fieldName: 'bytes',
})
.expect(404);
});
@@ -57,10 +57,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- field: {
- name: 'bytes',
- type: 'number',
- },
+ fieldName: 'bytes',
})
.expect(200);
@@ -75,11 +72,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: 'bytes',
- type: 'number',
- },
+ fieldName: 'bytes',
})
.expect(200);
@@ -186,11 +179,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: '@timestamp',
- type: 'date',
- },
+ fieldName: '@timestamp',
})
.expect(200);
@@ -223,11 +212,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: 'geo.src',
- type: 'string',
- },
+ fieldName: 'geo.src',
})
.expect(200);
@@ -290,11 +275,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: 'ip',
- type: 'ip',
- },
+ fieldName: 'ip',
})
.expect(200);
@@ -349,6 +330,113 @@ export default ({ getService }: FtrProviderContext) => {
});
});
+ it('should return histograms for scripted date fields', async () => {
+ const { body } = await supertest
+ .post('/api/lens/index_stats/logstash-2015.09.22/field')
+ .set(COMMON_HEADERS)
+ .send({
+ dslQuery: { match_all: {} },
+ fromDate: TEST_START_TIME,
+ toDate: TEST_END_TIME,
+ fieldName: 'scripted_date',
+ })
+ .expect(200);
+
+ expect(body).to.eql({
+ histogram: {
+ buckets: [
+ {
+ count: 4634,
+ key: 0,
+ },
+ ],
+ },
+ totalDocuments: 4634,
+ });
+ });
+
+ it('should return top values for scripted string fields', async () => {
+ const { body } = await supertest
+ .post('/api/lens/index_stats/logstash-2015.09.22/field')
+ .set(COMMON_HEADERS)
+ .send({
+ dslQuery: { match_all: {} },
+ fromDate: TEST_START_TIME,
+ toDate: TEST_END_TIME,
+ fieldName: 'scripted_string',
+ })
+ .expect(200);
+
+ expect(body).to.eql({
+ totalDocuments: 4634,
+ sampledDocuments: 4634,
+ sampledValues: 4634,
+ topValues: {
+ buckets: [
+ {
+ count: 4634,
+ key: 'hello',
+ },
+ ],
+ },
+ });
+ });
+
+ it('should return top values for index pattern runtime string fields', async () => {
+ const { body } = await supertest
+ .post('/api/lens/index_stats/logstash-2015.09.22/field')
+ .set(COMMON_HEADERS)
+ .send({
+ dslQuery: { match_all: {} },
+ fromDate: TEST_START_TIME,
+ toDate: TEST_END_TIME,
+ fieldName: 'runtime_string_field',
+ })
+ .expect(200);
+
+ expect(body).to.eql({
+ totalDocuments: 4634,
+ sampledDocuments: 4634,
+ sampledValues: 4634,
+ topValues: {
+ buckets: [
+ {
+ count: 4634,
+ key: 'hello world!',
+ },
+ ],
+ },
+ });
+ });
+
+ it('should apply filters and queries', async () => {
+ const { body } = await supertest
+ .post('/api/lens/index_stats/logstash-2015.09.22/field')
+ .set(COMMON_HEADERS)
+ .send({
+ dslQuery: {
+ bool: {
+ filter: [{ match: { 'geo.src': 'US' } }],
+ },
+ },
+ fromDate: TEST_START_TIME,
+ toDate: TEST_END_TIME,
+ fieldName: 'bytes',
+ })
+ .expect(200);
+
+ expect(body.totalDocuments).to.eql(425);
+ });
+ });
+
+ describe('histogram', () => {
+ before(async () => {
+ await esArchiver.loadIfNeeded('pre_calculated_histogram');
+ });
+ after(async () => {
+ await esArchiver.unload('pre_calculated_histogram');
+ });
+
it('should return an auto histogram for precalculated histograms', async () => {
const { body } = await supertest
.post('/api/lens/index_stats/histogram-test/field')
@@ -357,10 +445,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- field: {
- name: 'histogram-content',
- type: 'histogram',
- },
+ fieldName: 'histogram-content',
})
.expect(200);
@@ -428,10 +513,7 @@ export default ({ getService }: FtrProviderContext) => {
dslQuery: { match: { 'histogram-title': 'single value' } },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
- field: {
- name: 'histogram-content',
- type: 'histogram',
- },
+ fieldName: 'histogram-content',
})
.expect(200);
@@ -443,95 +525,6 @@ export default ({ getService }: FtrProviderContext) => {
topValues: { buckets: [] },
});
});
-
- it('should return histograms for scripted date fields', async () => {
- const { body } = await supertest
- .post('/api/lens/index_stats/logstash-2015.09.22/field')
- .set(COMMON_HEADERS)
- .send({
- dslQuery: { match_all: {} },
- fromDate: TEST_START_TIME,
- toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: 'scripted date',
- type: 'date',
- scripted: true,
- script: '1234',
- lang: 'painless',
- },
- })
- .expect(200);
-
- expect(body).to.eql({
- histogram: {
- buckets: [
- {
- count: 4634,
- key: 0,
- },
- ],
- },
- totalDocuments: 4634,
- });
- });
-
- it('should return top values for scripted string fields', async () => {
- const { body } = await supertest
- .post('/api/lens/index_stats/logstash-2015.09.22/field')
- .set(COMMON_HEADERS)
- .send({
- dslQuery: { match_all: {} },
- fromDate: TEST_START_TIME,
- toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: 'scripted string',
- type: 'string',
- scripted: true,
- script: 'return "hello"',
- lang: 'painless',
- },
- })
- .expect(200);
-
- expect(body).to.eql({
- totalDocuments: 4634,
- sampledDocuments: 4634,
- sampledValues: 4634,
- topValues: {
- buckets: [
- {
- count: 4634,
- key: 'hello',
- },
- ],
- },
- });
- });
-
- it('should apply filters and queries', async () => {
- const { body } = await supertest
- .post('/api/lens/index_stats/logstash-2015.09.22/field')
- .set(COMMON_HEADERS)
- .send({
- dslQuery: {
- bool: {
- filter: [{ match: { 'geo.src': 'US' } }],
- },
- },
- fromDate: TEST_START_TIME,
- toDate: TEST_END_TIME,
- timeFieldName: '@timestamp',
- field: {
- name: 'bytes',
- type: 'number',
- },
- })
- .expect(200);
-
- expect(body.totalDocuments).to.eql(425);
- });
});
});
};
diff --git a/x-pack/test/functional/es_archives/visualize/default/data.json b/x-pack/test/functional/es_archives/visualize/default/data.json
index 26b033e28b4da..7d0ad0c25f96d 100644
--- a/x-pack/test/functional/es_archives/visualize/default/data.json
+++ b/x-pack/test/functional/es_archives/visualize/default/data.json
@@ -145,6 +145,31 @@
}
}
+{
+ "type": "doc",
+ "value": {
+ "index": ".kibana",
+ "type": "doc",
+ "id": "index-pattern:logstash-2015.09.22",
+ "index": ".kibana_1",
+ "source": {
+ "index-pattern": {
+ "timeFieldName": "@timestamp",
+ "title": "logstash-2015.09.22",
+ "fields":"[{\"name\":\"scripted_date\",\"type\":\"date\",\"count\":0,\"scripted\":true,\"script\":\"1234\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"scripted_string\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"return 'hello'\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]",
+ "runtimeFieldMap":"{\"runtime_string_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('hello world!')\"}}}"
+ },
+ "migrationVersion": {
+ "index-pattern": "7.11.0"
+ },
+ "references": [
+ ],
+ "type": "index-pattern",
+ "updated_at": "2018-12-21T00:43:07.096Z"
+ }
+ }
+}
+
{
"type": "doc",
"value": {