From beb0ea1c93437f313962eab6c5a4264e39bcf4f6 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 28 Aug 2019 06:59:24 +0200 Subject: [PATCH 1/8] Add numeric_type to sort containing date_nanos --- .../public/discover/doc_table/lib/get_sort.js | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js index 0af57c158e21a..b8fb97c86f25a 100644 --- a/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js +++ b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js @@ -20,19 +20,16 @@ import _ from 'lodash'; function isSortable(field, indexPattern) { - return (indexPattern.fields.byName[field] && indexPattern.fields.byName[field].sortable); + return indexPattern.fields.byName[field] && indexPattern.fields.byName[field].sortable; } function createSortObject(sortPair, indexPattern) { - if (Array.isArray(sortPair) && sortPair.length === 2 && isSortable(sortPair[0], indexPattern)) { - const [ field, direction ] = sortPair; + const [field, direction] = sortPair; return { [field]: direction }; - } - else if (_.isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) { + } else if (_.isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) { return sortPair; - } - else { + } else { return undefined; } } @@ -45,24 +42,33 @@ function createSortObject(sortPair, indexPattern) { * @returns {object} a sort object suitable for returning to elasticsearch */ export function getSort(sort, indexPattern, defaultSortOrder = 'desc') { - let sortObjects; if (Array.isArray(sort)) { - sortObjects = _.compact(sort.map((sortPair) => createSortObject(sortPair, indexPattern))); + sortObjects = _.compact(sort.map(sortPair => createSortObject(sortPair, indexPattern))); } if (!_.isEmpty(sortObjects)) { return sortObjects; - } - else if (indexPattern.timeFieldName && isSortable(indexPattern.timeFieldName, indexPattern)) { - return [{ [indexPattern.timeFieldName]: defaultSortOrder }]; - } - else { + } else if (indexPattern.timeFieldName && isSortable(indexPattern.timeFieldName, indexPattern)) { + if (indexPattern.isTimeNanosBased()) { + // is necessary for sorting indices with date AND date_nanos + return [ + { + [indexPattern.timeFieldName]: { order: defaultSortOrder, numeric_type: 'date_nanos' }, + }, + ]; + } else { + return [{ [indexPattern.timeFieldName]: defaultSortOrder }]; + } + } else { return [{ _score: 'desc' }]; } } getSort.array = function (sort, indexPattern, defaultSortOrder) { - return getSort(sort, indexPattern, defaultSortOrder).map((sortPair) => _(sortPair).pairs().pop()); + return getSort(sort, indexPattern, defaultSortOrder).map(sortPair => + _(sortPair) + .pairs() + .pop() + ); }; - From f1efb3ac4c703dd2b7ed555c9884e2b027aab63f Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 4 Sep 2019 16:33:31 +0200 Subject: [PATCH 2/8] Add isTimeNanosBased to test index pattern --- src/fixtures/stubbed_logstash_index_pattern.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fixtures/stubbed_logstash_index_pattern.js b/src/fixtures/stubbed_logstash_index_pattern.js index dd0381c440bd6..effb48a43a081 100644 --- a/src/fixtures/stubbed_logstash_index_pattern.js +++ b/src/fixtures/stubbed_logstash_index_pattern.js @@ -42,6 +42,7 @@ export default function stubbedLogstashIndexPatternService(Private) { const indexPattern = new StubIndexPattern('logstash-*', cfg => cfg, 'time', fields); indexPattern.id = 'logstash-*'; + indexPattern.isTimeNanosBased = () => false; return indexPattern; From 80a2d06f3e5e1b94f94c1351cd744d20e1f95315 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 11 Sep 2019 07:35:36 +0200 Subject: [PATCH 3/8] Add testdata + functional test draft --- .../apps/discover/_date_nanos_mixed.js | 48 + test/functional/apps/discover/index.js | 1 + .../es_archiver/date_nanos_mixed/data.json | 140 ++ .../date_nanos_mixed/mappings.json | 1174 +++++++++++++++++ 4 files changed, 1363 insertions(+) create mode 100644 test/functional/apps/discover/_date_nanos_mixed.js create mode 100644 test/functional/fixtures/es_archiver/date_nanos_mixed/data.json create mode 100644 test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json diff --git a/test/functional/apps/discover/_date_nanos_mixed.js b/test/functional/apps/discover/_date_nanos_mixed.js new file mode 100644 index 0000000000000..741d9897bfe8a --- /dev/null +++ b/test/functional/apps/discover/_date_nanos_mixed.js @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import expect from '@kbn/expect'; + +export default function ({ getService, getPageObjects }) { + const esArchiver = getService('esArchiver'); + const PageObjects = getPageObjects(['common', 'timePicker', 'discover']); + const kibanaServer = getService('kibanaServer'); + const fromTime = '2019-01-01 00:00:00.000'; + const toTime = '2019-01-01 23:59:59.999'; + + describe('date_nanos_mixed', function () { + + before(async function () { + await esArchiver.loadIfNeeded('date_nanos_mixed'); + await kibanaServer.uiSettings.replace({ 'defaultIndex': 'timestamp-mixed-*' }); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + }); + + after(function unloadMakelogs() { + return esArchiver.unload('date_nanos_mixed'); + }); + + it('should show a timestamp with nanoseconds in the right order result row', async function () { + const rowData = await PageObjects.discover.getDocTableIndex(1); + expect(rowData.startsWith('Sep 22, 2019 @ 23:50:13.253123345')).to.be.ok(); + }); + }); + +} diff --git a/test/functional/apps/discover/index.js b/test/functional/apps/discover/index.js index 9e4430ca71c1f..902490bebd1ac 100644 --- a/test/functional/apps/discover/index.js +++ b/test/functional/apps/discover/index.js @@ -43,5 +43,6 @@ export default function ({ getService, loadTestFile }) { loadTestFile(require.resolve('./_inspector')); loadTestFile(require.resolve('./_doc_navigation')); loadTestFile(require.resolve('./_date_nanos')); + loadTestFile(require.resolve('./_date_nanos_mixed')); }); } diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json new file mode 100644 index 0000000000000..355181cfe69a5 --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json @@ -0,0 +1,140 @@ +{ + "type": "doc", + "value": { + "id": "space:default", + "index": ".kibana_1", + "source": { + "migrationVersion": { + "space": "6.6.0" + }, + "space": { + "_reserved": true, + "color": "#00bfb3", + "description": "This is your default space!", + "disabledFeatures": [ + ], + "name": "Default" + }, + "type": "space", + "updated_at": "2019-09-10T11:14:28.196Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "search:82116b30-d407-11e9-8004-932185690e7b", + "index": ".kibana_1", + "source": { + "migrationVersion": { + "search": "7.4.0" + }, + "references": [ + { + "id": "a66bb350-d402-11e9-8004-932185690e7b", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "c588eb50-d3ca-11e9-8230-fd04ee48c2f4", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "search": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"number\",\"negate\":false,\"params\":{\"query\":123},\"type\":\"phrase\",\"value\":\"123\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"number\":{\"query\":123,\"type\":\"phrase\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "_score", + "desc" + ] + ], + "title": "New Saved Search", + "version": 1 + }, + "type": "search", + "updated_at": "2019-09-10T20:13:52.354Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "index-pattern:timestamp-mixed-*", + "index": ".kibana_1", + "source": { + "index-pattern": { + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date_nanos\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "timeFieldName": "timestamp", + "title": "timestamp-mixed-*" + }, + "migrationVersion": { + "index-pattern": "6.5.0" + }, + "references": [ + ], + "type": "index-pattern", + "updated_at": "2019-09-10T20:14:46.570Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "timestamp-millis", + "source": { + "timestamp": "2019-01-01T12:10:30.124Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "timestamp-millis", + "source": { + "timestamp": "2019-01-01T12:10:30.123Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "timestamp-nanos", + "source": { + "timestamp": "2019-01-01T12:10:30.123456789Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "timestamp-nanos", + "source": { + "timestamp": "2019-01-01T12:10:30.123498765Z" + }, + "type": "_doc" + } +} diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json new file mode 100644 index 0000000000000..c96eca767177d --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json @@ -0,0 +1,1174 @@ +{ + "type": "index", + "value": { + "aliases": { + ".kibana": { + } + }, + "index": ".kibana_1", + "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "apm-telemetry": "07ee1939fa4302c62ddc052ec03fed90", + "canvas-element": "7390014e1091044523666d97247392fc", + "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", + "config": "87aca8fdb053154f11383fce3dbf3edf", + "dashboard": "d00f614b29a80360e1190193fd333bab", + "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e", + "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1", + "index-pattern": "66eccb05066c5a89924f48a9e9736499", + "infrastructure-ui-source": "ddc0ecb18383f6b26101a2fadb2dab0c", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "map": "23d7aa4a720d4938ccde3983f87bd58d", + "maps-telemetry": "a4229f8b16a6820c6d724b7e0c1f729d", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", + "namespace": "2f4316de49999235636386fe51dc06c1", + "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "181661168bbadd1eff5902361e2a0d5c", + "server": "ec97f1c5da1a19609a60874e5af1100c", + "siem-ui-timeline": "1f6f0860ad7bc0dba3e42467ca40470d", + "siem-ui-timeline-note": "8874706eedc49059d4cf0f5094559084", + "siem-ui-timeline-pinned-event": "20638091112f0e14f0e443d512301c29", + "space": "25de8c2deec044392922989cfcf24c54", + "telemetry": "e1c8bc94e443aefd9458932cc0697a4d", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "type": "2f4316de49999235636386fe51dc06c1", + "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "upgrade-assistant-reindex-operation": "a53a20fe086b72c9a86da3cc12dad8a6", + "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", + "url": "c7f66a0df8b1b52f17c28c4adb111105", + "visualization": "52d7a13ad68a150c4525b292d23e12cc" + } + }, + "dynamic": "strict", + "properties": { + "apm-telemetry": { + "properties": { + "has_any_services": { + "type": "boolean" + }, + "services_per_agent": { + "properties": { + "dotnet": { + "null_value": 0, + "type": "long" + }, + "go": { + "null_value": 0, + "type": "long" + }, + "java": { + "null_value": 0, + "type": "long" + }, + "js-base": { + "null_value": 0, + "type": "long" + }, + "nodejs": { + "null_value": 0, + "type": "long" + }, + "python": { + "null_value": 0, + "type": "long" + }, + "ruby": { + "null_value": 0, + "type": "long" + }, + "rum-js": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "canvas-element": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "content": { + "type": "text" + }, + "help": { + "type": "text" + }, + "image": { + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "canvas-workpad": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + }, + "defaultIndex": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "file-upload-telemetry": { + "properties": { + "filesUploadedTotalCount": { + "type": "long" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "typeMeta": { + "type": "keyword" + } + } + }, + "infrastructure-ui-source": { + "properties": { + "description": { + "type": "text" + }, + "fields": { + "properties": { + "container": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "pod": { + "type": "keyword" + }, + "tiebreaker": { + "type": "keyword" + }, + "timestamp": { + "type": "keyword" + } + } + }, + "logAlias": { + "type": "keyword" + }, + "logColumns": { + "properties": { + "fieldColumn": { + "properties": { + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + }, + "messageColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "timestampColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + } + }, + "type": "nested" + }, + "metricAlias": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "map": { + "properties": { + "bounds": { + "type": "geo_shape" + }, + "description": { + "type": "text" + }, + "layerListJSON": { + "type": "text" + }, + "mapStateJSON": { + "type": "text" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "maps-telemetry": { + "properties": { + "attributesPerMap": { + "properties": { + "dataSourcesCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "emsVectorLayersCount": { + "dynamic": "true", + "properties": { + "france_departments": { + "properties": { + "avg": { + "type": "float" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "uk_subdivisions": { + "properties": { + "avg": { + "type": "float" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "usa_states": { + "properties": { + "avg": { + "type": "float" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "world_countries": { + "properties": { + "avg": { + "type": "float" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "layerTypesCount": { + "dynamic": "true", + "properties": { + "VECTOR": { + "properties": { + "avg": { + "type": "float" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "VECTOR_TILE": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "layersCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "mapsTotalCount": { + "type": "long" + }, + "timeCaptured": { + "type": "date" + } + } + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "canvas-workpad": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "dashboard": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "index-pattern": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "map": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "search": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "space": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "visualization": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ml-telemetry": { + "properties": { + "file_data_visualizer": { + "properties": { + "index_creation_count": { + "type": "long" + } + } + } + } + }, + "namespace": { + "type": "keyword" + }, + "query": { + "properties": { + "description": { + "type": "text" + }, + "filters": { + "enabled": false, + "type": "object" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "index": false, + "type": "keyword" + } + } + }, + "timefilter": { + "enabled": false, + "type": "object" + }, + "title": { + "type": "text" + } + } + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "siem-ui-timeline": { + "properties": { + "columns": { + "properties": { + "aggregatable": { + "type": "boolean" + }, + "category": { + "type": "keyword" + }, + "columnHeaderType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "example": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "indexes": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "placeholder": { + "type": "text" + }, + "searchable": { + "type": "boolean" + }, + "type": { + "type": "keyword" + } + } + }, + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "dataProviders": { + "properties": { + "and": { + "properties": { + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" + } + } + } + } + }, + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" + } + } + } + } + }, + "dateRange": { + "properties": { + "end": { + "type": "date" + }, + "start": { + "type": "date" + } + } + }, + "description": { + "type": "text" + }, + "favorite": { + "properties": { + "favoriteDate": { + "type": "date" + }, + "fullName": { + "type": "text" + }, + "keySearch": { + "type": "text" + }, + "userName": { + "type": "text" + } + } + }, + "kqlMode": { + "type": "keyword" + }, + "kqlQuery": { + "properties": { + "filterQuery": { + "properties": { + "kuery": { + "properties": { + "expression": { + "type": "text" + }, + "kind": { + "type": "keyword" + } + } + }, + "serializedQuery": { + "type": "text" + } + } + } + } + }, + "sort": { + "properties": { + "columnId": { + "type": "keyword" + }, + "sortDirection": { + "type": "keyword" + } + } + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-note": { + "properties": { + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "eventId": { + "type": "keyword" + }, + "note": { + "type": "text" + }, + "timelineId": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-pinned-event": { + "properties": { + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "eventId": { + "type": "keyword" + }, + "timelineId": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "disabledFeatures": { + "type": "keyword" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "telemetry": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "type": { + "type": "keyword" + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "updated_at": { + "type": "date" + }, + "upgrade-assistant-reindex-operation": { + "dynamic": "true", + "properties": { + "indexName": { + "type": "keyword" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-telemetry": { + "properties": { + "features": { + "properties": { + "deprecation_logging": { + "properties": { + "enabled": { + "null_value": true, + "type": "boolean" + } + } + } + } + }, + "ui_open": { + "properties": { + "cluster": { + "null_value": 0, + "type": "long" + }, + "indices": { + "null_value": 0, + "type": "long" + }, + "overview": { + "null_value": 0, + "type": "long" + } + } + }, + "ui_reindex": { + "properties": { + "close": { + "null_value": 0, + "type": "long" + }, + "open": { + "null_value": 0, + "type": "long" + }, + "start": { + "null_value": 0, + "type": "long" + }, + "stop": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchRefName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "timestamp-millis", + "mappings": { + "properties": { + "timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "timestamp-nanos", + "mappings": { + "properties": { + "timestamp": { + "type": "date_nanos" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file From 951152b51b46666fbe4da10a2085051ae49c529e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 13 Sep 2019 17:20:17 +0200 Subject: [PATCH 4/8] Add functional test --- .../apps/discover/_date_nanos_mixed.js | 14 ++++++--- .../es_archiver/date_nanos_mixed/data.json | 31 +++---------------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/test/functional/apps/discover/_date_nanos_mixed.js b/test/functional/apps/discover/_date_nanos_mixed.js index 741d9897bfe8a..8c9a7eb4d5f13 100644 --- a/test/functional/apps/discover/_date_nanos_mixed.js +++ b/test/functional/apps/discover/_date_nanos_mixed.js @@ -30,7 +30,7 @@ export default function ({ getService, getPageObjects }) { before(async function () { await esArchiver.loadIfNeeded('date_nanos_mixed'); - await kibanaServer.uiSettings.replace({ 'defaultIndex': 'timestamp-mixed-*' }); + await kibanaServer.uiSettings.replace({ 'defaultIndex': 'timestamp-*' }); await PageObjects.common.navigateToApp('discover'); await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); }); @@ -39,9 +39,15 @@ export default function ({ getService, getPageObjects }) { return esArchiver.unload('date_nanos_mixed'); }); - it('should show a timestamp with nanoseconds in the right order result row', async function () { - const rowData = await PageObjects.discover.getDocTableIndex(1); - expect(rowData.startsWith('Sep 22, 2019 @ 23:50:13.253123345')).to.be.ok(); + it('shows a list of records of indices with date & date_nanos fields in the right order', async function () { + const rowData1 = await PageObjects.discover.getDocTableIndex(1); + expect(rowData1.startsWith('Jan 1, 2019 @ 12:10:30.124000000')).to.be.ok(); + const rowData2 = await PageObjects.discover.getDocTableIndex(3); + expect(rowData2.startsWith('Jan 1, 2019 @ 12:10:30.123498765')).to.be.ok(); + const rowData3 = await PageObjects.discover.getDocTableIndex(5); + expect(rowData3.startsWith('Jan 1, 2019 @ 12:10:30.123456789')).to.be.ok(); + const rowData4 = await PageObjects.discover.getDocTableIndex(7); + expect(rowData4.startsWith('Jan 1, 2019 @ 12:10:30.123000000')).to.be.ok(); }); }); diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json index 355181cfe69a5..235de2e68f6a2 100644 --- a/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json @@ -1,27 +1,3 @@ -{ - "type": "doc", - "value": { - "id": "space:default", - "index": ".kibana_1", - "source": { - "migrationVersion": { - "space": "6.6.0" - }, - "space": { - "_reserved": true, - "color": "#00bfb3", - "description": "This is your default space!", - "disabledFeatures": [ - ], - "name": "Default" - }, - "type": "space", - "updated_at": "2019-09-10T11:14:28.196Z" - }, - "type": "_doc" - } -} - { "type": "doc", "value": { @@ -71,13 +47,14 @@ { "type": "doc", "value": { - "id": "index-pattern:timestamp-mixed-*", + "id": "index-pattern:timestamp-*", "index": ".kibana_1", "source": { "index-pattern": { - "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date_nanos\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", "timeFieldName": "timestamp", - "title": "timestamp-mixed-*" + "title": "timestamp-*", + "fieldFormatMap": "{\"timestamp\":{\"id\":\"date_nanos\"}}" }, "migrationVersion": { "index-pattern": "6.5.0" From 0415f13aa99503fc8d2148e5cbf68334c382efda Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 13 Sep 2019 21:57:42 +0200 Subject: [PATCH 5/8] Refactor solution for working sort dir buttons --- .../public/discover/controllers/discover.js | 20 +++++++++- .../public/discover/doc_table/lib/get_sort.js | 38 ++++++++----------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js b/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js index 9f572a6f8bc2e..4e7482fdfff3e 100644 --- a/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js +++ b/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js @@ -880,9 +880,25 @@ function discoverController( }; $scope.updateDataSource = Promise.method(function updateDataSource() { - $scope.searchSource + const { indexPattern, searchSource } = $scope; + const { timeFieldName } = indexPattern; + const sort = getSort($state.sort, indexPattern) + .map(sortPair => { + if (indexPattern.isTimeNanosBased() && sortPair[timeFieldName]) { + // for sorting on indices where the same field can be date or date_nanos + // it's necessary to add the numeric_type + return ({ + [timeFieldName]: { + order: sortPair[timeFieldName], + numeric_type: 'date_nanos' + } + }); + } + return sortPair; + }); + searchSource .setField('size', $scope.opts.sampleSize) - .setField('sort', getSort($state.sort, $scope.indexPattern)) + .setField('sort', sort) .setField('query', !$state.query ? null : $state.query) .setField('filter', queryFilter.getFilters()); }); diff --git a/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js index b8fb97c86f25a..0af57c158e21a 100644 --- a/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js +++ b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js @@ -20,16 +20,19 @@ import _ from 'lodash'; function isSortable(field, indexPattern) { - return indexPattern.fields.byName[field] && indexPattern.fields.byName[field].sortable; + return (indexPattern.fields.byName[field] && indexPattern.fields.byName[field].sortable); } function createSortObject(sortPair, indexPattern) { + if (Array.isArray(sortPair) && sortPair.length === 2 && isSortable(sortPair[0], indexPattern)) { - const [field, direction] = sortPair; + const [ field, direction ] = sortPair; return { [field]: direction }; - } else if (_.isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) { + } + else if (_.isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) { return sortPair; - } else { + } + else { return undefined; } } @@ -42,33 +45,24 @@ function createSortObject(sortPair, indexPattern) { * @returns {object} a sort object suitable for returning to elasticsearch */ export function getSort(sort, indexPattern, defaultSortOrder = 'desc') { + let sortObjects; if (Array.isArray(sort)) { - sortObjects = _.compact(sort.map(sortPair => createSortObject(sortPair, indexPattern))); + sortObjects = _.compact(sort.map((sortPair) => createSortObject(sortPair, indexPattern))); } if (!_.isEmpty(sortObjects)) { return sortObjects; - } else if (indexPattern.timeFieldName && isSortable(indexPattern.timeFieldName, indexPattern)) { - if (indexPattern.isTimeNanosBased()) { - // is necessary for sorting indices with date AND date_nanos - return [ - { - [indexPattern.timeFieldName]: { order: defaultSortOrder, numeric_type: 'date_nanos' }, - }, - ]; - } else { - return [{ [indexPattern.timeFieldName]: defaultSortOrder }]; - } - } else { + } + else if (indexPattern.timeFieldName && isSortable(indexPattern.timeFieldName, indexPattern)) { + return [{ [indexPattern.timeFieldName]: defaultSortOrder }]; + } + else { return [{ _score: 'desc' }]; } } getSort.array = function (sort, indexPattern, defaultSortOrder) { - return getSort(sort, indexPattern, defaultSortOrder).map(sortPair => - _(sortPair) - .pairs() - .pop() - ); + return getSort(sort, indexPattern, defaultSortOrder).map((sortPair) => _(sortPair).pairs().pop()); }; + From bd6381a7d6bfcd17cf6e8ddf1e0d85065deea857 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Sat, 14 Sep 2019 09:24:24 +0200 Subject: [PATCH 6/8] Clean up mapping --- .../date_nanos_mixed/mappings.json | 1133 +---------------- 1 file changed, 1 insertion(+), 1132 deletions(-) diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json index c96eca767177d..c62918abced58 100644 --- a/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json @@ -1,1134 +1,3 @@ -{ - "type": "index", - "value": { - "aliases": { - ".kibana": { - } - }, - "index": ".kibana_1", - "mappings": { - "_meta": { - "migrationMappingPropertyHashes": { - "apm-telemetry": "07ee1939fa4302c62ddc052ec03fed90", - "canvas-element": "7390014e1091044523666d97247392fc", - "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", - "config": "87aca8fdb053154f11383fce3dbf3edf", - "dashboard": "d00f614b29a80360e1190193fd333bab", - "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e", - "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1", - "index-pattern": "66eccb05066c5a89924f48a9e9736499", - "infrastructure-ui-source": "ddc0ecb18383f6b26101a2fadb2dab0c", - "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", - "map": "23d7aa4a720d4938ccde3983f87bd58d", - "maps-telemetry": "a4229f8b16a6820c6d724b7e0c1f729d", - "migrationVersion": "4a1746014a75ade3a714e1db5763276f", - "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", - "namespace": "2f4316de49999235636386fe51dc06c1", - "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", - "references": "7997cf5a56cc02bdc9c93361bde732b0", - "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", - "search": "181661168bbadd1eff5902361e2a0d5c", - "server": "ec97f1c5da1a19609a60874e5af1100c", - "siem-ui-timeline": "1f6f0860ad7bc0dba3e42467ca40470d", - "siem-ui-timeline-note": "8874706eedc49059d4cf0f5094559084", - "siem-ui-timeline-pinned-event": "20638091112f0e14f0e443d512301c29", - "space": "25de8c2deec044392922989cfcf24c54", - "telemetry": "e1c8bc94e443aefd9458932cc0697a4d", - "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", - "type": "2f4316de49999235636386fe51dc06c1", - "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", - "updated_at": "00da57df13e94e9d98437d13ace4bfe0", - "upgrade-assistant-reindex-operation": "a53a20fe086b72c9a86da3cc12dad8a6", - "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", - "url": "c7f66a0df8b1b52f17c28c4adb111105", - "visualization": "52d7a13ad68a150c4525b292d23e12cc" - } - }, - "dynamic": "strict", - "properties": { - "apm-telemetry": { - "properties": { - "has_any_services": { - "type": "boolean" - }, - "services_per_agent": { - "properties": { - "dotnet": { - "null_value": 0, - "type": "long" - }, - "go": { - "null_value": 0, - "type": "long" - }, - "java": { - "null_value": 0, - "type": "long" - }, - "js-base": { - "null_value": 0, - "type": "long" - }, - "nodejs": { - "null_value": 0, - "type": "long" - }, - "python": { - "null_value": 0, - "type": "long" - }, - "ruby": { - "null_value": 0, - "type": "long" - }, - "rum-js": { - "null_value": 0, - "type": "long" - } - } - } - } - }, - "canvas-element": { - "dynamic": "false", - "properties": { - "@created": { - "type": "date" - }, - "@timestamp": { - "type": "date" - }, - "content": { - "type": "text" - }, - "help": { - "type": "text" - }, - "image": { - "type": "text" - }, - "name": { - "fields": { - "keyword": { - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "canvas-workpad": { - "dynamic": "false", - "properties": { - "@created": { - "type": "date" - }, - "@timestamp": { - "type": "date" - }, - "name": { - "fields": { - "keyword": { - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "config": { - "dynamic": "true", - "properties": { - "buildNum": { - "type": "keyword" - }, - "defaultIndex": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "dashboard": { - "properties": { - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "optionsJSON": { - "type": "text" - }, - "panelsJSON": { - "type": "text" - }, - "refreshInterval": { - "properties": { - "display": { - "type": "keyword" - }, - "pause": { - "type": "boolean" - }, - "section": { - "type": "integer" - }, - "value": { - "type": "integer" - } - } - }, - "timeFrom": { - "type": "keyword" - }, - "timeRestore": { - "type": "boolean" - }, - "timeTo": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "file-upload-telemetry": { - "properties": { - "filesUploadedTotalCount": { - "type": "long" - } - } - }, - "graph-workspace": { - "properties": { - "description": { - "type": "text" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "numLinks": { - "type": "integer" - }, - "numVertices": { - "type": "integer" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "wsState": { - "type": "text" - } - } - }, - "index-pattern": { - "properties": { - "fieldFormatMap": { - "type": "text" - }, - "fields": { - "type": "text" - }, - "intervalName": { - "type": "keyword" - }, - "notExpandable": { - "type": "boolean" - }, - "sourceFilters": { - "type": "text" - }, - "timeFieldName": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "type": { - "type": "keyword" - }, - "typeMeta": { - "type": "keyword" - } - } - }, - "infrastructure-ui-source": { - "properties": { - "description": { - "type": "text" - }, - "fields": { - "properties": { - "container": { - "type": "keyword" - }, - "host": { - "type": "keyword" - }, - "pod": { - "type": "keyword" - }, - "tiebreaker": { - "type": "keyword" - }, - "timestamp": { - "type": "keyword" - } - } - }, - "logAlias": { - "type": "keyword" - }, - "logColumns": { - "properties": { - "fieldColumn": { - "properties": { - "field": { - "type": "keyword" - }, - "id": { - "type": "keyword" - } - } - }, - "messageColumn": { - "properties": { - "id": { - "type": "keyword" - } - } - }, - "timestampColumn": { - "properties": { - "id": { - "type": "keyword" - } - } - } - }, - "type": "nested" - }, - "metricAlias": { - "type": "keyword" - }, - "name": { - "type": "text" - } - } - }, - "kql-telemetry": { - "properties": { - "optInCount": { - "type": "long" - }, - "optOutCount": { - "type": "long" - } - } - }, - "map": { - "properties": { - "bounds": { - "type": "geo_shape" - }, - "description": { - "type": "text" - }, - "layerListJSON": { - "type": "text" - }, - "mapStateJSON": { - "type": "text" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "maps-telemetry": { - "properties": { - "attributesPerMap": { - "properties": { - "dataSourcesCount": { - "properties": { - "avg": { - "type": "long" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - }, - "emsVectorLayersCount": { - "dynamic": "true", - "properties": { - "france_departments": { - "properties": { - "avg": { - "type": "float" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - }, - "uk_subdivisions": { - "properties": { - "avg": { - "type": "float" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - }, - "usa_states": { - "properties": { - "avg": { - "type": "float" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - }, - "world_countries": { - "properties": { - "avg": { - "type": "float" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - } - } - }, - "layerTypesCount": { - "dynamic": "true", - "properties": { - "VECTOR": { - "properties": { - "avg": { - "type": "float" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - }, - "VECTOR_TILE": { - "properties": { - "avg": { - "type": "long" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - } - } - }, - "layersCount": { - "properties": { - "avg": { - "type": "long" - }, - "max": { - "type": "long" - }, - "min": { - "type": "long" - } - } - } - } - }, - "mapsTotalCount": { - "type": "long" - }, - "timeCaptured": { - "type": "date" - } - } - }, - "migrationVersion": { - "dynamic": "true", - "properties": { - "canvas-workpad": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "dashboard": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "index-pattern": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "map": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "search": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "space": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "visualization": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "ml-telemetry": { - "properties": { - "file_data_visualizer": { - "properties": { - "index_creation_count": { - "type": "long" - } - } - } - } - }, - "namespace": { - "type": "keyword" - }, - "query": { - "properties": { - "description": { - "type": "text" - }, - "filters": { - "enabled": false, - "type": "object" - }, - "query": { - "properties": { - "language": { - "type": "keyword" - }, - "query": { - "index": false, - "type": "keyword" - } - } - }, - "timefilter": { - "enabled": false, - "type": "object" - }, - "title": { - "type": "text" - } - } - }, - "references": { - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - }, - "type": "nested" - }, - "sample-data-telemetry": { - "properties": { - "installCount": { - "type": "long" - }, - "unInstallCount": { - "type": "long" - } - } - }, - "search": { - "properties": { - "columns": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "sort": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "server": { - "properties": { - "uuid": { - "type": "keyword" - } - } - }, - "siem-ui-timeline": { - "properties": { - "columns": { - "properties": { - "aggregatable": { - "type": "boolean" - }, - "category": { - "type": "keyword" - }, - "columnHeaderType": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "example": { - "type": "text" - }, - "id": { - "type": "keyword" - }, - "indexes": { - "type": "keyword" - }, - "name": { - "type": "text" - }, - "placeholder": { - "type": "text" - }, - "searchable": { - "type": "boolean" - }, - "type": { - "type": "keyword" - } - } - }, - "created": { - "type": "date" - }, - "createdBy": { - "type": "text" - }, - "dataProviders": { - "properties": { - "and": { - "properties": { - "enabled": { - "type": "boolean" - }, - "excluded": { - "type": "boolean" - }, - "id": { - "type": "keyword" - }, - "kqlQuery": { - "type": "text" - }, - "name": { - "type": "text" - }, - "queryMatch": { - "properties": { - "displayField": { - "type": "text" - }, - "displayValue": { - "type": "text" - }, - "field": { - "type": "text" - }, - "operator": { - "type": "text" - }, - "value": { - "type": "text" - } - } - } - } - }, - "enabled": { - "type": "boolean" - }, - "excluded": { - "type": "boolean" - }, - "id": { - "type": "keyword" - }, - "kqlQuery": { - "type": "text" - }, - "name": { - "type": "text" - }, - "queryMatch": { - "properties": { - "displayField": { - "type": "text" - }, - "displayValue": { - "type": "text" - }, - "field": { - "type": "text" - }, - "operator": { - "type": "text" - }, - "value": { - "type": "text" - } - } - } - } - }, - "dateRange": { - "properties": { - "end": { - "type": "date" - }, - "start": { - "type": "date" - } - } - }, - "description": { - "type": "text" - }, - "favorite": { - "properties": { - "favoriteDate": { - "type": "date" - }, - "fullName": { - "type": "text" - }, - "keySearch": { - "type": "text" - }, - "userName": { - "type": "text" - } - } - }, - "kqlMode": { - "type": "keyword" - }, - "kqlQuery": { - "properties": { - "filterQuery": { - "properties": { - "kuery": { - "properties": { - "expression": { - "type": "text" - }, - "kind": { - "type": "keyword" - } - } - }, - "serializedQuery": { - "type": "text" - } - } - } - } - }, - "sort": { - "properties": { - "columnId": { - "type": "keyword" - }, - "sortDirection": { - "type": "keyword" - } - } - }, - "title": { - "type": "text" - }, - "updated": { - "type": "date" - }, - "updatedBy": { - "type": "text" - } - } - }, - "siem-ui-timeline-note": { - "properties": { - "created": { - "type": "date" - }, - "createdBy": { - "type": "text" - }, - "eventId": { - "type": "keyword" - }, - "note": { - "type": "text" - }, - "timelineId": { - "type": "keyword" - }, - "updated": { - "type": "date" - }, - "updatedBy": { - "type": "text" - } - } - }, - "siem-ui-timeline-pinned-event": { - "properties": { - "created": { - "type": "date" - }, - "createdBy": { - "type": "text" - }, - "eventId": { - "type": "keyword" - }, - "timelineId": { - "type": "keyword" - }, - "updated": { - "type": "date" - }, - "updatedBy": { - "type": "text" - } - } - }, - "space": { - "properties": { - "_reserved": { - "type": "boolean" - }, - "color": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "disabledFeatures": { - "type": "keyword" - }, - "initials": { - "type": "keyword" - }, - "name": { - "fields": { - "keyword": { - "ignore_above": 2048, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "telemetry": { - "properties": { - "enabled": { - "type": "boolean" - } - } - }, - "timelion-sheet": { - "properties": { - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "timelion_chart_height": { - "type": "integer" - }, - "timelion_columns": { - "type": "integer" - }, - "timelion_interval": { - "type": "keyword" - }, - "timelion_other_interval": { - "type": "keyword" - }, - "timelion_rows": { - "type": "integer" - }, - "timelion_sheet": { - "type": "text" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "type": { - "type": "keyword" - }, - "ui-metric": { - "properties": { - "count": { - "type": "integer" - } - } - }, - "updated_at": { - "type": "date" - }, - "upgrade-assistant-reindex-operation": { - "dynamic": "true", - "properties": { - "indexName": { - "type": "keyword" - }, - "status": { - "type": "integer" - } - } - }, - "upgrade-assistant-telemetry": { - "properties": { - "features": { - "properties": { - "deprecation_logging": { - "properties": { - "enabled": { - "null_value": true, - "type": "boolean" - } - } - } - } - }, - "ui_open": { - "properties": { - "cluster": { - "null_value": 0, - "type": "long" - }, - "indices": { - "null_value": 0, - "type": "long" - }, - "overview": { - "null_value": 0, - "type": "long" - } - } - }, - "ui_reindex": { - "properties": { - "close": { - "null_value": 0, - "type": "long" - }, - "open": { - "null_value": 0, - "type": "long" - }, - "start": { - "null_value": 0, - "type": "long" - }, - "stop": { - "null_value": 0, - "type": "long" - } - } - } - } - }, - "url": { - "properties": { - "accessCount": { - "type": "long" - }, - "accessDate": { - "type": "date" - }, - "createDate": { - "type": "date" - }, - "url": { - "fields": { - "keyword": { - "ignore_above": 2048, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "visualization": { - "properties": { - "description": { - "type": "text" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "savedSearchRefName": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "visState": { - "type": "text" - } - } - } - } - }, - "settings": { - "index": { - "auto_expand_replicas": "0-1", - "number_of_replicas": "0", - "number_of_shards": "1" - } - } - } -} - { "type": "index", "value": { @@ -1171,4 +40,4 @@ } } } -} \ No newline at end of file +} From 9d839eaae78a78df8d35ec9b1df35e710ad07eee Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Mon, 16 Sep 2019 21:26:36 +0200 Subject: [PATCH 7/8] Adapt testdata --- .../es_archiver/date_nanos_mixed/data.json | 64 ++++++------------- 1 file changed, 21 insertions(+), 43 deletions(-) diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json index 235de2e68f6a2..abde15e2b08c4 100644 --- a/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json @@ -1,24 +1,27 @@ { "type": "doc", "value": { - "id": "search:82116b30-d407-11e9-8004-932185690e7b", - "index": ".kibana_1", + "id": "index-pattern:timestamp-*", + "index": ".kibana", "source": { - "migrationVersion": { - "search": "7.4.0" + "index-pattern": { + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "timeFieldName": "timestamp", + "title": "timestamp-*", + "fieldFormatMap": "{\"timestamp\":{\"id\":\"date_nanos\"}}" }, - "references": [ - { - "id": "a66bb350-d402-11e9-8004-932185690e7b", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - }, - { - "id": "c588eb50-d3ca-11e9-8230-fd04ee48c2f4", - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "type": "index-pattern" - } - ], + "type": "index-pattern" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "search:82116b30-d407-11e9-8004-932185690e7b", + "index": ".kibana", + "source": { "search": { "columns": [ "_source" @@ -30,39 +33,14 @@ }, "sort": [ [ - "_score", + "@timestamp", "desc" ] ], "title": "New Saved Search", "version": 1 }, - "type": "search", - "updated_at": "2019-09-10T20:13:52.354Z" - }, - "type": "_doc" - } -} - -{ - "type": "doc", - "value": { - "id": "index-pattern:timestamp-*", - "index": ".kibana_1", - "source": { - "index-pattern": { - "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", - "timeFieldName": "timestamp", - "title": "timestamp-*", - "fieldFormatMap": "{\"timestamp\":{\"id\":\"date_nanos\"}}" - }, - "migrationVersion": { - "index-pattern": "6.5.0" - }, - "references": [ - ], - "type": "index-pattern", - "updated_at": "2019-09-10T20:14:46.570Z" + "type": "search" }, "type": "_doc" } From 54cac208dd09b47ff3e06c4e4de29a73c9bfeb94 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 18 Sep 2019 09:26:45 +0200 Subject: [PATCH 8/8] Migrate code to getSortForSearchSource, adapt usage --- .../public/discover/controllers/discover.js | 20 ++------- .../lib/get_sort_for_search_source.ts | 45 +++++++++++++++++++ .../discover/embeddable/search_embeddable.ts | 14 +++--- 3 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts diff --git a/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js b/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js index 4e7482fdfff3e..a54f278f3a937 100644 --- a/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js +++ b/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js @@ -29,6 +29,7 @@ import dateMath from '@elastic/datemath'; // doc table import '../doc_table'; import { getSort } from '../doc_table/lib/get_sort'; +import { getSortForSearchSource } from '../doc_table/lib/get_sort_for_search_source'; import * as columnActions from '../doc_table/actions/columns'; import * as filterActions from '../doc_table/actions/filter'; @@ -480,7 +481,7 @@ function discoverController( const { searchFields, selectFields } = await getSharingDataFields(); searchSource.setField('fields', searchFields); - searchSource.setField('sort', getSort($state.sort, $scope.indexPattern)); + searchSource.setField('sort', getSortForSearchSource($state.sort, $scope.indexPattern)); searchSource.setField('highlight', null); searchSource.setField('highlightAll', null); searchSource.setField('aggs', null); @@ -881,24 +882,9 @@ function discoverController( $scope.updateDataSource = Promise.method(function updateDataSource() { const { indexPattern, searchSource } = $scope; - const { timeFieldName } = indexPattern; - const sort = getSort($state.sort, indexPattern) - .map(sortPair => { - if (indexPattern.isTimeNanosBased() && sortPair[timeFieldName]) { - // for sorting on indices where the same field can be date or date_nanos - // it's necessary to add the numeric_type - return ({ - [timeFieldName]: { - order: sortPair[timeFieldName], - numeric_type: 'date_nanos' - } - }); - } - return sortPair; - }); searchSource .setField('size', $scope.opts.sampleSize) - .setField('sort', sort) + .setField('sort', getSortForSearchSource($state.sort, indexPattern)) .setField('query', !$state.query ? null : $state.query) .setField('filter', queryFilter.getFilters()); }); diff --git a/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts new file mode 100644 index 0000000000000..95adc20d89f81 --- /dev/null +++ b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { IndexPattern } from 'ui/index_patterns'; +import { SortOrder } from '../components/table_header/helpers'; +import { getSort } from './get_sort'; + +/** + * prepares sort for search source, that's sending the request to ES + * handles the special case when there's sorting by date_nanos typed fields + * the addon of the numeric_type guarantees the right sort order + * when there are indices with date and indices with date_nanos field + */ +export function getSortForSearchSource(sort?: SortOrder[], indexPattern?: IndexPattern) { + if (!sort || !indexPattern) { + return []; + } + const { timeFieldName } = indexPattern; + return getSort(sort, indexPattern).map((sortPair: Record) => { + if (indexPattern.isTimeNanosBased() && timeFieldName && sortPair[timeFieldName]) { + return { + [timeFieldName]: { + order: sortPair[timeFieldName], + numeric_type: 'date_nanos', + }, + }; + } + return sortPair; + }); +} diff --git a/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts b/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts index e307d5da7ebe8..9d862a1c36b70 100644 --- a/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts +++ b/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts @@ -26,7 +26,7 @@ import { getRequestInspectorStats, getResponseInspectorStats, } from 'ui/courier/utils/courier_inspector_utils'; -import { StaticIndexPattern } from 'ui/index_patterns'; +import { IndexPattern } from 'ui/index_patterns'; import { RequestAdapter } from 'ui/inspector/adapters'; import { Adapters } from 'ui/inspector/types'; import { Subscription } from 'rxjs'; @@ -48,8 +48,8 @@ import * as columnActions from '../doc_table/actions/columns'; import { SavedSearch } from '../types'; import searchTemplate from './search_template.html'; import { ISearchEmbeddable, SearchInput, SearchOutput } from './types'; -import { getSort } from '../doc_table/lib/get_sort'; import { SortOrder } from '../doc_table/components/table_header/helpers'; +import { getSortForSearchSource } from '../doc_table/lib/get_sort_for_search_source'; const config = chrome.getUiSettingsClient(); @@ -65,7 +65,7 @@ interface SearchScope extends ng.IScope { moveColumn?: (column: string, index: number) => void; filter?: (field: { name: string; scripted: boolean }, value: string[], operator: string) => void; hits?: any[]; - indexPattern?: StaticIndexPattern; + indexPattern?: IndexPattern; totalHitCount?: number; isLoading?: boolean; } @@ -87,7 +87,7 @@ interface SearchEmbeddableConfig { $compile: ng.ICompileService; savedSearch: SavedSearch; editUrl: string; - indexPatterns?: StaticIndexPattern[]; + indexPatterns?: IndexPattern[]; editable: boolean; queryFilter: unknown; } @@ -163,7 +163,6 @@ export class SearchEmbeddable extends Embeddable /** * * @param {Element} domNode - * @param {ContainerState} containerState */ public render(domNode: HTMLElement) { if (!this.searchScope) { @@ -275,7 +274,10 @@ export class SearchEmbeddable extends Embeddable searchSource.cancelQueued(); searchSource.setField('size', config.get('discover:sampleSize')); - searchSource.setField('sort', getSort(this.searchScope.sort, this.searchScope.indexPattern)); + searchSource.setField( + 'sort', + getSortForSearchSource(this.searchScope.sort, this.searchScope.indexPattern) + ); // Log request to inspector this.inspectorAdaptors.requests.reset();