Add numeric_type option for correct sort order on mixed date and date_nanos fields

src/fixtures/stubbed_logstash_index_pattern.js

@@ -42,6 +42,7 @@ export default function stubbedLogstashIndexPatternService(Private) {
 
   const indexPattern = new StubIndexPattern('logstash-*', cfg => cfg, 'time', fields);
   indexPattern.id = 'logstash-*';
+  indexPattern.isTimeNanosBased = () => false;
 
   return indexPattern;
 

src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort.js

@@ -20,19 +20,16 @@
 import _ from 'lodash';
 
 function isSortable(field, indexPattern) {
-  return (indexPattern.fields.byName[field] && indexPattern.fields.byName[field].sortable);
+  return indexPattern.fields.byName[field] && indexPattern.fields.byName[field].sortable;
 }
 
 function createSortObject(sortPair, indexPattern) {
-
   if (Array.isArray(sortPair) && sortPair.length === 2 && isSortable(sortPair[0], indexPattern)) {
-    const [ field, direction ] = sortPair;
+    const [field, direction] = sortPair;
     return { [field]: direction };
-  }
-  else if (_.isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) {
+  } else if (_.isPlainObject(sortPair) && isSortable(Object.keys(sortPair)[0], indexPattern)) {
     return sortPair;
-  }
-  else {
+  } else {
     return undefined;
   }
 }
@@ -45,24 +42,33 @@ function createSortObject(sortPair, indexPattern) {
  * @returns {object} a sort object suitable for returning to elasticsearch
  */
 export function getSort(sort, indexPattern, defaultSortOrder = 'desc') {
-
   let sortObjects;
   if (Array.isArray(sort)) {
-    sortObjects = _.compact(sort.map((sortPair) => createSortObject(sortPair, indexPattern)));
+    sortObjects = _.compact(sort.map(sortPair => createSortObject(sortPair, indexPattern)));
   }
 
   if (!_.isEmpty(sortObjects)) {
     return sortObjects;
-  }
-  else if (indexPattern.timeFieldName && isSortable(indexPattern.timeFieldName, indexPattern)) {
-    return [{ [indexPattern.timeFieldName]: defaultSortOrder }];
-  }
-  else {
+  } else if (indexPattern.timeFieldName && isSortable(indexPattern.timeFieldName, indexPattern)) {
+    if (indexPattern.isTimeNanosBased()) {
+      // is necessary for sorting indices with date AND date_nanos
+      return [
+        {
+          [indexPattern.timeFieldName]: { order: defaultSortOrder, numeric_type: 'date_nanos' },
+        },
+      ];
+    } else {
+      return [{ [indexPattern.timeFieldName]: defaultSortOrder }];
+    }
+  } else {
     return [{ _score: 'desc' }];
   }
 }
 
 getSort.array = function (sort, indexPattern, defaultSortOrder) {
-  return getSort(sort, indexPattern, defaultSortOrder).map((sortPair) => _(sortPair).pairs().pop());
+  return getSort(sort, indexPattern, defaultSortOrder).map(sortPair =>
+    _(sortPair)
+      .pairs()
+      .pop()
+  );
 };
-

test/functional/apps/discover/_date_nanos_mixed.js

@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function ({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const PageObjects = getPageObjects(['common', 'timePicker', 'discover']);
+  const kibanaServer = getService('kibanaServer');
+  const fromTime = '2019-01-01 00:00:00.000';
+  const toTime = '2019-01-01 23:59:59.999';
+
+  describe('date_nanos_mixed', function () {
+
+    before(async function () {
+      await esArchiver.loadIfNeeded('date_nanos_mixed');
+      await kibanaServer.uiSettings.replace({ 'defaultIndex': 'timestamp-*' });
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
+    });
+
+    after(function unloadMakelogs() {
+      return esArchiver.unload('date_nanos_mixed');
+    });
+
+    it('shows a list of records of indices with date & date_nanos fields in the right order', async function () {
+      const rowData1 = await PageObjects.discover.getDocTableIndex(1);
+      expect(rowData1.startsWith('Jan 1, 2019 @ 12:10:30.124000000')).to.be.ok();
+      const rowData2 = await PageObjects.discover.getDocTableIndex(3);
+      expect(rowData2.startsWith('Jan 1, 2019 @ 12:10:30.123498765')).to.be.ok();
+      const rowData3 = await PageObjects.discover.getDocTableIndex(5);
+      expect(rowData3.startsWith('Jan 1, 2019 @ 12:10:30.123456789')).to.be.ok();
+      const rowData4 = await PageObjects.discover.getDocTableIndex(7);
+      expect(rowData4.startsWith('Jan 1, 2019 @ 12:10:30.123000000')).to.be.ok();
+    });
+  });
+
+}

test/functional/apps/discover/index.js

@@ -43,5 +43,6 @@ export default function ({ getService, loadTestFile }) {
     loadTestFile(require.resolve('./_inspector'));
     loadTestFile(require.resolve('./_doc_navigation'));
     loadTestFile(require.resolve('./_date_nanos'));
+    loadTestFile(require.resolve('./_date_nanos_mixed'));
   });
 }

test/functional/fixtures/es_archiver/date_nanos_mixed/data.json

@@ -0,0 +1,117 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "search:82116b30-d407-11e9-8004-932185690e7b",
+    "index": ".kibana_1",
+    "source": {
+      "migrationVersion": {
+        "search": "7.4.0"
+      },
+      "references": [
+        {
+          "id": "a66bb350-d402-11e9-8004-932185690e7b",
+          "name": "kibanaSavedObjectMeta.searchSourceJSON.index",
+          "type": "index-pattern"
+        },
+        {
+          "id": "c588eb50-d3ca-11e9-8230-fd04ee48c2f4",
+          "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
+          "type": "index-pattern"
+        }
+      ],
+      "search": {
+        "columns": [
+          "_source"
+        ],
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"number\",\"negate\":false,\"params\":{\"query\":123},\"type\":\"phrase\",\"value\":\"123\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"number\":{\"query\":123,\"type\":\"phrase\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
+        },
+        "sort": [
+          [
+            "_score",
+            "desc"
+          ]
+        ],
+        "title": "New Saved Search",
+        "version": 1
+      },
+      "type": "search",
+      "updated_at": "2019-09-10T20:13:52.354Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "index-pattern:timestamp-*",
+    "index": ".kibana_1",
+    "source": {
+      "index-pattern": {
+        "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+        "timeFieldName": "timestamp",
+        "title": "timestamp-*",
+        "fieldFormatMap": "{\"timestamp\":{\"id\":\"date_nanos\"}}"
+      },
+      "migrationVersion": {
+        "index-pattern": "6.5.0"
+      },
+      "references": [
+      ],
+      "type": "index-pattern",
+      "updated_at": "2019-09-10T20:14:46.570Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "timestamp-millis",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.124Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "timestamp-millis",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.123Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "timestamp-nanos",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.123456789Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "4",
+    "index": "timestamp-nanos",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.123498765Z"
+    },
+    "type": "_doc"
+  }
+}