Add numeric_type option for correct sort order on mixed date and date_nanos fields

src/fixtures/stubbed_logstash_index_pattern.js

@@ -42,6 +42,7 @@ export default function stubbedLogstashIndexPatternService(Private) {
 
   const indexPattern = new StubIndexPattern('logstash-*', cfg => cfg, 'time', fields);
   indexPattern.id = 'logstash-*';
+  indexPattern.isTimeNanosBased = () => false;
 
   return indexPattern;
 

src/legacy/core_plugins/kibana/public/discover/controllers/discover.js

@@ -29,6 +29,7 @@ import dateMath from '@elastic/datemath';
 // doc table
 import '../doc_table';
 import { getSort } from '../doc_table/lib/get_sort';
+import { getSortForSearchSource } from '../doc_table/lib/get_sort_for_search_source';
 import * as columnActions from '../doc_table/actions/columns';
 import * as filterActions from '../doc_table/actions/filter';
 
@@ -480,7 +481,7 @@ function discoverController(
 
     const { searchFields, selectFields } = await getSharingDataFields();
     searchSource.setField('fields', searchFields);
-    searchSource.setField('sort', getSort($state.sort, $scope.indexPattern));
+    searchSource.setField('sort', getSortForSearchSource($state.sort, $scope.indexPattern));
     searchSource.setField('highlight', null);
     searchSource.setField('highlightAll', null);
     searchSource.setField('aggs', null);
@@ -880,9 +881,10 @@ function discoverController(
   };
 
   $scope.updateDataSource = Promise.method(function updateDataSource() {
-    $scope.searchSource
+    const { indexPattern, searchSource } = $scope;
+    searchSource
       .setField('size', $scope.opts.sampleSize)
-      .setField('sort', getSort($state.sort, $scope.indexPattern))
+      .setField('sort', getSortForSearchSource($state.sort, indexPattern))
       .setField('query', !$state.query ? null : $state.query)
       .setField('filter', queryFilter.getFilters());
   });

src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { IndexPattern } from 'ui/index_patterns';
+import { SortOrder } from '../components/table_header/helpers';
+import { getSort } from './get_sort';
+
+/**
+ * prepares sort for search source, that's sending the request to ES
+ * handles the special case when there's sorting by date_nanos typed fields
+ * the addon of the numeric_type guarantees the right sort order
+ * when there are indices with date and indices with date_nanos field
+ */
+export function getSortForSearchSource(sort?: SortOrder[], indexPattern?: IndexPattern) {
+  if (!sort || !indexPattern) {
+    return [];
+  }
+  const { timeFieldName } = indexPattern;
+  return getSort(sort, indexPattern).map((sortPair: Record<string, string>) => {
+    if (indexPattern.isTimeNanosBased() && timeFieldName && sortPair[timeFieldName]) {
+      return {
+        [timeFieldName]: {
+          order: sortPair[timeFieldName],
+          numeric_type: 'date_nanos',
+        },
+      };
+    }
+    return sortPair;
+  });
+}

src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts

@@ -26,7 +26,7 @@ import {
   getRequestInspectorStats,
   getResponseInspectorStats,
 } from 'ui/courier/utils/courier_inspector_utils';
-import { StaticIndexPattern } from 'ui/index_patterns';
+import { IndexPattern } from 'ui/index_patterns';
 import { RequestAdapter } from 'ui/inspector/adapters';
 import { Adapters } from 'ui/inspector/types';
 import { Subscription } from 'rxjs';
@@ -48,8 +48,8 @@ import * as columnActions from '../doc_table/actions/columns';
 import { SavedSearch } from '../types';
 import searchTemplate from './search_template.html';
 import { ISearchEmbeddable, SearchInput, SearchOutput } from './types';
-import { getSort } from '../doc_table/lib/get_sort';
 import { SortOrder } from '../doc_table/components/table_header/helpers';
+import { getSortForSearchSource } from '../doc_table/lib/get_sort_for_search_source';
 
 const config = chrome.getUiSettingsClient();
 
@@ -65,7 +65,7 @@ interface SearchScope extends ng.IScope {
   moveColumn?: (column: string, index: number) => void;
   filter?: (field: { name: string; scripted: boolean }, value: string[], operator: string) => void;
   hits?: any[];
-  indexPattern?: StaticIndexPattern;
+  indexPattern?: IndexPattern;
   totalHitCount?: number;
   isLoading?: boolean;
 }
@@ -87,7 +87,7 @@ interface SearchEmbeddableConfig {
   $compile: ng.ICompileService;
   savedSearch: SavedSearch;
   editUrl: string;
-  indexPatterns?: StaticIndexPattern[];
+  indexPatterns?: IndexPattern[];
   editable: boolean;
   queryFilter: unknown;
 }
@@ -163,7 +163,6 @@ export class SearchEmbeddable extends Embeddable<SearchInput, SearchOutput>
   /**
    *
    * @param {Element} domNode
-   * @param {ContainerState} containerState
    */
   public render(domNode: HTMLElement) {
     if (!this.searchScope) {
@@ -275,7 +274,10 @@ export class SearchEmbeddable extends Embeddable<SearchInput, SearchOutput>
     searchSource.cancelQueued();
 
     searchSource.setField('size', config.get('discover:sampleSize'));
-    searchSource.setField('sort', getSort(this.searchScope.sort, this.searchScope.indexPattern));
+    searchSource.setField(
+      'sort',
+      getSortForSearchSource(this.searchScope.sort, this.searchScope.indexPattern)
+    );
 
     // Log request to inspector
     this.inspectorAdaptors.requests.reset();

test/functional/apps/discover/_date_nanos_mixed.js

@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function ({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const PageObjects = getPageObjects(['common', 'timePicker', 'discover']);
+  const kibanaServer = getService('kibanaServer');
+  const fromTime = '2019-01-01 00:00:00.000';
+  const toTime = '2019-01-01 23:59:59.999';
+
+  describe('date_nanos_mixed', function () {
+
+    before(async function () {
+      await esArchiver.loadIfNeeded('date_nanos_mixed');
+      await kibanaServer.uiSettings.replace({ 'defaultIndex': 'timestamp-*' });
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
+    });
+
+    after(function unloadMakelogs() {
+      return esArchiver.unload('date_nanos_mixed');
+    });
+
+    it('shows a list of records of indices with date & date_nanos fields in the right order', async function () {
+      const rowData1 = await PageObjects.discover.getDocTableIndex(1);
+      expect(rowData1.startsWith('Jan 1, 2019 @ 12:10:30.124000000')).to.be.ok();
+      const rowData2 = await PageObjects.discover.getDocTableIndex(3);
+      expect(rowData2.startsWith('Jan 1, 2019 @ 12:10:30.123498765')).to.be.ok();
+      const rowData3 = await PageObjects.discover.getDocTableIndex(5);
+      expect(rowData3.startsWith('Jan 1, 2019 @ 12:10:30.123456789')).to.be.ok();
+      const rowData4 = await PageObjects.discover.getDocTableIndex(7);
+      expect(rowData4.startsWith('Jan 1, 2019 @ 12:10:30.123000000')).to.be.ok();
+    });
+  });
+
+}

test/functional/apps/discover/index.js

@@ -43,5 +43,6 @@ export default function ({ getService, loadTestFile }) {
     loadTestFile(require.resolve('./_inspector'));
     loadTestFile(require.resolve('./_doc_navigation'));
     loadTestFile(require.resolve('./_date_nanos'));
+    loadTestFile(require.resolve('./_date_nanos_mixed'));
   });
 }

test/functional/fixtures/es_archiver/date_nanos_mixed/data.json

@@ -0,0 +1,95 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "index-pattern:timestamp-*",
+    "index": ".kibana",
+    "source": {
+      "index-pattern": {
+        "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+        "timeFieldName": "timestamp",
+        "title": "timestamp-*",
+        "fieldFormatMap": "{\"timestamp\":{\"id\":\"date_nanos\"}}"
+      },
+      "type": "index-pattern"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "search:82116b30-d407-11e9-8004-932185690e7b",
+    "index": ".kibana",
+    "source": {
+      "search": {
+        "columns": [
+          "_source"
+        ],
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"number\",\"negate\":false,\"params\":{\"query\":123},\"type\":\"phrase\",\"value\":\"123\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"number\":{\"query\":123,\"type\":\"phrase\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
+        },
+        "sort": [
+          [
+            "@timestamp",
+            "desc"
+          ]
+        ],
+        "title": "New Saved Search",
+        "version": 1
+      },
+      "type": "search"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "timestamp-millis",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.124Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "timestamp-millis",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.123Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "timestamp-nanos",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.123456789Z"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "4",
+    "index": "timestamp-nanos",
+    "source": {
+      "timestamp": "2019-01-01T12:10:30.123498765Z"
+    },
+    "type": "_doc"
+  }
+}

test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json

@@ -0,0 +1,43 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": "timestamp-millis",
+    "mappings": {
+      "properties": {
+        "timestamp": {
+          "type": "date"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}
+
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": "timestamp-nanos",
+    "mappings": {
+      "properties": {
+        "timestamp": {
+          "type": "date_nanos"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}