From 4662f351fdaa3418719da7897a3175f6ef7c2696 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 19 Aug 2020 11:26:44 +0200 Subject: [PATCH 01/11] Convert ES body of discover doc request to use fields API --- .../public/application/components/doc/use_es_doc_search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts b/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts index 522ebad1691a9..8a20237b65708 100644 --- a/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts +++ b/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts @@ -45,7 +45,7 @@ export function buildSearchBody(id: string, indexPattern: IndexPattern): Record< stored_fields: computedFields.storedFields, _source: true, script_fields: computedFields.scriptFields, - docvalue_fields: computedFields.docvalueFields, + fields: computedFields.docvalueFields, }; } From 379fd8caa96a77468d80e740253238bd45887328 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 19 Aug 2020 12:04:34 +0200 Subject: [PATCH 02/11] Adapt search_source.ts and discover to display alias fields --- src/plugins/data/public/search/search_source/search_source.ts | 2 +- src/plugins/discover/public/application/angular/discover.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/search/search_source/search_source.ts b/src/plugins/data/public/search/search_source/search_source.ts index 847dc8853d6ba..2997ab8b2ec11 100644 --- a/src/plugins/data/public/search/search_source/search_source.ts +++ b/src/plugins/data/public/search/search_source/search_source.ts @@ -439,7 +439,7 @@ export class SearchSource { // request the remaining fields from both stored_fields and _source const remainingFields = difference(fields, keys(body.script_fields)); - body.stored_fields = remainingFields; + body.fields = remainingFields; setWith(body, '_source.includes', remainingFields, (nsValue) => isObject(nsValue) ? {} : nsValue ); diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 22da3e877054a..62d3aaa68dbff 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -950,6 +950,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise const { indexPattern, searchSource } = $scope; searchSource .setField('index', $scope.indexPattern) + .setField('fields', $scope.state.columns) .setField('size', $scope.opts.sampleSize) .setField( 'sort', From 008bdc164f4308c8d9a77d9e2416721a1698ffe5 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 19 Aug 2020 12:52:16 +0200 Subject: [PATCH 03/11] Add source to field --- src/plugins/discover/public/application/angular/discover.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 62d3aaa68dbff..de607f09c47bf 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -951,6 +951,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise searchSource .setField('index', $scope.indexPattern) .setField('fields', $scope.state.columns) + .setField('source', true) .setField('size', $scope.opts.sampleSize) .setField( 'sort', From 0c7bf3a54ac44bb2bc01e0c967cba1889b54cfd0 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 19 Aug 2020 17:54:18 +0200 Subject: [PATCH 04/11] Submit request once columns changed --- src/plugins/discover/public/application/angular/discover.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index de607f09c47bf..78bf908d8fbdc 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -256,7 +256,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.state = { ...newState }; // detect changes that should trigger fetching of new data - const changes = ['interval', 'sort'].filter( + const changes = ['interval', 'sort', 'columns'].filter( (prop) => !_.isEqual(newStatePartial[prop], oldStatePartial[prop]) ); From 4650db8a0460ee9976b476b76d9c5ffc1a65e8e7 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 19 Aug 2020 17:55:01 +0200 Subject: [PATCH 05/11] Request all fields in doc view --- .../public/application/components/doc/use_es_doc_search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts b/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts index 8a20237b65708..619c6ecb0f88f 100644 --- a/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts +++ b/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts @@ -45,7 +45,7 @@ export function buildSearchBody(id: string, indexPattern: IndexPattern): Record< stored_fields: computedFields.storedFields, _source: true, script_fields: computedFields.scriptFields, - fields: computedFields.docvalueFields, + fields: ['*'], }; } From 3d5e8b75f556cf956f03ca02738948e3ddc49e09 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 21 Aug 2020 16:52:49 +0200 Subject: [PATCH 06/11] Force rerendering of table row --- .../angular/doc_table/components/table_row.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts b/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts index e7fafde2e68d0..ab8c5cc059aed 100644 --- a/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts +++ b/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts @@ -103,9 +103,12 @@ export function createTableRowDirective($compile: ng.ICompileService, $httpParam $compile($detailsTr)($detailsScope); }; - $scope.$watchMulti(['indexPattern.timeFieldName', 'row.highlight', '[]columns'], () => { - createSummaryRow($scope.row); - }); + $scope.$watchMulti( + ['indexPattern.timeFieldName', 'row.highlight', '[]columns', 'row'], + () => { + createSummaryRow($scope.row); + } + ); $scope.inlineFilter = function inlineFilter($event: any, type: string) { const column = $($event.target).data().column; @@ -174,6 +177,7 @@ export function createTableRowDirective($compile: ng.ICompileService, $httpParam let $cells = $el.children(); newHtmls.forEach(function (html, i) { const $cell = $cells.eq(i); + if ($cell.data('discover:html') === html) return; const reuse = find($cells.slice(i + 1), function (cell: any) { From f84b36da49d699936daeb0c2aca223b29d0ba88e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 9 Sep 2020 11:31:51 +0200 Subject: [PATCH 07/11] Undo use_es_doc_search.ts changes --- .../public/application/components/doc/use_es_doc_search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts b/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts index 619c6ecb0f88f..522ebad1691a9 100644 --- a/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts +++ b/src/plugins/discover/public/application/components/doc/use_es_doc_search.ts @@ -45,7 +45,7 @@ export function buildSearchBody(id: string, indexPattern: IndexPattern): Record< stored_fields: computedFields.storedFields, _source: true, script_fields: computedFields.scriptFields, - fields: ['*'], + docvalue_fields: computedFields.docvalueFields, }; } From 32c01843e4d60facefed207483623a5bd23e4589 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 9 Sep 2020 11:45:40 +0200 Subject: [PATCH 08/11] Merge selected fields with doc_value fields --- src/plugins/discover/public/application/angular/discover.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 78bf908d8fbdc..ff19935cb5532 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -948,9 +948,11 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.updateDataSource = () => { const { indexPattern, searchSource } = $scope; + const indexPatternFields = indexPattern.getComputedFields(); + const fields = [...$scope.state.columns, ...indexPatternFields.docvalueFields]; searchSource .setField('index', $scope.indexPattern) - .setField('fields', $scope.state.columns) + .setField('fields', fields) .setField('source', true) .setField('size', $scope.opts.sampleSize) .setField( From 9ed5a9509118aadf95ffa3c9f045684e5f672e8e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 9 Sep 2020 14:50:07 +0200 Subject: [PATCH 09/11] Fix scripted fields tests --- .../discover/public/application/angular/discover.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index ff19935cb5532..584d85cfa3d3f 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -949,7 +949,14 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.updateDataSource = () => { const { indexPattern, searchSource } = $scope; const indexPatternFields = indexPattern.getComputedFields(); - const fields = [...$scope.state.columns, ...indexPatternFields.docvalueFields]; + const columns = $scope.state.columns.filter((name) => { + return !indexPatternFields.docvalueFields.find((docVal) => docVal.field === name); + }); + const fields = [ + ...columns, + ...indexPatternFields.docvalueFields, + ...Object.keys(indexPatternFields.scriptFields), + ]; searchSource .setField('index', $scope.indexPattern) .setField('fields', fields) From 9c5160f5219222ba7436f3d36bef40554a186051 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 10 Sep 2020 08:31:43 +0200 Subject: [PATCH 10/11] Extend SearchSource with fieldsApi handling --- ...plugin-plugins-data-public.querystringinput.md | 2 +- ...ugin-plugins-data-public.searchsourcefields.md | 1 + src/plugins/data/public/public.api.md | 2 ++ .../public/search/search_source/search_source.ts | 4 +++- .../data/public/search/search_source/types.ts | 1 + .../public/application/angular/discover.js | 15 +++++---------- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md index 9f3ed8c1263ba..2693e448565fe 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md @@ -7,5 +7,5 @@ Signature: ```typescript -QueryStringInput: React.FC> +React.FC> ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md index 7a64af0f8b2b8..05d5a23cd0c1f 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md @@ -16,6 +16,7 @@ export interface SearchSourceFields | --- | --- | --- | | [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | any | | | [fields](./kibana-plugin-plugins-data-public.searchsourcefields.fields.md) | NameList | | +| [fieldsApi](./kibana-plugin-plugins-data-public.searchsourcefields.fieldsapi.md) | NameList | | | [filter](./kibana-plugin-plugins-data-public.searchsourcefields.filter.md) | Filter[] | Filter | (() => Filter[] | Filter | undefined) | | | [from](./kibana-plugin-plugins-data-public.searchsourcefields.from.md) | number | | | [highlight](./kibana-plugin-plugins-data-public.searchsourcefields.highlight.md) | any | | diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index c2cc2fdc3c134..8c10ce2967abd 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1771,6 +1771,8 @@ export interface SearchSourceFields { // (undocumented) fields?: NameList; // (undocumented) + fieldsApi?: NameList; + // (undocumented) filter?: Filter[] | Filter | (() => Filter[] | Filter | undefined); // (undocumented) from?: number; diff --git a/src/plugins/data/public/search/search_source/search_source.ts b/src/plugins/data/public/search/search_source/search_source.ts index add45e206c36d..f56c3d0b79bd6 100644 --- a/src/plugins/data/public/search/search_source/search_source.ts +++ b/src/plugins/data/public/search/search_source/search_source.ts @@ -381,6 +381,8 @@ export class SearchSource { case 'fields': const fields = uniq((data[key] || []).concat(val)); return addToRoot(key, fields); + case 'fieldsApi': + return key && data[key] == null && addToBody('fields', val); case 'index': case 'type': case 'highlightAll': @@ -465,7 +467,7 @@ export class SearchSource { // request the remaining fields from both stored_fields and _source const remainingFields = difference(fields, keys(body.script_fields)); - body.fields = remainingFields; + body.stored_fields = remainingFields; setWith(body, '_source.includes', remainingFields, (nsValue) => isObject(nsValue) ? {} : nsValue ); diff --git a/src/plugins/data/public/search/search_source/types.ts b/src/plugins/data/public/search/search_source/types.ts index c2f8701a64fa3..9c02cfbe6cb7c 100644 --- a/src/plugins/data/public/search/search_source/types.ts +++ b/src/plugins/data/public/search/search_source/types.ts @@ -47,6 +47,7 @@ export interface SearchSourceFields { source?: NameList; version?: boolean; fields?: NameList; + fieldsApi?: NameList; index?: IndexPattern; searchAfter?: EsQuerySearchAfter; timeout?: string; diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 584d85cfa3d3f..7994398a11e92 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -948,18 +948,13 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.updateDataSource = () => { const { indexPattern, searchSource } = $scope; - const indexPatternFields = indexPattern.getComputedFields(); - const columns = $scope.state.columns.filter((name) => { - return !indexPatternFields.docvalueFields.find((docVal) => docVal.field === name); - }); - const fields = [ - ...columns, - ...indexPatternFields.docvalueFields, - ...Object.keys(indexPatternFields.scriptFields), - ]; + const { docvalueFields } = indexPattern.getComputedFields(); + const columns = $scope.state.columns.filter( + (name) => !docvalueFields.find((docVal) => docVal.field === name) + ); searchSource .setField('index', $scope.indexPattern) - .setField('fields', fields) + .setField('fieldsApi', columns) .setField('source', true) .setField('size', $scope.opts.sampleSize) .setField( From 61b7fb0ad5aaf2b76c07c5dbb10456ed6fcd6d00 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 17 Sep 2020 10:31:29 +0200 Subject: [PATCH 11/11] Fetch all fields, not _source --- .../discover/public/application/angular/discover.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index a9acc39fae606..390e30b53ea2f 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -863,7 +863,9 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise } $scope.hits = resp.hits.total; - $scope.rows = resp.hits.hits; + $scope.rows = resp.hits.hits.map((hit) => { + return { ...hit, ...{ _source: hit.fields } }; + }); // if we haven't counted yet, reset the counts const counts = ($scope.fieldCounts = $scope.fieldCounts || {}); @@ -938,14 +940,10 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.updateDataSource = () => { const { indexPattern, searchSource } = $scope; - const { docvalueFields } = indexPattern.getComputedFields(); - const columns = $scope.state.columns.filter( - (name) => !docvalueFields.find((docVal) => docVal.field === name) - ); searchSource .setField('index', $scope.indexPattern) - .setField('fieldsApi', columns) - .setField('source', true) + .setField('fieldsApi', ['*']) + .setField('source', false) .setField('size', $scope.opts.sampleSize) .setField( 'sort',