diff --git a/packages/kbn-data-service/src/search/tabify/tabify_docs.ts b/packages/kbn-data-service/src/search/tabify/tabify_docs.ts index 2a0aa91fe01fc..1d204f1ae6677 100644 --- a/packages/kbn-data-service/src/search/tabify/tabify_docs.ts +++ b/packages/kbn-data-service/src/search/tabify/tabify_docs.ts @@ -87,6 +87,58 @@ function flattenAccum( } } +function getValidDataViewMetaFieldNames(dataView?: DataView): string[] { + if (!dataView?.metaFields) { + return []; + } + + const fieldNames: string[] = []; + + for (let i = 0; i < dataView.metaFields.length; i++) { + const fieldName = dataView.metaFields[i]; + const isExcludedMetaField = + EXCLUDED_META_FIELDS.includes(fieldName) || fieldName.charAt(0) !== '_'; + if (!isExcludedMetaField) { + fieldNames.push(fieldName); + } + } + + return fieldNames; +} + +export function getHitFieldNamesSet(hit: Hit, dataView?: DataView): Set { + const fieldsSet = new Set(); + + if (hit.fields) { + for (const key in hit.fields) { + if (hit.fields.hasOwnProperty(key)) { + fieldsSet.add(key); + } + } + } else if (hit._source) { + for (const key in hit._source) { + if (hit._source.hasOwnProperty(key)) { + fieldsSet.add(key); + } + } + } + + if (hit.ignored_field_values) { + for (const key in hit.ignored_field_values) { + if (hit.ignored_field_values.hasOwnProperty(key)) { + fieldsSet.add(key); + } + } + } + + const metaFieldNames = getValidDataViewMetaFieldNames(dataView); + metaFieldNames.forEach((fieldName) => { + fieldsSet.add(fieldName); + }); + + return fieldsSet; +} + /** * Flattens an individual hit (from an ES response) into an object. This will * create flattened field names, like `user.name`. @@ -131,15 +183,9 @@ export function flattenHit(hit: Hit, indexPattern?: DataView, params?: TabifyDoc } // Merge all valid meta fields into the flattened object - if (indexPattern?.metaFields) { - for (let i = 0; i < indexPattern?.metaFields.length; i++) { - const fieldName = indexPattern?.metaFields[i]; - const isExcludedMetaField = - EXCLUDED_META_FIELDS.includes(fieldName) || fieldName.charAt(0) !== '_'; - if (!isExcludedMetaField) { - flat[fieldName] = hit[fieldName as keyof estypes.SearchHit]; - } - } + const metaFieldNames = getValidDataViewMetaFieldNames(indexPattern); + for (const fieldName of metaFieldNames) { + flat[fieldName] = hit[fieldName as keyof estypes.SearchHit]; } // Use a proxy to make sure that keys are always returned in a specific order, diff --git a/packages/kbn-discover-utils/src/data_types/data_table_record.ts b/packages/kbn-discover-utils/src/data_types/data_table_record.ts new file mode 100644 index 0000000000000..4a9bbcb19a2a0 --- /dev/null +++ b/packages/kbn-discover-utils/src/data_types/data_table_record.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DataView } from '@kbn/data-views-plugin/common'; +import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { flattenHit } from '@kbn/data-service'; +import { getDocId } from '../utils/get_doc_id'; + +export interface EsHitRecord extends Omit { + _source?: Record; +} + +/** + * This is the record/row of data provided to our Data Table + */ +export interface DataTableRecordSpec { + /** + * The document returned by Elasticsearch for search queries + */ + raw: EsHitRecord; + /** + * Determines that the given doc is the anchor doc when rendering view surrounding docs + */ + isAnchor?: boolean; + /* + * Data view + */ + dataView?: DataView; + + // TODO: remove + lazy?: boolean; +} + +// let created = 0; +// let flattened = 0; + +export class DataTableRecord { + /** + * A unique id generated by index, id and routing of a record + */ + readonly id: string; + /** + * The document returned by Elasticsearch for search queries + */ + readonly raw: EsHitRecord; + /** + * Determines that the given doc is the anchor doc when rendering view surrounding docs + */ + readonly isAnchor?: boolean; + + #flattened: Record | undefined; + readonly #dataView: DataView | undefined; + + constructor(spec: DataTableRecordSpec) { + this.id = getDocId(spec.raw); + this.raw = spec.raw; + this.isAnchor = spec.isAnchor; + this.#dataView = spec.dataView; + + // console.log('created', ++created); + + if (spec.lazy) { + return; + } + + this.#flattened = flattenHit(this.raw, this.#dataView, { includeIgnoredValues: true }); + // console.log('flattened', ++flattened); + } + + public get flattened() { + if (!this.#flattened) { + this.#flattened = flattenHit(this.raw, this.#dataView, { includeIgnoredValues: true }); + // console.log('lazy flattened', ++flattened); + } + return this.#flattened; + } +} diff --git a/packages/kbn-discover-utils/src/types.ts b/packages/kbn-discover-utils/src/types.ts index 05d9a79b0bbbd..2cd96da32707d 100644 --- a/packages/kbn-discover-utils/src/types.ts +++ b/packages/kbn-discover-utils/src/types.ts @@ -6,36 +6,13 @@ * Side Public License, v 1. */ -import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - +export type { + EsHitRecord, + DataTableRecord, + DataTableRecordSpec, +} from './data_types/data_table_record'; export type { IgnoredReason, ShouldShowFieldInTableHandler } from './utils'; -export interface EsHitRecord extends Omit { - _source?: Record; -} - -/** - * This is the record/row of data provided to our Data Table - */ -export interface DataTableRecord { - /** - * A unique id generated by index, id and routing of a record - */ - id: string; - /** - * The document returned by Elasticsearch for search queries - */ - raw: EsHitRecord; - /** - * A flattened version of the ES doc or data provided by SQL, aggregations ... - */ - flattened: Record; - /** - * Determines that the given doc is the anchor doc when rendering view surrounding docs - */ - isAnchor?: boolean; -} - type FormattedHitPair = readonly [ fieldDisplayName: string, formattedValue: string, diff --git a/packages/kbn-discover-utils/src/utils/build_data_record.ts b/packages/kbn-discover-utils/src/utils/build_data_record.ts index 43adf7b9c8b66..5130f2c616dac 100644 --- a/packages/kbn-discover-utils/src/utils/build_data_record.ts +++ b/packages/kbn-discover-utils/src/utils/build_data_record.ts @@ -7,9 +7,8 @@ */ import type { DataView } from '@kbn/data-views-plugin/common'; -import { flattenHit } from '@kbn/data-service'; -import type { DataTableRecord, EsHitRecord } from '../types'; -import { getDocId } from './get_doc_id'; +import type { EsHitRecord } from '../types'; +import { DataTableRecord } from '../data_types/data_table_record'; /** * Build a record for data table, explorer + classic one @@ -22,12 +21,7 @@ export function buildDataTableRecord( dataView?: DataView, isAnchor?: boolean ): DataTableRecord { - return { - id: getDocId(doc), - raw: doc, - flattened: flattenHit(doc, dataView, { includeIgnoredValues: true }), - isAnchor, - }; + return new DataTableRecord({ raw: doc, isAnchor, dataView, lazy: true }); } /** @@ -39,5 +33,8 @@ export function buildDataTableRecordList( docs: EsHitRecord[], dataView?: DataView ): DataTableRecord[] { - return docs.map((doc) => buildDataTableRecord(doc, dataView)); + // const startTime = window.performance.now(); + const result = docs.map((doc) => buildDataTableRecord(doc, dataView)); + // console.log('duration', window.performance.now() - startTime); + return result; } diff --git a/src/plugins/discover/public/application/main/data_fetching/fetch_esql.ts b/src/plugins/discover/public/application/main/data_fetching/fetch_esql.ts index 3aba795d26920..7851ca0b9998d 100644 --- a/src/plugins/discover/public/application/main/data_fetching/fetch_esql.ts +++ b/src/plugins/discover/public/application/main/data_fetching/fetch_esql.ts @@ -74,7 +74,7 @@ export function fetchEsql( id: String(idx), raw: row, flattened: row, - } as unknown as DataTableRecord; + } as unknown as DataTableRecord; // TODO: refactor too }); } }); diff --git a/src/plugins/discover/public/application/main/utils/calc_field_counts.ts b/src/plugins/discover/public/application/main/utils/calc_field_counts.ts index 8f4fe80762795..bb900fa2427c2 100644 --- a/src/plugins/discover/public/application/main/utils/calc_field_counts.ts +++ b/src/plugins/discover/public/application/main/utils/calc_field_counts.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ import type { DataTableRecord } from '@kbn/discover-utils/types'; +import { getHitFieldNamesSet } from '@kbn/data-service/src/search/tabify/tabify_docs'; /** * This function is calculating stats of the available fields, for usage in sidebar and sharing @@ -18,8 +19,8 @@ export function calcFieldCounts(rows?: DataTableRecord[]) { } rows.forEach((hit) => { - const fields = Object.keys(hit.flattened); - fields.forEach((fieldName) => { + const fieldNamesSet = getHitFieldNamesSet(hit.raw); + fieldNamesSet.forEach((fieldName) => { counts[fieldName] = (counts[fieldName] || 0) + 1; }); });