diff --git a/src/core/public/mocks.ts b/src/core/public/mocks.ts index 163e34b4cf1a..e863d627c801 100644 --- a/src/core/public/mocks.ts +++ b/src/core/public/mocks.ts @@ -83,6 +83,7 @@ function createCoreSetupMock({ uiSettings: uiSettingsServiceMock.createSetupContract(), injectedMetadata: { getInjectedVar: injectedMetadataServiceMock.createSetupContract().getInjectedVar, + getBranding: injectedMetadataServiceMock.createSetupContract().getBranding, }, }; diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_bucket.scss b/src/plugins/vis_builder/public/application/components/data_tab/field_bucket.scss new file mode 100644 index 000000000000..50951d850a62 --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_bucket.scss @@ -0,0 +1,4 @@ +.vbFieldDetails__barContainer { + // Constrains value to the flex item, and allows for truncation when necessary + min-width: 0; +} diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_bucket.tsx b/src/plugins/vis_builder/public/application/components/data_tab/field_bucket.tsx new file mode 100644 index 000000000000..603c43eb1ac9 --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_bucket.tsx @@ -0,0 +1,114 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { + EuiText, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiProgress, +} from '@elastic/eui'; +import { i18n } from '@osd/i18n'; + +import { IndexPatternField } from '../../../../../data/public'; + +import { Bucket } from './types'; +import './field_bucket.scss'; + +interface Props { + bucket: Bucket; + field: IndexPatternField; + onAddFilter: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; +} + +export function VisBuilderFieldBucket({ bucket, field, onAddFilter }: Props) { + const { count, display, percent, value } = bucket; + const { filterable: isFilterableField, name: fieldName } = field; + + const emptyTxt = i18n.translate('visBuilder.fieldChooser.detailViews.emptyStringText', { + // We need this to communicate to users when a top value is actually an empty string + defaultMessage: 'Empty string', + }); + const addLabel = i18n.translate( + 'visBuilder.fieldChooser.detailViews.filterValueButtonAriaLabel', + { + defaultMessage: 'Filter for {fieldName}: "{value}"', + values: { fieldName, value }, + } + ); + const removeLabel = i18n.translate( + 'visBuilder.fieldChooser.detailViews.filterOutValueButtonAriaLabel', + { + defaultMessage: 'Filter out {fieldName}: "{value}"', + values: { fieldName, value }, + } + ); + + const displayValue = display || emptyTxt; + + return ( + <> + + + + + + {displayValue} + + + + + {percent.toFixed(1)}% + + + + + + {/* TODO: Should we have any explanation for non-filterable fields? */} + {isFilterableField && ( + +
+ onAddFilter(field, value, '+')} + aria-label={addLabel} + data-test-subj={`plus-${fieldName}-${value}`} + /> + onAddFilter(field, value, '-')} + aria-label={removeLabel} + data-test-subj={`minus-${fieldName}-${value}`} + /> +
+ + )} + + + + ); +} + +export function StringFieldProgressBar({ + value, + percent, + count, +}: Pick) { + const ariaLabel = `${value}: ${count} (${percent}%)`; + + return ( + + ); +} diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_details.test.tsx b/src/plugins/vis_builder/public/application/components/data_tab/field_details.test.tsx new file mode 100644 index 000000000000..03002bb415dd --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_details.test.tsx @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +// @ts-ignore +import { findTestSubject } from '@elastic/eui/lib/test'; +// @ts-ignore +import stubbedLogstashFields from 'fixtures/logstash_fields'; +// @ts-ignore +import { mountWithIntl, nextTick } from 'test_utils/enzyme_helpers'; + +import { IndexPatternField } from '../../../../../data/public'; + +import { VisBuilderFieldDetails } from './field_details'; + +const mockOnAddFilter = jest.fn(); + +describe('visBuilder sidebar field details', function () { + const defaultProps = { + isMetaField: false, + details: { buckets: [], error: '', exists: 1, total: 1, columns: [] }, + onAddFilter: mockOnAddFilter, + }; + + function mountComponent(field: IndexPatternField, props?: Record) { + const compProps = { ...defaultProps, ...props, field }; + return mountWithIntl(); + } + + it('should render buckets if they exist', async function () { + const field = new IndexPatternField( + { + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'bytes' + ); + const buckets = [1, 2, 3].map((n) => ({ + display: `display-${n}`, + value: `value-${n}`, + percent: 25, + count: 100, + })); + const comp = mountComponent(field, { + details: { ...defaultProps.details, buckets }, + }); + expect(findTestSubject(comp, 'fieldDetailsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsError').length).toBe(0); + expect(findTestSubject(comp, 'fieldDetailsBucketsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsBucketsContainer').children().length).toBe( + buckets.length + ); + expect(findTestSubject(comp, 'fieldDetailsExistsLink').length).toBe(1); + }); + + it('should only render buckets if they exist', async function () { + const field = new IndexPatternField( + { + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'bytes' + ); + const comp = mountComponent(field); + expect(findTestSubject(comp, 'fieldDetailsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsError').length).toBe(0); + expect(findTestSubject(comp, 'fieldDetailsBucketsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsBucketsContainer').children().length).toBe(0); + expect(findTestSubject(comp, 'fieldDetailsExistsLink').length).toBe(1); + }); + + it('should render a details error', async function () { + const field = new IndexPatternField( + { + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'bytes' + ); + const errText = 'Some error'; + const comp = mountComponent(field, { + details: { ...defaultProps.details, error: errText }, + }); + expect(findTestSubject(comp, 'fieldDetailsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsBucketsContainer').children().length).toBe(0); + expect(findTestSubject(comp, 'fieldDetailsError').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsError').text()).toBe(errText); + expect(findTestSubject(comp, 'fieldDetailsExistsLink').length).toBe(0); + }); + + it('should not render an exists filter link for scripted fields', async function () { + const field = new IndexPatternField( + { + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: true, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'bytes' + ); + const comp = mountComponent(field); + expect(findTestSubject(comp, 'fieldDetailsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsError').length).toBe(0); + expect(findTestSubject(comp, 'fieldDetailsExistsLink').length).toBe(0); + }); + + it('should not render an exists filter link for meta fields', async function () { + const field = new IndexPatternField( + { + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: true, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, + 'bytes' + ); + const comp = mountComponent(field, { + ...defaultProps, + isMetaField: true, + }); + expect(findTestSubject(comp, 'fieldDetailsContainer').length).toBe(1); + expect(findTestSubject(comp, 'fieldDetailsError').length).toBe(0); + expect(findTestSubject(comp, 'fieldDetailsExistsLink').length).toBe(0); + }); +}); diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_details.tsx b/src/plugins/vis_builder/public/application/components/data_tab/field_details.tsx new file mode 100644 index 000000000000..ecbe7a770f68 --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_details.tsx @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { EuiLink, EuiPopoverFooter, EuiPopoverTitle, EuiText } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; + +import { IndexPatternField } from '../../../../../data/public'; + +import { VisBuilderFieldBucket } from './field_bucket'; +import { Bucket, FieldDetails } from './types'; + +interface FieldDetailsProps { + field: IndexPatternField; + isMetaField: boolean; + details: FieldDetails; + onAddFilter: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; +} + +export function VisBuilderFieldDetails({ + field, + isMetaField, + details, + onAddFilter, +}: FieldDetailsProps) { + const { buckets, error, exists, total } = details; + + const bucketsTitle = + buckets.length > 1 + ? i18n.translate('visBuilder.fieldChooser.detailViews.fieldTopValuesLabel', { + defaultMessage: 'Top {n} values', + values: { n: buckets.length }, + }) + : i18n.translate('visBuilder.fieldChooser.detailViews.fieldTopValueLabel', { + defaultMessage: 'Top value', + }); + const errorTitle = i18n.translate('visBuilder.fieldChooser.detailViews.fieldNoValuesLabel', { + defaultMessage: 'No values found', + }); + const existsIn = i18n.translate('visBuilder.fieldChooser.detailViews.fieldExistsIn', { + defaultMessage: 'Exists in {exists}', + values: { exists }, + }); + const totalRecords = i18n.translate('visBuilder.fieldChooser.detailViews.fieldTotalRecords', { + defaultMessage: '/ {total} records', + values: { total }, + }); + + const title = buckets.length ? bucketsTitle : errorTitle; + + const shouldAllowExistsFilter = !isMetaField && !field.scripted; + + return ( + <> + {title} +
+ {error ? ( + + {error} + + ) : ( +
+ {buckets.map((bucket: Bucket, idx: number) => ( + + ))} +
+ )} +
+ {!error && ( + + + {shouldAllowExistsFilter ? ( + onAddFilter('_exists_', field.name, '+')} + data-test-subj="fieldDetailsExistsLink" + > + {existsIn} + + ) : ( + <>{exists} + )}{' '} + {totalRecords} + + + )} + + ); +} diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_selector.tsx b/src/plugins/vis_builder/public/application/components/data_tab/field_selector.tsx index 6d3831363c1b..366ec3c829a1 100644 --- a/src/plugins/vis_builder/public/application/components/data_tab/field_selector.tsx +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_selector.tsx @@ -3,21 +3,29 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useState, useEffect, useMemo } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useLayoutEffect } from 'react'; import { EuiFlexItem, EuiAccordion, EuiNotificationBadge, EuiTitle } from '@elastic/eui'; -import { FieldSearch } from './field_search'; import { + FilterManager, + IndexPattern, IndexPatternField, OPENSEARCH_FIELD_TYPES, OSD_FIELD_TYPES, + SortDirection, } from '../../../../../data/public'; -import { FieldSelectorField } from './field_selector_field'; +import { IExpressionLoaderParams } from '../../../../../expressions/public'; +import { useOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public'; -import './field_selector.scss'; +import { VisBuilderServices } from '../../../types'; +import { COUNT_FIELD } from '../../utils/drag_drop'; import { useTypedSelector } from '../../utils/state_management'; import { useIndexPatterns } from '../../utils/use'; -import { getAvailableFields } from './utils'; +import { FieldSearch } from './field_search'; +import { FieldSelectorField, SelectorFieldButton } from './field_selector_field'; +import { FieldDetails } from './types'; +import { getAvailableFields, getDetails } from './utils'; +import './field_selector.scss'; interface IFieldCategories { categorical: IndexPatternField[]; @@ -33,14 +41,34 @@ const META_FIELDS: string[] = [ ]; export const FieldSelector = () => { + const { + services: { + data: { + query: { + filterManager, + queryString, + state$, + timefilter: { timefilter }, + }, + search: { searchSource }, + }, + uiSettings: config, + }, + } = useOpenSearchDashboards(); const indexPattern = useIndexPatterns().selected; const fieldSearchValue = useTypedSelector((state) => state.visualization.searchField); const [filteredFields, setFilteredFields] = useState([]); + const [hits, setHits] = useState>>([]); + const [searchContext, setSearchContext] = useState({ + query: queryString.getQuery(), + filters: filterManager.getFilters(), + }); useEffect(() => { - const indexFields = indexPattern?.fields ?? []; + const indexFields = indexPattern?.fields.getAll() ?? []; const filteredSubset = getAvailableFields(indexFields).filter((field) => - field.displayName.includes(fieldSearchValue) + // case-insensitive field search + field.displayName.toLowerCase().includes(fieldSearchValue.toLowerCase()) ); setFilteredFields(filteredSubset); @@ -65,6 +93,63 @@ export const FieldSelector = () => { [filteredFields] ); + useEffect(() => { + async function getData() { + if (indexPattern && searchContext) { + const newSearchSource = await searchSource.create(); + const timeRangeFilter = timefilter.createFilter(indexPattern); + + newSearchSource + .setField('index', indexPattern) + .setField('size', config.get('discover:sampleSize') ?? 500) + .setField('sort', [{ [indexPattern.timeFieldName || '_score']: 'desc' as SortDirection }]) + .setField('filter', [ + ...(searchContext.filters ?? []), + ...(timeRangeFilter ? [timeRangeFilter] : []), + ]); + + if (searchContext.query) { + const contextQuery = + searchContext.query instanceof Array ? searchContext.query[0] : searchContext.query; + + newSearchSource.setField('query', contextQuery); + } + + const searchResponse = await newSearchSource.fetch(); + + setHits(searchResponse.hits.hits); + } + } + + getData(); + }, [config, searchContext, searchSource, indexPattern, timefilter]); + + useLayoutEffect(() => { + const subscription = state$.subscribe(({ state }) => { + setSearchContext({ + query: state.query, + filters: state.filters, + }); + }); + + return () => { + subscription.unsubscribe(); + }; + }, [state$]); + + const getDetailsByField = useCallback( + (ipField: IndexPatternField) => { + return getDetails(ipField, hits, indexPattern); + }, + [hits, indexPattern] + ); + + const commonFieldGroupProps = { + filterManager, + indexPattern, + getDetails: getDetailsByField, + }; + return (
@@ -74,20 +159,28 @@ export const FieldSelector = () => {
{/* Count Field */} - + + - -
); @@ -95,37 +188,42 @@ export const FieldSelector = () => { interface FieldGroupProps { fields?: IndexPatternField[]; + filterManager: FilterManager; + getDetails: (ipField: IndexPatternField) => FieldDetails; header: string; id: string; + indexPattern?: IndexPattern; } -const FieldGroup = ({ fields, header, id }: FieldGroupProps) => ( - - {header} - - } - extraAction={ - - {fields?.length || 0} - - } - initialIsOpen - > - {fields?.map((field, i) => ( - - - - ))} - -); - -function getFieldCategory(field: IndexPatternField): keyof IFieldCategories { - if (META_FIELDS.includes(field.name)) return 'meta'; - if (field.type === OSD_FIELD_TYPES.NUMBER) return 'numerical'; +const FieldGroup = ({ fields, header, id, ...rest }: FieldGroupProps) => { + return ( + + {header} + + } + extraAction={ + + {fields?.length || 0} + + } + initialIsOpen + > + {fields?.map((field, i) => ( + + + + ))} + + ); +}; + +function getFieldCategory({ name, type }: IndexPatternField): keyof IFieldCategories { + if (META_FIELDS.includes(name)) return 'meta'; + if (type === OSD_FIELD_TYPES.NUMBER) return 'numerical'; return 'categorical'; } diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.scss b/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.scss index c129f7a997a1..ff522d5b8a44 100644 --- a/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.scss +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.scss @@ -26,3 +26,8 @@ height: 100%; } } + +.vbItem__fieldPopoverPanel { + min-width: 260px; + max-width: 300px; +} diff --git a/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.tsx b/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.tsx index a87e2d184eed..be63a7c12cf9 100644 --- a/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.tsx +++ b/src/plugins/vis_builder/public/application/components/data_tab/field_selector_field.tsx @@ -28,32 +28,97 @@ * under the License. */ -import React, { useState } from 'react'; -import { IndexPatternField } from '../../../../../data/public'; -import { FieldButton, FieldIcon } from '../../../../../opensearch_dashboards_react/public'; -import { useDrag } from '../../utils/drag_drop/drag_drop_context'; -import { COUNT_FIELD } from '../../utils/drag_drop/types'; +import React, { useCallback, useState } from 'react'; +import { EuiPopover } from '@elastic/eui'; +import { + FilterManager, + IndexPattern, + IndexPatternField, + opensearchFilters, +} from '../../../../../data/public'; +import { + FieldButton, + FieldButtonProps, + FieldIcon, +} from '../../../../../opensearch_dashboards_react/public'; + +import { COUNT_FIELD, useDrag } from '../../utils/drag_drop'; +import { VisBuilderFieldDetails } from './field_details'; +import { FieldDetails } from './types'; import './field_selector_field.scss'; export interface FieldSelectorFieldProps { - field: Partial & Pick; + field: IndexPatternField; + filterManager: FilterManager; + indexPattern?: IndexPattern; + getDetails: (field) => FieldDetails; } -// TODO: -// 1. Add field sections (Available fields, popular fields from src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx) -// 2. Add popover for fields stats from discover as well -export const FieldSelectorField = ({ field }: FieldSelectorFieldProps) => { +// TODO: Add field sections (Available fields, popular fields from src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx) +export const FieldSelectorField = ({ + field, + filterManager, + indexPattern, + getDetails, +}: FieldSelectorFieldProps) => { + const { id: indexPatternId = '', metaFields = [] } = indexPattern ?? {}; + const isMetaField = metaFields.includes(field.name); const [infoIsOpen, setOpen] = useState(false); - const [dragProps] = useDrag({ - namespace: 'field-data', - value: field.name || COUNT_FIELD, - }); + + const onAddFilter = useCallback( + (fieldToFilter, value, operation) => { + const newFilters = opensearchFilters.generateFilters( + filterManager, + fieldToFilter, + value, + operation, + indexPatternId + ); + return filterManager.addFilters(newFilters); + }, + [filterManager, indexPatternId] + ); function togglePopover() { setOpen(!infoIsOpen); } + return ( + } + isOpen={infoIsOpen} + closePopover={() => setOpen(false)} + anchorPosition="rightUp" + panelClassName="vbItem__fieldPopoverPanel" + repositionOnScroll + > + {infoIsOpen && ( + + )} + + ); +}; + +export interface SelectorFieldButtonProps extends Partial { + dragValue?: IndexPatternField['name'] | null | typeof COUNT_FIELD; + field: Partial & Pick; +} + +export const SelectorFieldButton = ({ dragValue, field, ...rest }: SelectorFieldButtonProps) => { + const { name, displayName, type, scripted = false } = field ?? {}; + const [dragProps] = useDrag({ + namespace: 'field-data', + value: dragValue ?? name, + }); + function wrapOnDot(str?: string) { // u200B is a non-width white-space character, which allows // the browser to efficiently word-wrap right after the dot @@ -61,26 +126,21 @@ export const FieldSelectorField = ({ field }: FieldSelectorFieldProps) => { return str ? str.replace(/\./g, '.\u200B') : ''; } - const fieldName = ( - - {wrapOnDot(field.displayName)} + const defaultIcon = ; + + const defaultFieldName = ( + + {wrapOnDot(displayName)} ); - return ( - } - // fieldAction={actionButton} - fieldName={fieldName} - {...dragProps} - /> - ); + const defaultProps = { + className: 'vbFieldSelectorField', + dataTestSubj: `field-${name}-showDetails`, + fieldIcon: defaultIcon, + fieldName: defaultFieldName, + onClick: () => {}, + }; + + return ; }; diff --git a/src/plugins/vis_builder/public/application/components/data_tab/types.ts b/src/plugins/vis_builder/public/application/components/data_tab/types.ts new file mode 100644 index 000000000000..c7e0327070e7 --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface FieldDetails { + buckets: Bucket[]; + error: string; + exists: number; + total: number; +} + +export interface FieldValueCounts extends Partial { + missing?: number; +} + +export interface Bucket { + count: number; + display: string; + percent: number; + value: string; +} diff --git a/src/plugins/vis_builder/public/application/components/data_tab/utils/field_calculator.test.ts b/src/plugins/vis_builder/public/application/components/data_tab/utils/field_calculator.test.ts new file mode 100644 index 000000000000..4f1dfd98fc3b --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/utils/field_calculator.test.ts @@ -0,0 +1,268 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import _ from 'lodash'; +// @ts-ignore +import realHits from 'fixtures/real_hits.js'; +// @ts-ignore +import stubbedLogstashFields from 'fixtures/logstash_fields'; +import { coreMock } from '../../../../../../../core/public/mocks'; +import { IndexPattern, IndexPatternField } from '../../../../../../data/public'; +import { getStubIndexPattern } from '../../../../../../data/public/test_utils'; +import { Bucket } from '../types'; +import { + groupValues, + getFieldValues, + getFieldValueCounts, + FieldValueCountsParams, +} from './field_calculator'; + +let indexPattern: IndexPattern; + +describe('field_calculator', function () { + beforeEach(function () { + indexPattern = getStubIndexPattern( + 'logstash-*', + (cfg: any) => cfg, + 'time', + stubbedLogstashFields(), + coreMock.createSetup() + ); + }); + + describe('groupValues', function () { + let groups: Record; + let grouped: boolean; + let values: any[]; + beforeEach(function () { + values = [ + ['foo', 'bar'], + 'foo', + 'foo', + undefined, + ['foo', 'bar'], + 'bar', + 'baz', + null, + null, + null, + 'foo', + undefined, + ]; + groups = groupValues(values, grouped); + }); + + it('should return an object', function () { + expect(groups).toBeInstanceOf(Object); + }); + + it('should throw an error if any value is a plain object', function () { + expect(function () { + groupValues([{}, true, false], grouped); + }).toThrowError(); + }); + + it('should handle values with dots in them', function () { + values = ['0', '0.........', '0.......,.....']; + groups = groupValues(values, grouped); + expect(groups[values[0]].count).toBe(1); + expect(groups[values[1]].count).toBe(1); + expect(groups[values[2]].count).toBe(1); + }); + + it('should have a key for value in the array when not grouping array terms', function () { + expect(_.keys(groups).length).toBe(3); + expect(groups.foo).toBeInstanceOf(Object); + expect(groups.bar).toBeInstanceOf(Object); + expect(groups.baz).toBeInstanceOf(Object); + }); + + it('should count array terms independently', function () { + expect(groups['foo,bar']).toBeUndefined(); + expect(groups.foo.count).toBe(5); + expect(groups.bar.count).toBe(3); + expect(groups.baz.count).toBe(1); + }); + + describe('grouped array terms', function () { + beforeEach(function () { + grouped = true; + groups = groupValues(values, grouped); + }); + + it('should group array terms when grouped is true', function () { + expect(_.keys(groups).length).toBe(4); + expect(groups['foo,bar']).toBeInstanceOf(Object); + }); + + it('should contain the original array as the value', function () { + expect(groups['foo,bar'].value).toEqual(['foo', 'bar']); + }); + + it('should count the pairs separately from the values they contain', function () { + expect(groups['foo,bar'].count).toBe(2); + expect(groups.foo.count).toBe(3); + expect(groups.bar.count).toBe(1); + }); + }); + }); + + describe('getFieldValues', function () { + let hits: any; + + beforeEach(function () { + hits = _.each(_.cloneDeep(realHits), (hit) => indexPattern.flattenHit(hit)); + }); + + it('should return an array of values for _source fields', function () { + const extensions = getFieldValues({ + hits, + field: indexPattern.fields.getByName('extension') as IndexPatternField, + indexPattern, + }); + expect(extensions).toBeInstanceOf(Array); + expect( + _.filter(extensions, function (v) { + return v === 'html'; + }).length + ).toBe(8); + expect(_.uniq(_.clone(extensions)).sort()).toEqual(['gif', 'html', 'php', 'png']); + }); + + it('should return an array of values for core meta fields', function () { + const types = getFieldValues({ + hits, + field: indexPattern.fields.getByName('_type') as IndexPatternField, + indexPattern, + }); + expect(types).toBeInstanceOf(Array); + expect( + _.filter(types, function (v) { + return v === 'apache'; + }).length + ).toBe(18); + expect(_.uniq(_.clone(types)).sort()).toEqual(['apache', 'nginx']); + }); + }); + + describe('getFieldValueCounts', function () { + let params: FieldValueCountsParams; + beforeEach(function () { + params = { + hits: _.cloneDeep(realHits), + field: indexPattern.fields.getByName('extension') as IndexPatternField, + count: 3, + indexPattern, + }; + }); + + it('counts the top 5 values by default', function () { + params.hits = params.hits.map((hit: Record, i) => ({ + ...hit, + _source: { + extension: `${hit._source.extension}-${i}`, + }, + })); + params.count = undefined; + const extensions = getFieldValueCounts(params); + expect(extensions).toBeInstanceOf(Object); + expect(extensions.buckets).toBeInstanceOf(Array); + const buckets = extensions.buckets as Bucket[]; + expect(buckets.length).toBe(5); + expect(extensions.error).toBeUndefined(); + }); + + it('counts only distinct values if less than default', function () { + params.count = undefined; + const extensions = getFieldValueCounts(params); + expect(extensions).toBeInstanceOf(Object); + expect(extensions.buckets).toBeInstanceOf(Array); + const buckets = extensions.buckets as Bucket[]; + expect(buckets.length).toBe(4); + expect(extensions.error).toBeUndefined(); + }); + + it('counts only distinct values if less than specified count', function () { + params.count = 10; + const extensions = getFieldValueCounts(params); + expect(extensions).toBeInstanceOf(Object); + expect(extensions.buckets).toBeInstanceOf(Array); + const buckets = extensions.buckets as Bucket[]; + expect(buckets.length).toBe(4); + expect(extensions.error).toBeUndefined(); + }); + + it('counts the top 3 values', function () { + const extensions = getFieldValueCounts(params); + expect(extensions).toBeInstanceOf(Object); + expect(extensions.buckets).toBeInstanceOf(Array); + const buckets = extensions.buckets as Bucket[]; + expect(buckets.length).toBe(3); + expect(_.map(buckets, 'value')).toEqual(['html', 'gif', 'php']); + expect(extensions.error).toBeUndefined(); + }); + + it('fails to analyze geo and attachment types', function () { + params.field = indexPattern.fields.getByName('point') as IndexPatternField; + expect(getFieldValueCounts(params).error).not.toBeUndefined(); + + params.field = indexPattern.fields.getByName('area') as IndexPatternField; + expect(getFieldValueCounts(params).error).not.toBeUndefined(); + + params.field = indexPattern.fields.getByName('request_body') as IndexPatternField; + expect(getFieldValueCounts(params).error).not.toBeUndefined(); + }); + + it('fails to analyze fields that are in the mapping, but not the hits', function () { + params.field = indexPattern.fields.getByName('ip') as IndexPatternField; + expect(getFieldValueCounts(params).error).not.toBeUndefined(); + }); + + it('counts the total hits', function () { + expect(getFieldValueCounts(params).total).toBe(params.hits.length); + }); + + it('counts the hits the field exists in', function () { + params.field = indexPattern.fields.getByName('phpmemory') as IndexPatternField; + expect(getFieldValueCounts(params).exists).toBe(5); + }); + + it('catches and returns errors', function () { + params.hits = params.hits.map((hit: Record) => ({ + ...hit, + _source: { + extension: { foo: hit._source.extension }, + }, + })); + params.grouped = true; + expect(typeof getFieldValueCounts(params).error).toBe('string'); + }); + }); +}); diff --git a/src/plugins/vis_builder/public/application/components/data_tab/utils/field_calculator.ts b/src/plugins/vis_builder/public/application/components/data_tab/utils/field_calculator.ts new file mode 100644 index 000000000000..bd3cde945d95 --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/utils/field_calculator.ts @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { IndexPattern, IndexPatternField } from '../../../../../../data/public'; +import { FieldValueCounts } from '../types'; + +const NO_ANALYSIS_TYPES = ['geo_point', 'geo_shape', 'attachment']; + +interface FieldValuesParams { + hits: Array>; + field: IndexPatternField; + indexPattern: IndexPattern; +} + +interface FieldValueCountsParams extends FieldValuesParams { + count?: number; + grouped?: boolean; +} + +const getFieldValues = ({ hits, field, indexPattern }: FieldValuesParams) => { + // For multi-value fields, we want to flatten based on the parent name instead + const name = field.subType?.multi?.parent ?? field.name; + const flattenHit = indexPattern.flattenHit; + return hits.map((hit) => flattenHit(hit)[name]); +}; + +const getFieldValueCounts = (params: FieldValueCountsParams): FieldValueCounts => { + const { hits, field, indexPattern, count = 5, grouped = false } = params; + const { type: fieldType } = field; + + if (NO_ANALYSIS_TYPES.includes(fieldType)) { + return { + error: i18n.translate( + 'visBuilder.fieldChooser.fieldCalculator.analysisIsNotAvailableForGeoFieldsErrorMessage', + { + defaultMessage: 'Analysis is not available for {fieldType} fields.', + values: { + fieldType, + }, + } + ), + }; + } + + const allValues = getFieldValues({ hits, field, indexPattern }); + const missing = allValues.filter((v) => v === undefined || v === null).length; + + try { + const groups = groupValues(allValues, grouped); + const counts = Object.keys(groups) + .sort((a, b) => groups[b].count - groups[a].count) + .slice(0, count) + .map((key) => ({ + value: groups[key].value, + count: groups[key].count, + percent: (groups[key].count / (hits.length - missing)) * 100, + display: indexPattern.getFormatterForField(field).convert(groups[key].value), + })); + + if (hits.length === missing) { + return { + error: i18n.translate( + 'visBuilder.fieldChooser.fieldCalculator.fieldIsNotPresentInDocumentsErrorMessage', + { + defaultMessage: + 'This field is present in your OpenSearch mapping but not in the {hitsLength} documents sampled. You may still be able to visualize it.', + values: { + hitsLength: hits.length, + }, + } + ), + }; + } + + return { + total: hits.length, + exists: hits.length - missing, + missing, + buckets: counts, + }; + } catch (e) { + return { + error: e instanceof Error ? e.message : String(e), + }; + } +}; + +const groupValues = ( + allValues: any[], + grouped?: boolean +): Record => { + const values = grouped ? allValues : allValues.flat(); + + return values + .filter((v) => { + if (v instanceof Object && !Array.isArray(v)) { + throw new Error( + i18n.translate( + 'visBuilder.fieldChooser.fieldCalculator.analysisIsNotAvailableForObjectFieldsErrorMessage', + { + defaultMessage: 'Analysis is not available for object fields.', + } + ) + ); + } + return v !== undefined && v !== null; + }) + .reduce((groups, value) => { + if (groups.hasOwnProperty(value)) { + groups[value].count++; + } else { + groups[value] = { + value, + count: 1, + }; + } + return groups; + }, {}); +}; + +export { FieldValueCountsParams, groupValues, getFieldValues, getFieldValueCounts }; diff --git a/src/plugins/vis_builder/public/application/components/data_tab/utils/get_details.ts b/src/plugins/vis_builder/public/application/components/data_tab/utils/get_details.ts new file mode 100644 index 000000000000..a2a0d6ce439e --- /dev/null +++ b/src/plugins/vis_builder/public/application/components/data_tab/utils/get_details.ts @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; + +import { IndexPattern, IndexPatternField } from '../../../../../../data/public'; +import { FieldDetails } from '../types'; + +import { getFieldValueCounts } from './field_calculator'; + +export function getDetails( + field: IndexPatternField, + hits: Array>, + indexPattern?: IndexPattern +): FieldDetails { + const defaultDetails = { + error: '', + exists: 0, + total: 0, + buckets: [], + }; + if (!indexPattern) { + return { + ...defaultDetails, + error: i18n.translate('visBuilder.fieldChooser.noIndexPatternSelectedErrorMessage', { + defaultMessage: 'Index pattern not specified.', + }), + }; + } + if (!hits.length) { + return { + ...defaultDetails, + error: i18n.translate('visBuilder.fieldChooser.noHits', { + defaultMessage: + 'No documents match the selected query and filters. Try increasing time range or removing filters.', + }), + }; + } + const details = { + ...defaultDetails, + ...getFieldValueCounts({ + hits, + field, + indexPattern, + count: 5, + grouped: false, + }), + }; + if (details.buckets) { + for (const bucket of details.buckets) { + bucket.display = indexPattern.getFormatterForField(field).convert(bucket.value); + } + } + return details; +} diff --git a/src/plugins/vis_builder/public/application/components/data_tab/utils/index.ts b/src/plugins/vis_builder/public/application/components/data_tab/utils/index.ts index dd0cdea3e23e..45d9c1042295 100644 --- a/src/plugins/vis_builder/public/application/components/data_tab/utils/index.ts +++ b/src/plugins/vis_builder/public/application/components/data_tab/utils/index.ts @@ -4,3 +4,4 @@ */ export { getAvailableFields } from './get_available_fields'; +export { getDetails } from './get_details'; diff --git a/src/plugins/vis_builder/public/application/utils/drag_drop/drag_drop_context.tsx b/src/plugins/vis_builder/public/application/utils/drag_drop/drag_drop_context.tsx index c0f8725a501a..b5c809c9b359 100644 --- a/src/plugins/vis_builder/public/application/utils/drag_drop/drag_drop_context.tsx +++ b/src/plugins/vis_builder/public/application/utils/drag_drop/drag_drop_context.tsx @@ -14,7 +14,7 @@ import React, { } from 'react'; import { DragDataType } from './types'; -// TODO: Replace any with corret type +// TODO: Replace any with correct type // TODO: Split into separate files interface IDragDropContext { data: DragDataType; diff --git a/src/plugins/vis_builder/public/application/utils/drag_drop/index.ts b/src/plugins/vis_builder/public/application/utils/drag_drop/index.ts index 3799a2eb6052..4516a90575ab 100644 --- a/src/plugins/vis_builder/public/application/utils/drag_drop/index.ts +++ b/src/plugins/vis_builder/public/application/utils/drag_drop/index.ts @@ -4,3 +4,4 @@ */ export * from './drag_drop_context'; +export * from './types';