Skip to content

Commit

Permalink
[Serverless Search] Improve awareness on using pipelines (#182177)
Browse files Browse the repository at this point in the history
## Summary

- Adds a section for `Preprocessing data `
  - [x] home page 
  - [x] new connector ->connect to index tab

- Remove `Transform and enrich your data` from home page

### Screen Recording


https://github.com/elastic/kibana/assets/55930906/47820f68-a726-4ad5-83ab-2fabcb899c91


### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
saarikabhasi and kibanamachine authored May 3, 2024
1 parent 31e8c54 commit 63dfc7d
Show file tree
Hide file tree
Showing 24 changed files with 483 additions and 89 deletions.
5 changes: 5 additions & 0 deletions packages/kbn-doc-links/src/get_doc_links.ts
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,11 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D
csvPipelines: `${ELASTIC_WEBSITE_URL}guide/en/ecs/${ECS_VERSION}/ecs-converting.html`,
pipelineFailure: `${ELASTICSEARCH_DOCS}ingest.html#handling-pipeline-failures`,
processors: `${ELASTICSEARCH_DOCS}processors.html`,
arrayOrJson: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-array-json-handling`,
dataEnrichment: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-data-enrichment`,
dataFiltering: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-data-filtering`,
dataTransformation: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-data-transformation`,
pipelineHandling: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-pipeline-handling`,
remove: `${ELASTICSEARCH_DOCS}remove-processor.html`,
rename: `${ELASTICSEARCH_DOCS}rename-processor.html`,
script: `${ELASTICSEARCH_DOCS}script-processor.html`,
Expand Down
245 changes: 245 additions & 0 deletions packages/kbn-search-api-panels/components/preprocess_data.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,245 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import {
EuiFlexGroup,
EuiFlexItem,
EuiSpacer,
EuiText,
EuiThemeProvider,
EuiTitle,
EuiPanel,
EuiLink,
EuiFlexGrid,
EuiImage,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { euiThemeVars } from '@kbn/ui-theme';
import React from 'react';

export const PreprocessDataPanel: React.FC<{
docLinks: {
arrayOrJson: string;
dataEnrichment: string;
dataFiltering: string;
dataTransformation: string;
pipelineHandling: string;
};
images: {
dataEnrichment: string;
dataTransformation: string;
dataFiltering: string;
pipelineHandling: string;
arrayHandling: string;
};
}> = ({ docLinks, images }) => {
const color = euiThemeVars.euiColorVis1_behindText;
return (
<EuiThemeProvider colorMode="dark">
<EuiPanel paddingSize="xl">
<EuiFlexGrid direction="column" columns={2} gutterSize="xl">
<EuiFlexItem>
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
<EuiFlexItem grow={false}>
<EuiImage alt="" src={images.dataEnrichment} color={color} />
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h3>
{i18n.translate(
'searchApiPanels.preprocessData.overview.dataEnrichment.title',
{
defaultMessage: 'Data enrichment',
}
)}
</h3>
</EuiTitle>
<EuiSpacer size="s" />
<EuiText size="s">
<p>
{i18n.translate(
'searchApiPanels.preprocessData.overview.dataEnrichment.description',
{
defaultMessage:
'Add information from external sources or apply transformations to your documents for more contextual, insightful search.',
}
)}
</p>
</EuiText>
<EuiSpacer size="s" />
<EuiText>
<p>
<EuiLink href={docLinks.dataEnrichment} target="_blank">
{i18n.translate(
'searchApiPanels.preprocessData.overview.dataEnrichment.learnMore',
{
defaultMessage: 'Learn more',
}
)}
</EuiLink>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem>
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
<EuiFlexItem grow={false}>
<EuiImage alt="" src={images.dataFiltering} color={color} />
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h3>
{i18n.translate('searchApiPanels.pipeline.overview.dataFiltering.title', {
defaultMessage: 'Data filtering',
})}
</h3>
</EuiTitle>
<EuiSpacer size="s" />
<EuiText size="s">
{i18n.translate('searchApiPanels.pipeline.overview.dataFiltering.description', {
defaultMessage:
'Remove specific fields from documents before indexing, to exclude unnecessary or sensitive information.',
})}
</EuiText>
<EuiSpacer size="s" />
<EuiText>
<p>
<EuiLink href={docLinks.dataFiltering} target="_blank">
{i18n.translate(
'searchApiPanels.preprocessData.overview.dataFiltering.learnMore',
{
defaultMessage: 'Learn more',
}
)}
</EuiLink>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem>
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
<EuiFlexItem grow={false}>
<EuiImage alt="" src={images.arrayHandling} color={color} />
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h3>
{i18n.translate('searchApiPanels.pipeline.overview.arrayJsonHandling.title', {
defaultMessage: 'Array/JSON handling',
})}
</h3>
</EuiTitle>
<EuiSpacer size="s" />
<EuiText size="s">
{i18n.translate(
'searchApiPanels.pipeline.overview.arrayJsonHandling.description',
{
defaultMessage: 'Run batch processors, parse JSON data and sort elements.',
}
)}
</EuiText>
<EuiSpacer size="s" />
<EuiText>
<p>
<EuiLink href={docLinks.arrayOrJson} target="_blank">
{i18n.translate(
'searchApiPanels.preprocessData.overview.arrayJsonHandling.learnMore',
{
defaultMessage: 'Learn more',
}
)}
</EuiLink>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem>
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
<EuiFlexItem grow={false}>
<EuiImage alt="" src={images.dataTransformation} color={color} />
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h3>
{i18n.translate('searchApiPanels.pipeline.overview.dataTransformation.title', {
defaultMessage: 'Data transformation',
})}
</h3>
</EuiTitle>
<EuiSpacer size="s" />
<EuiText size="s">
{i18n.translate(
'searchApiPanels.pipeline.overview.dataTransformation.description',
{
defaultMessage:
'Parse information from your documents to ensure they conform to a standardized format.',
}
)}
</EuiText>
<EuiSpacer size="s" />
<EuiText size="s">
<p>
<EuiLink href={docLinks.dataTransformation} target="_blank">
{i18n.translate(
'searchApiPanels.preprocessData.overview.dataTransformation.learnMore',
{
defaultMessage: 'Learn more',
}
)}
</EuiLink>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem>
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
<EuiFlexItem grow={false}>
<EuiImage alt="" src={images.pipelineHandling} color={color} />
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h3>
{i18n.translate('searchApiPanels.pipeline.overview.pipelineHandling.title', {
defaultMessage: 'Pipeline handling',
})}
</h3>
</EuiTitle>

<EuiText size="s">
{i18n.translate(
'searchApiPanels.pipeline.overview.pipelineHandling.description',
{
defaultMessage:
'Handle error exceptions, execute another pipeline, or reroute documents to another index',
}
)}
</EuiText>
<EuiSpacer size="s" />
<EuiText>
<p>
<EuiLink href={docLinks.pipelineHandling} target="_blank">
{i18n.translate(
'searchApiPanels.preprocessData.overview.pipelineHandling.learnMore',
{
defaultMessage: 'Learn more',
}
)}
</EuiLink>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
</EuiFlexGrid>
</EuiPanel>
</EuiThemeProvider>
);
};
1 change: 1 addition & 0 deletions packages/kbn-search-api-panels/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export * from './components/pipeline_panel';
export * from './components/select_client';
export * from './components/try_in_console_button';
export * from './components/install_client';
export * from './components/preprocess_data';

export * from './types';
export * from './utils';
Expand Down
3 changes: 2 additions & 1 deletion packages/kbn-search-api-panels/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"@kbn/share-plugin",
"@kbn/i18n-react",
"@kbn/security-plugin",
"@kbn/console-plugin"
"@kbn/console-plugin",
"@kbn/ui-theme"
]
}
18 changes: 17 additions & 1 deletion x-pack/plugins/serverless_search/common/doc_links.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ class ESDocLinks {
public gettingStartedSearch: string = '';
public gettingStartedExplore: string = '';

// Ingest processor
public dataEnrichment: string = '';
public dataFiltering: string = '';
public arrayOrJson: string = '';
public dataTransformation: string = '';
public pipelineHandling: string = '';
public pipelines: string = '';

constructor() {}

setDocLinks(newDocLinks: DocLinks) {
Expand Down Expand Up @@ -94,7 +102,7 @@ class ESDocLinks {
this.pythonApiReference = newDocLinks.serverlessClients.pythonGettingStarted;
this.pythonBasicConfig = newDocLinks.serverlessClients.pythonGettingStarted;
this.pythonClient = newDocLinks.serverlessClients.pythonGettingStarted;
// Python
// Ruby
this.rubyBasicConfig = newDocLinks.serverlessClients.rubyGettingStarted;
this.rubyExamples = newDocLinks.serverlessClients.rubyApiReference;
this.rubyClient = newDocLinks.serverlessClients.rubyGettingStarted;
Expand All @@ -103,6 +111,14 @@ class ESDocLinks {
this.gettingStartedIngest = newDocLinks.serverlessSearch.gettingStartedIngest;
this.gettingStartedSearch = newDocLinks.serverlessSearch.gettingStartedSearch;
this.gettingStartedExplore = newDocLinks.serverlessSearch.gettingStartedExplore;

// Ingest processor
this.dataEnrichment = newDocLinks.ingest.dataEnrichment;
this.dataFiltering = newDocLinks.ingest.dataFiltering;
this.arrayOrJson = newDocLinks.ingest.arrayOrJson;
this.dataTransformation = newDocLinks.ingest.dataTransformation;
this.pipelineHandling = newDocLinks.ingest.pipelineHandling;
this.pipelines = newDocLinks.ingest.pipelines;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,31 @@
* 2.0.
*/

import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui';
import {
EuiButton,
EuiFlexGroup,
EuiFlexItem,
EuiLink,
EuiPanel,
EuiSpacer,
EuiText,
EuiTitle,
EuiCode,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { Connector, ConnectorStatus } from '@kbn/search-connectors';
import React, { useState } from 'react';
import { useQueryClient, useMutation } from '@tanstack/react-query';
import { FormattedMessage } from '@kbn/i18n-react';
import { isValidIndexName } from '../../../../utils/validate_index_name';
import { SAVE_LABEL } from '../../../../../common/i18n_string';
import { useConnector } from '../../../hooks/api/use_connector';
import { useKibanaServices } from '../../../hooks/use_kibana';
import { ApiKeyPanel } from './api_key_panel';
import { ConnectorIndexNameForm } from './connector_index_name_form';
import { SyncScheduledCallOut } from './sync_scheduled_callout';

import { docLinks } from '../../../../../common/doc_links';
import { DEFAULT_INGESTION_PIPELINE } from '../../../constants';
interface ConnectorIndexNameProps {
connector: Connector;
}
Expand Down Expand Up @@ -80,6 +92,51 @@ export const ConnectorIndexName: React.FC<ConnectorIndexNameProps> = ({ connecto
onChange={(name) => setNewIndexname(name)}
/>
<EuiSpacer />
<EuiPanel hasBorder>
<EuiFlexGroup direction="column" justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
<EuiTitle size="s">
<h3>
{i18n.translate('xpack.serverlessSearch.connectors.config.preprocessData.title', {
defaultMessage: 'Preprocess your data',
})}
</h3>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText>
<p>
<FormattedMessage
id="xpack.serverlessSearch.connectors.config.preprocessData.description"
defaultMessage="Use ingest pipelines to preprocess data before indexing into Elasticsearch. Note that connector clients use the {clientIngestionPipeline} pipeline for preprocessing."
values={{
clientIngestionPipeline: <EuiCode>{DEFAULT_INGESTION_PIPELINE}</EuiCode>,
}}
/>
</p>
</EuiText>
</EuiFlexItem>
<EuiFlexItem>
<EuiText>
<p>
<EuiLink
data-test-subj="serverlessSearchConnectorIndexNameLearnMoreLink"
href={docLinks.pipelines}
target="_blank"
>
{i18n.translate(
'xpack.serverlessSearch.connectors.config.preprocessDataTitle.learnMore',
{
defaultMessage: 'Learn More',
}
)}
</EuiLink>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiPanel>
<EuiSpacer />
<ApiKeyPanel connector={connector} />
<EuiSpacer />
<EuiFlexGroup>
Expand Down
Loading

0 comments on commit 63dfc7d

Please sign in to comment.