Merge branch 'main' into main

elastic · Jan 18, 2023 · ad6eb87 · ad6eb87
2 parents 1cfa3d2 + f8ecb3b
commit ad6eb87
Show file tree

Hide file tree

Showing 45 changed files with 1,446 additions and 239 deletions.
diff --git a/x-pack/plugins/fleet/common/services/policy_template.ts b/x-pack/plugins/fleet/common/services/policy_template.ts
@@ -13,6 +13,7 @@ import type {
   PackageInfo,
   RegistryVarsEntry,
   RegistryDataStream,
+  InstallablePackage,
 } from '../types';
 
 const DATA_STREAM_DATASET_VAR: RegistryVarsEntry = {
@@ -52,7 +53,10 @@ export const getNormalizedInputs = (policyTemplate: RegistryPolicyTemplate): Reg
   return [input];
 };
 
-export const getNormalizedDataStreams = (packageInfo: PackageInfo): RegistryDataStream[] => {
+export const getNormalizedDataStreams = (
+  packageInfo: PackageInfo | InstallablePackage,
+  datasetName?: string
+): RegistryDataStream[] => {
   if (packageInfo.type !== 'input') {
     return packageInfo.data_streams || [];
   }
@@ -66,11 +70,12 @@ export const getNormalizedDataStreams = (packageInfo: PackageInfo): RegistryData
   return policyTemplates.map((policyTemplate) => {
     const dataStream: RegistryDataStream = {
       type: policyTemplate.type,
-      dataset: createDefaultDatasetName(packageInfo, policyTemplate),
+      dataset: datasetName || createDefaultDatasetName(packageInfo, policyTemplate),
       title: policyTemplate.title + ' Dataset',
       release: packageInfo.release || 'ga',
       package: packageInfo.name,
       path: packageInfo.name,
+      elasticsearch: packageInfo.elasticsearch,
       streams: [
         {
           input: policyTemplate.input,
@@ -104,6 +109,6 @@ const addDatasetVarIfNotPresent = (vars?: RegistryVarsEntry[]): RegistryVarsEntr
 };
 
 const createDefaultDatasetName = (
-  packageInfo: PackageInfo,
-  policyTemplate: RegistryPolicyInputOnlyTemplate
+  packageInfo: { name: string },
+  policyTemplate: { name: string }
 ): string => packageInfo.name + '.' + policyTemplate.name;
diff --git a/x-pack/plugins/fleet/common/types/models/package_spec.ts b/x-pack/plugins/fleet/common/types/models/package_spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { RegistryPolicyTemplate, RegistryVarsEntry } from './epm';
+import type { RegistryElasticsearch, RegistryPolicyTemplate, RegistryVarsEntry } from './epm';
 
 // Based on https://github.com/elastic/package-spec/blob/master/versions/1/manifest.spec.yml#L8
 export interface PackageSpecManifest {
@@ -27,6 +27,10 @@ export interface PackageSpecManifest {
   policy_templates?: RegistryPolicyTemplate[];
   vars?: RegistryVarsEntry[];
   owner: { github: string };
+  elasticsearch?: Pick<
+    RegistryElasticsearch,
+    'index_template.settings' | 'index_template.mappings'
+  >;
 }
 
 export type PackageSpecPackageType = 'integration' | 'input';

diff --git a/...ons/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx b/...ons/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx
@@ -9,27 +9,52 @@ import React, { useEffect, useState } from 'react';
 import { EuiComboBox } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 
+import type { DataStream } from '../../../../../../../../../common/types';
+
+interface SelectedDataset {
+  dataset: string;
+  package: string;
+}
+
+const GENERIC_DATASET_NAME = 'generic';
+
 export const DatasetComboBox: React.FC<{
-  value: any;
-  onChange: (newValue: any) => void;
-  datasets: string[];
+  value?: SelectedDataset | string;
+  onChange: (newValue: SelectedDataset) => void;
+  datastreams: DataStream[];
+  pkgName?: string;
   isDisabled?: boolean;
-}> = ({ value, onChange, datasets, isDisabled }) => {
-  const datasetOptions = datasets.map((dataset: string) => ({ label: dataset })) ?? [];
-  const defaultOption = 'generic';
-  const [selectedOptions, setSelectedOptions] = useState<Array<{ label: string }>>([
-    {
-      label: value ?? defaultOption,
-    },
-  ]);
+}> = ({ value, onChange, datastreams, isDisabled, pkgName = '' }) => {
+  const datasetOptions =
+    datastreams.map((datastream: DataStream) => ({
+      label: datastream.dataset,
+      value: datastream,
+    })) ?? [];
+  const existingGenericStream = datasetOptions.find((ds) => ds.label === GENERIC_DATASET_NAME);
+  const valueAsOption = value
+    ? typeof value === 'string'
+      ? { label: value, value: { dataset: value, package: pkgName } }
+      : { label: value.dataset, value: { dataset: value.dataset, package: value.package } }
+    : undefined;
+  const defaultOption = valueAsOption ||
+    existingGenericStream || {
+      label: GENERIC_DATASET_NAME,
+      value: { dataset: GENERIC_DATASET_NAME, package: pkgName },
+    };
+
+  const [selectedOptions, setSelectedOptions] = useState<Array<{ label: string }>>([defaultOption]);
 
   useEffect(() => {
-    if (!value) onChange(defaultOption);
-  }, [value, defaultOption, onChange]);
+    if (!value || typeof value === 'string') onChange(defaultOption.value as SelectedDataset);
+  }, [value, defaultOption.value, onChange, pkgName]);
 
-  const onDatasetChange = (newSelectedOptions: Array<{ label: string }>) => {
+  const onDatasetChange = (newSelectedOptions: Array<{ label: string; value?: DataStream }>) => {
     setSelectedOptions(newSelectedOptions);
-    onChange(newSelectedOptions[0]?.label);
+    const dataStream = newSelectedOptions[0].value;
+    onChange({
+      dataset: newSelectedOptions[0].label,
+      package: !dataStream || typeof dataStream === 'string' ? pkgName : dataStream.package,
+    });
   };
 
   const onCreateOption = (searchValue: string = '') => {
@@ -39,9 +64,13 @@ export const DatasetComboBox: React.FC<{
     }
     const newOption = {
       label: searchValue,
+      value: { dataset: searchValue, package: pkgName },
     };
     setSelectedOptions([newOption]);
-    onChange(searchValue);
+    onChange({
+      dataset: searchValue,
+      package: pkgName,
+    });
   };
   return (
     <EuiComboBox

diff --git a/...gent_policy/create_package_policy_page/components/steps/components/order_datasets.test.ts b/...gent_policy/create_package_policy_page/components/steps/components/order_datasets.test.ts
diff --git a/...ons/agent_policy/create_package_policy_page/components/steps/components/order_datasets.ts b/...ons/agent_policy/create_package_policy_page/components/steps/components/order_datasets.ts
diff --git a/...cy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx b/...cy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx
@@ -42,7 +42,7 @@ import { PackagePolicyEditorDatastreamMappings } from '../../datastream_mappings
 import { ExperimentDatastreamSettings } from './experimental_datastream_settings';
 import { PackagePolicyInputVarField } from './package_policy_input_var_field';
 import { useDataStreamId } from './hooks';
-import { orderDatasets } from './order_datasets';
+import { sortDatastreamsByDataset } from './sort_datastreams';
 
 const ScrollAnchor = styled.div`
   display: none;
@@ -144,9 +144,8 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
     );
 
     const { data: dataStreamsData } = useGetDataStreams();
-    const datasetList =
-      uniq(dataStreamsData?.data_streams.map((dataStream) => dataStream.dataset)) ?? [];
-    const datasets = orderDatasets(datasetList, packageInfo.name);
+    const datasetList = uniq(dataStreamsData?.data_streams) ?? [];
+    const datastreams = sortDatastreamsByDataset(datasetList, packageInfo.name);
 
     return (
       <>
@@ -227,7 +226,8 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                       errors={inputStreamValidationResults?.vars![varName]}
                       forceShowErrors={forceShowErrors}
                       packageType={packageInfo.type}
-                      datasets={datasets}
+                      packageName={packageInfo.name}
+                      datastreams={datastreams}
                       isEditPage={isEditPage}
                     />
                   </EuiFlexItem>
@@ -289,7 +289,8 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                             errors={inputStreamValidationResults?.vars![varName]}
                             forceShowErrors={forceShowErrors}
                             packageType={packageInfo.type}
-                            datasets={datasets}
+                            packageName={packageInfo.name}
+                            datastreams={datastreams}
                             isEditPage={isEditPage}
                           />
                         </EuiFlexItem>

diff --git a/...create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx b/...create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx
@@ -22,7 +22,7 @@ import styled from 'styled-components';
 
 import { CodeEditor } from '@kbn/kibana-react-plugin/public';
 
-import type { RegistryVarsEntry } from '../../../../../../types';
+import type { DataStream, RegistryVarsEntry } from '../../../../../../types';
 
 import { MultiTextInput } from './multi_text_input';
 import { DatasetComboBox } from './dataset_combo';
@@ -39,7 +39,8 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
   forceShowErrors?: boolean;
   frozen?: boolean;
   packageType?: string;
-  datasets?: string[];
+  packageName?: string;
+  datastreams?: DataStream[];
   isEditPage?: boolean;
 }> = memo(
   ({
@@ -50,7 +51,8 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
     forceShowErrors,
     frozen,
     packageType,
-    datasets = [],
+    packageName,
+    datastreams = [],
     isEditPage = false,
   }) => {
     const [isDirty, setIsDirty] = useState<boolean>(false);
@@ -73,7 +75,8 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
       if (name === 'data_stream.dataset' && packageType === 'input') {
         return (
           <DatasetComboBox
-            datasets={datasets}
+            pkgName={packageName}
+            datastreams={datastreams}
             value={value}
             onChange={onChange}
             isDisabled={isEditPage}
@@ -168,7 +171,8 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
       value,
       onChange,
       frozen,
-      datasets,
+      packageName,
+      datastreams,
       isEditPage,
       isInvalid,
       fieldLabel,

diff --git a/...nt_policy/create_package_policy_page/components/steps/components/sort_datastreams.test.ts b/...nt_policy/create_package_policy_page/components/steps/components/sort_datastreams.test.ts
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataStream } from '../../../../../../../../../common/types';
+
+import { sortDatastreamsByDataset } from './sort_datastreams';
+
+const ds = (dataset: string) => ({ dataset } as DataStream);
+describe('orderDatasets', () => {
+  it('should move datasets up that match package name', () => {
+    const datasets = sortDatastreamsByDataset(
+      [ds('system.memory'), ds('elastic_agent'), ds('elastic_agent.filebeat'), ds('system.cpu')],
+      'elastic_agent'
+    );
+
+    expect(datasets).toEqual([
+      ds('elastic_agent'),
+      ds('elastic_agent.filebeat'),
+      ds('system.cpu'),
+      ds('system.memory'),
+    ]);
+  });
+
+  it('should order alphabetically if name does not match', () => {
+    const datasets = sortDatastreamsByDataset([ds('system.memory'), ds('elastic_agent')], 'nginx');
+
+    expect(datasets).toEqual([ds('elastic_agent'), ds('system.memory')]);
+  });
+});
diff --git a/...s/agent_policy/create_package_policy_page/components/steps/components/sort_datastreams.ts b/...s/agent_policy/create_package_policy_page/components/steps/components/sort_datastreams.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { partition, sortBy } from 'lodash';
+
+import type { DataStream } from '../../../../../../../../../common/types';
+
+// sort data streams by dataset name, but promote datastreams that are from this package to the start
+export function sortDatastreamsByDataset(datasetList: DataStream[], name: string): DataStream[] {
+  const [relevantDatasets, otherDatasets] = partition(sortBy(datasetList, 'dataset'), (record) =>
+    record.dataset.startsWith(name)
+  );
+  const datasets = relevantDatasets.concat(otherDatasets);
+  return datasets;
+}
diff --git a/...cy/create_package_policy_page/multi_page_layout/components/page_steps/add_integration.tsx b/...cy/create_package_policy_page/multi_page_layout/components/page_steps/add_integration.tsx
@@ -28,6 +28,7 @@ import type { PackagePolicyValidationResults } from '../../../services';
 import { validatePackagePolicy, validationHasErrors } from '../../../services';
 import { NotObscuredByBottomBar } from '..';
 import { StepConfigurePackagePolicy, StepDefinePackagePolicy } from '../../../components';
+import { prepareInputPackagePolicyDataset } from '../../../services/prepare_input_pkg_policy_dataset';
 
 const ExpandableAdvancedSettings: React.FC = ({ children }) => {
   const [isShowingAdvanced, setIsShowingAdvanced] = useState<boolean>(false);
@@ -147,7 +148,11 @@ export const AddIntegrationPageStep: React.FC<MultiPageStepLayoutProps> = (props
     force?: boolean;
   }) => {
     setFormState('LOADING');
-    const result = await sendCreatePackagePolicy({ ...newPackagePolicy, force });
+    const { policy, forceCreateNeeded } = await prepareInputPackagePolicyDataset(newPackagePolicy);
+    const result = await sendCreatePackagePolicy({
+      ...policy,
+      force: forceCreateNeeded || force,
+    });
     setFormState('SUBMITTED');
     return result;
   };