[Ingest Manager] Update dataset.* to data_stream.* in package con…

…fig SO attributes (#74414) (#74558) * Update `dataset.*` to `data_stream.*` in full agent config yaml * Replace `dataset.*` with `data_stream.*` in package config saved object attributes # Conflicts: # x-pack/test/page_load_metrics/es_archives/default/mappings.json
elastic · Aug 6, 2020 · ed08468 · ed08468
1 parent 65d8288
commit ed08468
Show file tree

Hide file tree

Showing 20 changed files with 74 additions and 73 deletions.
diff --git a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts
@@ -10,6 +10,7 @@ const CONFIG_KEYS_ORDER = [
   'id',
   'name',
   'revision',
+  'dataset',
   'type',
   'outputs',
   'agent',

diff --git a/x-pack/plugins/ingest_manager/common/services/package_configs_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/package_configs_to_agent_inputs.test.ts
@@ -39,7 +39,7 @@ describe('Ingest Manager - storedPackageConfigsToAgentInputs', () => {
       {
         id: 'test-logs-foo',
         enabled: true,
-        dataset: { name: 'foo', type: 'logs' },
+        data_stream: { dataset: 'foo', type: 'logs' },
         vars: {
           fooVar: { value: 'foo-value' },
           fooVar2: { value: [1, 2] },
@@ -52,7 +52,7 @@ describe('Ingest Manager - storedPackageConfigsToAgentInputs', () => {
       {
         id: 'test-logs-bar',
         enabled: true,
-        dataset: { name: 'bar', type: 'logs' },
+        data_stream: { dataset: 'bar', type: 'logs' },
         vars: {
           barVar: { value: 'bar-value' },
           barVar2: { value: [1, 2] },
@@ -118,7 +118,7 @@ describe('Ingest Manager - storedPackageConfigsToAgentInputs', () => {
         id: 'some-uuid',
         name: 'mock-package-config',
         type: 'test-logs',
-        dataset: { namespace: 'default' },
+        data_stream: { namespace: 'default' },
         use_output: 'default',
         meta: {
           package: {
@@ -129,13 +129,13 @@ describe('Ingest Manager - storedPackageConfigsToAgentInputs', () => {
         streams: [
           {
             id: 'test-logs-foo',
-            dataset: { name: 'foo', type: 'logs' },
+            data_stream: { dataset: 'foo', type: 'logs' },
             fooKey: 'fooValue1',
             fooKey2: ['fooValue2'],
           },
           {
             id: 'test-logs-bar',
-            dataset: { name: 'bar', type: 'logs' },
+            data_stream: { dataset: 'bar', type: 'logs' },
           },
         ],
       },
@@ -160,12 +160,12 @@ describe('Ingest Manager - storedPackageConfigsToAgentInputs', () => {
         id: 'some-uuid',
         name: 'mock-package-config',
         type: 'test-logs',
-        dataset: { namespace: 'default' },
+        data_stream: { namespace: 'default' },
         use_output: 'default',
         streams: [
           {
             id: 'test-logs-foo',
-            dataset: { name: 'foo', type: 'logs' },
+            data_stream: { dataset: 'foo', type: 'logs' },
             fooKey: 'fooValue1',
             fooKey2: ['fooValue2'],
           },

diff --git a/x-pack/plugins/ingest_manager/common/services/package_configs_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/package_configs_to_agent_inputs.ts
@@ -24,7 +24,7 @@ export const storedPackageConfigsToAgentInputs = (
         id: packageConfig.id || packageConfig.name,
         name: packageConfig.name,
         type: input.type,
-        dataset: {
+        data_stream: {
           namespace: packageConfig.namespace || 'default',
         },
         use_output: DEFAULT_OUTPUT.name,
@@ -37,7 +37,7 @@ export const storedPackageConfigsToAgentInputs = (
           .map((stream) => {
             const fullStream: FullAgentConfigInputStream = {
               id: stream.id,
-              dataset: stream.dataset,
+              data_stream: stream.data_stream,
               ...stream.compiled_stream,
               ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
                 acc[key] = value;

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
@@ -83,14 +83,16 @@ describe('Ingest Manager - packageToConfig', () => {
         {
           type: 'foo',
           enabled: true,
-          streams: [{ id: 'foo-foo', enabled: true, dataset: { name: 'foo', type: 'logs' } }],
+          streams: [
+            { id: 'foo-foo', enabled: true, data_stream: { dataset: 'foo', type: 'logs' } },
+          ],
         },
         {
           type: 'bar',
           enabled: true,
           streams: [
-            { id: 'bar-bar', enabled: true, dataset: { name: 'bar', type: 'logs' } },
-            { id: 'bar-bar2', enabled: true, dataset: { name: 'bar2', type: 'logs' } },
+            { id: 'bar-bar', enabled: true, data_stream: { dataset: 'bar', type: 'logs' } },
+            { id: 'bar-bar2', enabled: true, data_stream: { dataset: 'bar2', type: 'logs' } },
           ],
         },
       ]);
@@ -141,7 +143,7 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'foo-foo',
               enabled: true,
-              dataset: { name: 'foo', type: 'logs' },
+              data_stream: { dataset: 'foo', type: 'logs' },
               vars: { 'var-name': { value: 'foo-var-value' } },
             },
           ],
@@ -153,13 +155,13 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'bar-bar',
               enabled: true,
-              dataset: { name: 'bar', type: 'logs' },
+              data_stream: { dataset: 'bar', type: 'logs' },
               vars: { 'var-name': { type: 'text', value: 'bar-var-value' } },
             },
             {
               id: 'bar-bar2',
               enabled: true,
-              dataset: { name: 'bar2', type: 'logs' },
+              data_stream: { dataset: 'bar2', type: 'logs' },
               vars: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } },
             },
           ],
@@ -257,7 +259,7 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'foo-foo',
               enabled: true,
-              dataset: { name: 'foo', type: 'logs' },
+              data_stream: { dataset: 'foo', type: 'logs' },
               vars: {
                 'var-name': { value: 'foo-var-value' },
               },
@@ -275,15 +277,15 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'bar-bar',
               enabled: true,
-              dataset: { name: 'bar', type: 'logs' },
+              data_stream: { dataset: 'bar', type: 'logs' },
               vars: {
                 'var-name': { value: 'bar-var-value' },
               },
             },
             {
               id: 'bar-bar2',
               enabled: true,
-              dataset: { name: 'bar2', type: 'logs' },
+              data_stream: { dataset: 'bar2', type: 'logs' },
               vars: {
                 'var-name': { value: 'bar2-var-value' },
               },
@@ -297,15 +299,15 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'with-disabled-streams-disabled',
               enabled: false,
-              dataset: { name: 'disabled', type: 'logs' },
+              data_stream: { dataset: 'disabled', type: 'logs' },
               vars: {
                 'var-name': { value: [] },
               },
             },
             {
               id: 'with-disabled-streams-disabled2',
               enabled: false,
-              dataset: { name: 'disabled2', type: 'logs' },
+              data_stream: { dataset: 'disabled2', type: 'logs' },
             },
           ],
         },

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts
@@ -19,17 +19,17 @@ import {
 const getStreamsForInputType = (
   inputType: string,
   packageInfo: PackageInfo
-): Array<RegistryStream & { dataset: { type: string; name: string } }> => {
-  const streams: Array<RegistryStream & { dataset: { type: string; name: string } }> = [];
+): Array<RegistryStream & { data_stream: { type: string; dataset: string } }> => {
+  const streams: Array<RegistryStream & { data_stream: { type: string; dataset: string } }> = [];
 
   (packageInfo.datasets || []).forEach((dataset) => {
     (dataset.streams || []).forEach((stream) => {
       if (stream.input === inputType) {
         streams.push({
           ...stream,
-          dataset: {
+          data_stream: {
             type: dataset.type,
-            name: dataset.name,
+            dataset: dataset.name,
           },
         });
       }
@@ -76,12 +76,9 @@ export const packageToPackageConfigInputs = (packageInfo: PackageInfo): PackageC
         packageInfo
       ).map((packageStream) => {
         const stream: PackageConfigInputStream = {
-          id: `${packageInput.type}-${packageStream.dataset.name}`,
+          id: `${packageInput.type}-${packageStream.data_stream.dataset}`,
           enabled: packageStream.enabled === false ? false : true,
-          dataset: {
-            name: packageStream.dataset.name,
-            type: packageStream.dataset.type,
-          },
+          data_stream: packageStream.data_stream,
         };
         if (packageStream.vars && packageStream.vars.length) {
           stream.vars = packageStream.vars.reduce(varsReducer, {});

diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts
@@ -32,8 +32,8 @@ export type AgentConfigSOAttributes = Omit<AgentConfig, 'id'>;
 
 export interface FullAgentConfigInputStream {
   id: string;
-  dataset: {
-    name: string;
+  data_stream: {
+    dataset: string;
     type: string;
   };
   [key: string]: any;
@@ -43,7 +43,7 @@ export interface FullAgentConfigInput {
   id: string;
   name: string;
   type: string;
-  dataset: { namespace: string };
+  data_stream: { namespace: string };
   use_output: string;
   meta?: {
     package?: Pick<PackageConfigPackage, 'name' | 'version'>;

diff --git a/x-pack/plugins/ingest_manager/common/types/models/package_config.ts b/x-pack/plugins/ingest_manager/common/types/models/package_config.ts
@@ -20,8 +20,8 @@ export type PackageConfigConfigRecord = Record<string, PackageConfigConfigRecord
 export interface NewPackageConfigInputStream {
   id: string;
   enabled: boolean;
-  dataset: {
-    name: string;
+  data_stream: {
+    dataset: string;
     type: string;
   };
   vars?: PackageConfigConfigRecord;

diff --git a/...ections/agent_config/create_package_config_page/components/package_config_input_panel.tsx b/...ections/agent_config/create_package_config_page/components/package_config_input_panel.tsx
@@ -39,7 +39,7 @@ const ShortenedHorizontalRule = styled(EuiHorizontalRule)`
 
 const shouldShowStreamsByDefault = (
   packageInput: RegistryInput,
-  packageInputStreams: Array<RegistryStream & { dataset: { name: string } }>,
+  packageInputStreams: Array<RegistryStream & { data_stream: { dataset: string } }>,
   packageConfigInput: PackageConfigInput
 ): boolean => {
   return (
@@ -52,7 +52,7 @@ const shouldShowStreamsByDefault = (
             hasInvalidButRequiredVar(
               stream.vars,
               packageConfigInput.streams.find(
-                (pkgStream) => stream.dataset.name === pkgStream.dataset.name
+                (pkgStream) => stream.data_stream.dataset === pkgStream.data_stream.dataset
               )?.vars
             )
         )
@@ -62,7 +62,7 @@ const shouldShowStreamsByDefault = (
 
 export const PackageConfigInputPanel: React.FunctionComponent<{
   packageInput: RegistryInput;
-  packageInputStreams: Array<RegistryStream & { dataset: { name: string } }>;
+  packageInputStreams: Array<RegistryStream & { data_stream: { dataset: string } }>;
   packageConfigInput: PackageConfigInput;
   updatePackageConfigInput: (updatedInput: Partial<PackageConfigInput>) => void;
   inputValidationResults: PackageConfigInputValidationResults;
@@ -90,7 +90,7 @@ export const PackageConfigInputPanel: React.FunctionComponent<{
         return {
           packageInputStream,
           packageConfigInputStream: packageConfigInput.streams.find(
-            (stream) => stream.dataset.name === packageInputStream.dataset.name
+            (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset
           ),
         };
       })
@@ -201,7 +201,8 @@ export const PackageConfigInputPanel: React.FunctionComponent<{
                     updatedStream: Partial<PackageConfigInputStream>
                   ) => {
                     const indexOfUpdatedStream = packageConfigInput.streams.findIndex(
-                      (stream) => stream.dataset.name === packageInputStream.dataset.name
+                      (stream) =>
+                        stream.data_stream.dataset === packageInputStream.data_stream.dataset
                     );
                     const newStreams = [...packageConfigInput.streams];
                     newStreams[indexOfUpdatedStream] = {

diff --git a/...ager/sections/agent_config/create_package_config_page/services/validate_package_config.ts b/...ager/sections/agent_config/create_package_config_page/services/validate_package_config.ts
@@ -134,7 +134,7 @@ export const validatePackageConfig = (
         if (stream.vars) {
           const streamVarsByName = (
             (
-              registryStreamsByDataset[stream.dataset.name].find(
+              registryStreamsByDataset[stream.data_stream.dataset].find(
                 (registryStream) => registryStream.input === input.type
               ) || {}
             ).vars || []

diff --git a/...tions/agent_config/create_package_config_page/services/validate_package_config.ts.test.ts b/...tions/agent_config/create_package_config_page/services/validate_package_config.ts.test.ts
@@ -159,7 +159,7 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'foo-foo',
-            dataset: { name: 'foo', type: 'logs' },
+            data_stream: { dataset: 'foo', type: 'logs' },
             enabled: true,
             vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } },
           },
@@ -175,13 +175,13 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'bar-bar',
-            dataset: { name: 'bar', type: 'logs' },
+            data_stream: { dataset: 'bar', type: 'logs' },
             enabled: true,
             vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } },
           },
           {
             id: 'bar-bar2',
-            dataset: { name: 'bar2', type: 'logs' },
+            data_stream: { dataset: 'bar2', type: 'logs' },
             enabled: true,
             vars: { 'var-name': { value: undefined, type: 'text' } },
           },
@@ -198,13 +198,13 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'with-disabled-streams-disabled',
-            dataset: { name: 'disabled', type: 'logs' },
+            data_stream: { dataset: 'disabled', type: 'logs' },
             enabled: false,
             vars: { 'var-name': { value: undefined, type: 'text' } },
           },
           {
             id: 'with-disabled-streams-disabled-without-vars',
-            dataset: { name: 'disabled2', type: 'logs' },
+            data_stream: { dataset: 'disabled2', type: 'logs' },
             enabled: false,
           },
         ],
@@ -218,7 +218,7 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'with-no-stream-vars-bar',
-            dataset: { name: 'bar', type: 'logs' },
+            data_stream: { dataset: 'bar', type: 'logs' },
             enabled: true,
           },
         ],
@@ -241,7 +241,7 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'foo-foo',
-            dataset: { name: 'foo', type: 'logs' },
+            data_stream: { dataset: 'foo', type: 'logs' },
             enabled: true,
             vars: { 'var-name': { value: 'invalidyaml: test\n foo bar:', type: 'yaml' } },
           },
@@ -257,13 +257,13 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'bar-bar',
-            dataset: { name: 'bar', type: 'logs' },
+            data_stream: { dataset: 'bar', type: 'logs' },
             enabled: true,
             vars: { 'var-name': { value: '    \n\n', type: 'yaml' } },
           },
           {
             id: 'bar-bar2',
-            dataset: { name: 'bar2', type: 'logs' },
+            data_stream: { dataset: 'bar2', type: 'logs' },
             enabled: true,
             vars: { 'var-name': { value: undefined, type: 'text' } },
           },
@@ -280,7 +280,7 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'with-disabled-streams-disabled',
-            dataset: { name: 'disabled', type: 'logs' },
+            data_stream: { dataset: 'disabled', type: 'logs' },
             enabled: false,
             vars: {
               'var-name': {
@@ -291,7 +291,7 @@ describe('Ingest Manager - validatePackageConfig()', () => {
           },
           {
             id: 'with-disabled-streams-disabled-without-vars',
-            dataset: { name: 'disabled2', type: 'logs' },
+            data_stream: { dataset: 'disabled2', type: 'logs' },
             enabled: false,
           },
         ],
@@ -305,7 +305,7 @@ describe('Ingest Manager - validatePackageConfig()', () => {
         streams: [
           {
             id: 'with-no-stream-vars-bar',
-            dataset: { name: 'bar', type: 'logs' },
+            data_stream: { dataset: 'bar', type: 'logs' },
             enabled: true,
           },
         ],