[Fleet] Fix upgrades for packages with restructured inputs (#109887) (#…

…110149) * Fix upgrades for packages with restructured inputs Addresses errors surfaced when testing upgrades from AWS 0.6.1 to 0.10.4. Namely, when inputs are removed from a package between versions,we were initially throwing errors for each input in the new package that didn't exist on the outdated package version. Now, we instead simply skip over cases like this in which an input no longer exists on the new package version. * Add basic test cases for restructured packages Co-authored-by: Kyle Pollich <[email protected]>
elastic · Aug 25, 2021 · 8834c99 · 8834c99
1 parent 0fbab51
commit 8834c99
Show file tree

Hide file tree

Showing 25 changed files with 482 additions and 74 deletions.
diff --git a/x-pack/plugins/fleet/common/services/validate_package_policy.ts b/x-pack/plugins/fleet/common/services/validate_package_policy.ts
@@ -97,12 +97,14 @@ export const validatePackagePolicy = (
   >((varDefs, policyTemplate) => {
     (policyTemplate.inputs || []).forEach((input) => {
       const varDefKey = hasIntegrations ? `${policyTemplate.name}-${input.type}` : input.type;
+
       if ((input.vars || []).length) {
         varDefs[varDefKey] = keyBy(input.vars || [], 'name');
       }
     });
     return varDefs;
   }, {});
+
   const streamsByDatasetAndInput = (packageInfo.data_streams || []).reduce<
     Record<string, RegistryStream>
   >((streams, dataStream) => {
@@ -149,6 +151,7 @@ export const validatePackagePolicy = (
     if (input.streams.length) {
       input.streams.forEach((stream) => {
         const streamValidationResults: PackagePolicyConfigValidationResults = {};
+
         const streamVarDefs =
           streamVarDefsByDatasetAndInput[`${stream.data_stream.dataset}-${input.type}`];
 
@@ -157,7 +160,7 @@ export const validatePackagePolicy = (
           streamValidationResults.vars = Object.entries(stream.vars).reduce(
             (results, [name, configEntry]) => {
               results[name] =
-                streamVarDefs[name] && input.enabled && stream.enabled
+                streamVarDefs && streamVarDefs[name] && input.enabled && stream.enabled
                   ? validatePackagePolicyConfig(configEntry, streamVarDefs[name])
                   : null;
               return results;

diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts
@@ -890,41 +890,51 @@ export function overridePackageInputs(
 ): DryRunPackagePolicy {
   if (!inputsOverride) return basePackagePolicy;
 
-  const inputs = [...basePackagePolicy.inputs];
-  const packageName = basePackagePolicy.package!.name;
-  let errors = [];
+  const availablePolicyTemplates = packageInfo.policy_templates ?? [];
+
+  const inputs = [
+    ...basePackagePolicy.inputs.filter((input) => {
+      if (!input.policy_template) {
+        return true;
+      }
+
+      const policyTemplate = availablePolicyTemplates.find(
+        ({ name }) => name === input.policy_template
+      );
+
+      // Ignore any policy template removes in the new package version
+      if (!policyTemplate) {
+        return false;
+      }
+
+      // Ignore any inputs removed from this policy template in the new package version
+      const policyTemplateStillIncludesInput =
+        policyTemplate.inputs?.some(
+          (policyTemplateInput) => policyTemplateInput.type === input.type
+        ) ?? false;
+
+      return policyTemplateStillIncludesInput;
+    }),
+  ];
 
   for (const override of inputsOverride) {
     let originalInput = inputs.find((i) => i.type === override.type);
 
+    // If there's no corresponding input on the original package policy, just
+    // take the override value from the new package as-is. This case typically
+    // occurs when inputs or package policies are added/removed between versions.
     if (!originalInput) {
-      const e = {
-        error: new IngestManagerError(
-          i18n.translate('xpack.fleet.packagePolicyInputOverrideError', {
-            defaultMessage: 'Input type {inputType} does not exist on package {packageName}',
-            values: {
-              inputType: override.type,
-              packageName,
-            },
-          })
-        ),
-        package: { name: packageName, version: basePackagePolicy.package!.version },
-      };
+      inputs.push(override as NewPackagePolicyInput);
+      continue;
+    }
 
-      if (dryRun) {
-        errors.push({
-          key: override.type,
-          message: String(e.error),
-        });
-        continue;
-      } else {
-        throw e;
-      }
+    if (typeof override.enabled !== 'undefined') {
+      originalInput.enabled = override.enabled;
     }
 
-    if (typeof override.enabled !== 'undefined') originalInput.enabled = override.enabled;
-    if (typeof override.keep_enabled !== 'undefined')
+    if (typeof override.keep_enabled !== 'undefined') {
       originalInput.keep_enabled = override.keep_enabled;
+    }
 
     if (override.vars) {
       originalInput = deepMergeVars(originalInput, override);
@@ -936,36 +946,7 @@ export function overridePackageInputs(
           (s) => s.data_stream.dataset === stream.data_stream.dataset
         );
 
-        if (!originalStream) {
-          const streamSet = stream.data_stream.dataset;
-          const e = {
-            error: new IngestManagerError(
-              i18n.translate('xpack.fleet.packagePolicyStreamOverrideError', {
-                defaultMessage:
-                  'Data stream {streamSet} does not exist on {inputType} of package {packageName}',
-                values: {
-                  streamSet,
-                  inputType: override.type,
-                  packageName,
-                },
-              })
-            ),
-            package: { name: packageName, version: basePackagePolicy.package!.version },
-          };
-
-          if (dryRun) {
-            errors.push({
-              key: `${override.type}.streams.${streamSet}`,
-              message: String(e.error),
-            });
-
-            continue;
-          } else {
-            throw e;
-          }
-        }
-
-        if (typeof stream.enabled !== 'undefined') {
+        if (typeof stream.enabled !== 'undefined' && originalStream) {
           originalStream.enabled = stream.enabled;
         }
 
@@ -991,20 +972,22 @@ export function overridePackageInputs(
       }))
       .filter(({ message }) => !!message);
 
-    errors = [...errors, ...responseFormattedValidationErrors];
-  }
+    if (responseFormattedValidationErrors.length) {
+      if (dryRun) {
+        return { ...resultingPackagePolicy, errors: responseFormattedValidationErrors };
+      }
 
-  if (errors.length) {
-    if (dryRun) {
-      return { ...resultingPackagePolicy, errors };
+      throw new IngestManagerError(
+        i18n.translate('xpack.fleet.packagePolicyInvalidError', {
+          defaultMessage: 'Package policy is invalid: {errors}',
+          values: {
+            errors: responseFormattedValidationErrors
+              .map(({ key, message }) => `${key}: ${message}`)
+              .join('\n'),
+          },
+        })
+      );
     }
-
-    throw new IngestManagerError(
-      i18n.translate('xpack.fleet.packagePolicyInvalidError', {
-        defaultMessage: 'Package policy is invalid: {errors}',
-        values: { errors: errors.map(({ key, message }) => `${key}: ${message}`).join('\n') },
-      })
-    );
   }
 
   return resultingPackagePolicy;

diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json
@@ -9980,8 +9980,6 @@
     "xpack.fleet.oldAppTitle": "Ingest Manager",
     "xpack.fleet.overviewPageSubtitle": "ElasticElasticエージェントの集中管理",
     "xpack.fleet.overviewPageTitle": "Fleet",
-    "xpack.fleet.packagePolicyInputOverrideError": "パッケージ{packageName}には入力タイプ{inputType}が存在しません。",
-    "xpack.fleet.packagePolicyStreamOverrideError": "パッケージ{packageName}の{inputType}にはデータストリーム{streamSet}が存在しません",
     "xpack.fleet.packagePolicyValidation.invalidArrayErrorMessage": "無効なフォーマット",
     "xpack.fleet.packagePolicyValidation.invalidYamlFormatErrorMessage": "YAML形式が無効です",
     "xpack.fleet.packagePolicyValidation.nameRequiredErrorMessage": "名前が必要です",

diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json
@@ -10068,8 +10068,6 @@
     "xpack.fleet.oldAppTitle": "采集管理器",
     "xpack.fleet.overviewPageSubtitle": "Elastic 代理的集中管理",
     "xpack.fleet.overviewPageTitle": "Fleet",
-    "xpack.fleet.packagePolicyInputOverrideError": "输入类型 {inputType} 在软件包 {packageName} 上不存在",
-    "xpack.fleet.packagePolicyStreamOverrideError": "数据流 {streamSet} 在软件包 {packageName} 的 {inputType} 上不存在",
     "xpack.fleet.packagePolicyValidation.invalidArrayErrorMessage": "格式无效",
     "xpack.fleet.packagePolicyValidation.invalidYamlFormatErrorMessage": "YAML 格式无效",
     "xpack.fleet.packagePolicyValidation.nameRequiredErrorMessage": "“名称”必填",

diff --git a/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs b/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs
diff --git a/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs b/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs
diff --git a/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs b/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs
diff --git a/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs b/...t_stream/agent/stream/test_stream.yml.hbs → ...m/test_stream/agent/stream/stream.yml.hbs
diff --git a/..._upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new/agent/stream/stream.yml.hbs b/..._upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new/agent/stream/stream.yml.hbs
@@ -0,0 +1 @@
+config.version: "2"
diff --git a/...age_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new/fields/fields.yml b/...age_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/.../package_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new/manifest.yml b/.../package_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new/manifest.yml
@@ -0,0 +1,17 @@
+title: Test stream
+type: logs
+streams:
+  - input: test_input_new
+    vars:
+    - name: test_var_new
+      type: text
+      title: Test Var New
+      default: Test Var New
+      required: true
+      show_user: true
+    - name: test_var_new_2
+      type: text
+      title: Test Var New 2
+      default: Test Var New 2
+      required: true
+      show_user: true
diff --git a/...pgrade/0.5.0-restructure-inputs/data_stream/test_stream_new_2/agent/stream/stream.yml.hbs b/...pgrade/0.5.0-restructure-inputs/data_stream/test_stream_new_2/agent/stream/stream.yml.hbs
@@ -0,0 +1 @@
+config.version: "2"
diff --git a/...e_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new_2/fields/fields.yml b/...e_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new_2/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/...ackage_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new_2/manifest.yml b/...ackage_policy_upgrade/0.5.0-restructure-inputs/data_stream/test_stream_new_2/manifest.yml
@@ -0,0 +1,17 @@
+title: Test stream
+type: logs
+streams:
+  - input: test_input_new_2
+    vars:
+    - name: test_input_new_2_var_1
+      type: text
+      title: Test Input New 2 Var 1
+      default: Test Input New 2 Var 1
+      required: true
+      show_user: true
+    - name: test_input_new_2_var_2
+      type: text
+      title: Test Input New 2 Var 2
+      default: Test Input New 2 Var 2
+      required: true
+      show_user: true
diff --git a/...es/test_packages/package_policy_upgrade/0.5.0-restructure-inputs/docs/README.md b/...es/test_packages/package_policy_upgrade/0.5.0-restructure-inputs/docs/README.md
@@ -0,0 +1,3 @@
+# Test package
+
+This is a test package for testing automated upgrades for package policies
diff --git a/.../apis/fixtures/test_packages/package_policy_upgrade/0.5.0-restructure-inputs/manifest.yml b/.../apis/fixtures/test_packages/package_policy_upgrade/0.5.0-restructure-inputs/manifest.yml
@@ -0,0 +1,27 @@
+format_version: 1.0.0
+name: package_policy_upgrade
+title: Tests package policy upgrades
+description: This is a test package for upgrading package policies
+version: 0.5.0-restructure-inputs
+categories: []
+release: beta
+type: integration
+license: basic
+requirement:
+  elasticsearch:
+    versions: '>7.7.0'
+  kibana:
+    versions: '>7.7.0'
+policy_templates:
+  - name: package_policy_upgrade
+    title: Package Policy Upgrade
+    description: Test Package for Upgrading Package Policies
+    inputs:
+      - type: test_input_new
+        title: Test Input New
+        description: Test Input New
+        enabled: true
+      - type: test_input_new_2
+        title: Test Input New 2
+        description: Test Input New 2
+        enabled: true
diff --git a/....6.0-restructure-policy-templates/data_stream/test_stream_new/agent/stream/stream.yml.hbs b/....6.0-restructure-policy-templates/data_stream/test_stream_new/agent/stream/stream.yml.hbs
@@ -0,0 +1 @@
+config.version: "2"
diff --git a/..._upgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new/fields/fields.yml b/..._upgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/...olicy_upgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new/manifest.yml b/...olicy_upgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new/manifest.yml
@@ -0,0 +1,17 @@
+title: Test stream
+type: logs
+streams:
+  - input: test_input_new
+    vars:
+    - name: test_var_new
+      type: text
+      title: Test Var New
+      default: Test Var New
+      required: true
+      show_user: true
+    - name: test_var_new_2
+      type: text
+      title: Test Var New 2
+      default: Test Var New 2
+      required: true
+      show_user: true
diff --git a/....0-restructure-policy-templates/data_stream/test_stream_new_2/agent/stream/stream.yml.hbs b/....0-restructure-policy-templates/data_stream/test_stream_new_2/agent/stream/stream.yml.hbs
@@ -0,0 +1 @@
+config.version: "2"
diff --git a/...pgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new_2/fields/fields.yml b/...pgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new_2/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/...icy_upgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new_2/manifest.yml b/...icy_upgrade/0.6.0-restructure-policy-templates/data_stream/test_stream_new_2/manifest.yml
@@ -0,0 +1,17 @@
+title: Test stream
+type: logs
+streams:
+  - input: test_input_new_2
+    vars:
+    - name: test_input_new_2_var_1
+      type: text
+      title: Test Input New 2 Var 1
+      default: Test Input New 2 Var 1
+      required: true
+      show_user: true
+    - name: test_input_new_2_var_2
+      type: text
+      title: Test Input New 2 Var 2
+      default: Test Input New 2 Var 2
+      required: true
+      show_user: true
diff --git a/...ckages/package_policy_upgrade/0.6.0-restructure-policy-templates/docs/README.md b/...ckages/package_policy_upgrade/0.6.0-restructure-policy-templates/docs/README.md
@@ -0,0 +1,3 @@
+# Test package
+
+This is a test package for testing automated upgrades for package policies
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Test package

		This is a test package for testing automated upgrades for package policies