[Fleet] Add install_format_version to package and reinstall on setup

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -418,6 +418,7 @@ export interface Installation extends SavedObjectAttributes {
   install_source: InstallSource;
   installed_kibana_space_id?: string;
   keep_policies_up_to_date?: boolean;
+  install_format_schema_version?: string;
 }
 
 export interface PackageUsageStats {

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -9,6 +9,8 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta';
 
 const meta = getESAssetMetadata();
 
+export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0';
+
 export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';
 
 export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1';

x-pack/plugins/fleet/server/constants/index.ts

@@ -75,4 +75,5 @@ export {
   FLEET_FINAL_PIPELINE_ID,
   FLEET_FINAL_PIPELINE_CONTENT,
   FLEET_FINAL_PIPELINE_VERSION,
+  FLEET_INSTALL_FORMAT_VERSION,
 } from './fleet_es_assets';

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -257,6 +257,7 @@ const getSavedObjectTypes = (
         install_version: { type: 'keyword' },
         install_status: { type: 'keyword' },
         install_source: { type: 'keyword' },
+        install_format_schema_version: { type: 'version' },
       },
     },
     migrations: {

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -20,7 +20,7 @@ import {
   SO_SEARCH_LIMIT,
 } from '../../../../common';
 import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common';
-import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
+import { PACKAGES_SAVED_OBJECT_TYPE, FLEET_INSTALL_FORMAT_VERSION } from '../../../constants';
 import type { AssetReference, Installation, InstallType } from '../../../types';
 import { prepareToInstallTemplates } from '../elasticsearch/template/install';
 import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy';
@@ -88,7 +88,7 @@ export async function _installPackage({
       } else {
         // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL
         // (it might be stuck) update the saved object and proceed
-        await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
+        await savedObjectsClient.update<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
           install_version: pkgVersion,
           install_status: 'installing',
           install_started_at: new Date().toISOString(),
@@ -254,6 +254,7 @@ export async function _installPackage({
         install_version: pkgVersion,
         install_status: 'installed',
         package_assets: packageAssetRefs,
+        install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
       })
     );
 

x-pack/plugins/fleet/server/services/epm/packages/index.ts

@@ -27,7 +27,12 @@ export {
 export { getBundledPackages } from './bundled_packages';
 
 export type { BulkInstallResponse, IBulkInstallPackageError } from './install';
-export { handleInstallPackageFailure, installPackage, ensureInstalledPackage } from './install';
+export {
+  handleInstallPackageFailure,
+  installPackage,
+  reinstallPackageFromInstallation,
+  ensureInstalledPackage,
+} from './install';
 export { removeInstallation } from './remove';
 
 export class PackageNotInstalledError extends Error {

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -51,6 +51,7 @@ import { getPackageSavedObjects } from './get';
 import { _installPackage } from './_install_package';
 import { removeOldAssets } from './cleanup';
 import { getBundledPackages } from './bundled_packages';
+import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants/fleet_es_assets';
 
 export async function isPackageInstalled(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -214,6 +215,13 @@ interface InstallRegistryPackageParams {
   force?: boolean;
   ignoreConstraints?: boolean;
 }
+interface InstallUploadedArchiveParams {
+  savedObjectsClient: SavedObjectsClientContract;
+  esClient: ElasticsearchClient;
+  archiveBuffer: Buffer;
+  contentType: string;
+  spaceId: string;
+}
 
 function getTelemetryEvent(pkgName: string, pkgVersion: string): PackageUpdateEvent {
   return {
@@ -389,14 +397,6 @@ async function installPackageFromRegistry({
   }
 }
 
-interface InstallUploadedArchiveParams {
-  savedObjectsClient: SavedObjectsClientContract;
-  esClient: ElasticsearchClient;
-  archiveBuffer: Buffer;
-  contentType: string;
-  spaceId: string;
-}
-
 async function installPackageByUpload({
   savedObjectsClient,
   esClient,
@@ -498,6 +498,33 @@ export type InstallPackageParams = {
   | ({ installSource: Extract<InstallSource, 'bundled'> } & InstallUploadedArchiveParams)
 );
 
+export async function reinstallPackageFromInstallation({
+  soClient,
+  esClient,
+  installation,
+}: {
+  soClient: SavedObjectsClientContract;
+  esClient: ElasticsearchClient;
+  installation: Installation;
+}) {
+  if (installation.install_source === 'upload') {
+    throw new Error('Cannot reinstall an uploaded package');
+  }
+  return installPackage({
+    // If the package is bundled reinstall from the registry will still use the bundled package.
+    installSource: 'registry',
+    savedObjectsClient: soClient,
+    pkgkey: Registry.pkgToPkgKey({
+      name: installation.name,
+      version: installation.version,
+    }),
+    esClient,
+    spaceId: installation.installed_kibana_space_id || DEFAULT_SPACE_ID,
+    // Force install the package will update the index template and the datastream write indices
+    force: true,
+  });
+}
+
 export async function installPackage(args: InstallPackageParams): Promise<InstallResult> {
   if (!('installSource' in args)) {
     throw new Error('installSource is required');
@@ -612,6 +639,7 @@ export async function createInstallation(options: {
       install_status: 'installing',
       install_started_at: new Date().toISOString(),
       install_source: installSource,
+      install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
       keep_policies_up_to_date: defaultKeepPoliciesUpToDate,
     },
     { id: pkgName, overwrite: true }

x-pack/plugins/fleet/server/services/setup.ts

@@ -39,6 +39,8 @@ import { pkgToPkgKey } from './epm/registry';
 import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies';
 import { upgradeManagedPackagePolicies } from './managed_package_policies';
 import { getBundledPackages } from './epm/packages';
+import { upgradePackageInstallVersion } from './setup/upgrade_package_install_version';
+
 export interface SetupStatus {
   isInitialized: boolean;
   nonFatalErrors: Array<
@@ -122,6 +124,9 @@ async function createSetupSideEffects(
 
   const nonFatalErrors = [...preconfiguredPackagesNonFatalErrors, ...packagePolicyUpgradeErrors];
 
+  logger.debug('Upgrade Fleet package instal versions');
+  await upgradePackageInstallVersion({ soClient, esClient, logger });
+
   logger.debug('Setting up Fleet enrollment keys');
   await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient);
 

x-pack/plugins/fleet/server/services/setup/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { upgradePackageInstallVersion } from './upgrade_package_install_version';

x-pack/plugins/fleet/server/services/setup/upgrade_package_install_version.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
+import pMap from 'p-map';
+import type { Logger } from '@kbn/logging';
+
+import { PACKAGES_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../constants';
+import { FLEET_INSTALL_FORMAT_VERSION } from '../../constants/fleet_es_assets';
+import type { Installation } from '../../types';
+
+import { reinstallPackageFromInstallation } from '../epm/packages';
+
+/**
+ * Upgrade package install version for packages installed with an older version of Kibana
+ */
+export async function upgradePackageInstallVersion({
+  soClient,
+  esClient,
+  logger,
+}: {
+  soClient: SavedObjectsClientContract;
+  esClient: ElasticsearchClient;
+  logger: Logger;
+}) {
+  const res = await soClient.find<Installation>({
+    type: PACKAGES_SAVED_OBJECT_TYPE,
+    perPage: SO_SEARCH_LIMIT,
+    filter: `${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_status:installed and (${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_format_schema_version < ${FLEET_INSTALL_FORMAT_VERSION} or not ${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_format_schema_version:*)`,
+  });
+
+  if (res.total === 0) {
+    return;
+  }
+
+  await pMap(
+    res.saved_objects,
+    ({ attributes: installation }) => {
+      if (installation.install_source === 'upload') {
+        return;
+      }
+      return reinstallPackageFromInstallation({
+        soClient,
+        esClient,
+        installation,
+      }).catch((err) => {
+        logger.error(
+          `Package needs to be manually reinstalled ${installation.name} updating install_version failed.`
+        );
+      });
+    },
+    { concurrency: 10 }
+  );
+}

x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts

@@ -4,10 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import type { Client } from '@elastic/elasticsearch';
 import expect from '@kbn/expect';
 import { sortBy } from 'lodash';
 import { AssetReference } from '@kbn/fleet-plugin/common';
+import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
 import { setupFleetAndAgents } from '../agents/services';
@@ -764,6 +766,7 @@ const expectAssetsInstalled = ({
       install_status: 'installed',
       install_started_at: res.attributes.install_started_at,
       install_source: 'registry',
+      install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
     });
   });
 };

x-pack/test/fleet_api_integration/apis/epm/update_assets.ts

@@ -6,6 +6,8 @@
  */
 
 import expect from '@kbn/expect';
+import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants';
+
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
 import { setupFleetAndAgents } from '../agents/services';
@@ -510,6 +512,7 @@ export default function (providerContext: FtrProviderContext) {
         install_status: 'installed',
         install_started_at: res.attributes.install_started_at,
         install_source: 'registry',
+        install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
       });
     });
   });