From f4d44c534780e5c173040f97109e34cb5ba75b88 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 4 Jul 2022 10:56:45 -0400 Subject: [PATCH] [Fleet] Add install_format_version to package and reinstall on setup (#135485) --- .../plugins/fleet/common/types/models/epm.ts | 1 + x-pack/plugins/fleet/dev_docs/epm.md | 4 + .../fleet/server/constants/fleet_es_assets.ts | 2 + .../plugins/fleet/server/constants/index.ts | 1 + .../upgrade_package_install_version.test.ts | 194 ++++++++++++++++++ .../fleet/server/saved_objects/index.ts | 1 + .../services/epm/packages/_install_package.ts | 5 +- .../server/services/epm/packages/index.ts | 1 + .../server/services/epm/packages/install.ts | 18 +- .../services/epm/packages/reinstall.test.ts | 84 ++++++++ .../server/services/epm/packages/reinstall.ts | 41 ++++ .../fleet/server/services/setup.test.ts | 1 + x-pack/plugins/fleet/server/services/setup.ts | 43 ++-- .../fleet/server/services/setup/index.ts | 8 + .../setup/upgrade_package_install_version.ts | 63 ++++++ .../apis/epm/install_remove_assets.ts | 3 + .../apis/epm/update_assets.ts | 3 + 17 files changed, 435 insertions(+), 38 deletions(-) create mode 100644 x-pack/plugins/fleet/server/integration_tests/upgrade_package_install_version.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/reinstall.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/reinstall.ts create mode 100644 x-pack/plugins/fleet/server/services/setup/index.ts create mode 100644 x-pack/plugins/fleet/server/services/setup/upgrade_package_install_version.ts diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 902b32745d0e6..d33616e369abe 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -418,6 +418,7 @@ export interface Installation extends SavedObjectAttributes { install_source: InstallSource; installed_kibana_space_id?: string; keep_policies_up_to_date?: boolean; + install_format_schema_version?: string; } export interface PackageUsageStats { diff --git a/x-pack/plugins/fleet/dev_docs/epm.md b/x-pack/plugins/fleet/dev_docs/epm.md index a066b6deb3bc8..7fa96378d7c6d 100644 --- a/x-pack/plugins/fleet/dev_docs/epm.md +++ b/x-pack/plugins/fleet/dev_docs/epm.md @@ -28,3 +28,7 @@ When a package is installed or upgraded, certain Kibana and Elasticsearch assets - Index templates are generated from `YAML` files contained in the package. - There is one index template per data stream. - For the generation of an index template, all `yml` files contained in the package subdirectory `data_stream/DATASET_NAME/fields/` are used. + +# Install format changes + +It is possible that the way we install ES assets change, (adding a new metadata, ...) in this case we use an `install_format_version` attributes on the package saved object and we have a constant `FLEET_INSTALL_FORMAT_VERSION` in Kibana. You can bump the `FLEET_INSTALL_FORMAT_VERSION` so all the packages not installed with that version will be reinstalled and their assets updated. diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index ded329ba79bab..7124ee78f1faf 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -9,6 +9,8 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); +export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0'; + export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1'; diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 11f44ab07896c..3123f2910ed54 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -75,4 +75,5 @@ export { FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, + FLEET_INSTALL_FORMAT_VERSION, } from './fleet_es_assets'; diff --git a/x-pack/plugins/fleet/server/integration_tests/upgrade_package_install_version.test.ts b/x-pack/plugins/fleet/server/integration_tests/upgrade_package_install_version.test.ts new file mode 100644 index 0000000000000..ed7cf5e6d1fd6 --- /dev/null +++ b/x-pack/plugins/fleet/server/integration_tests/upgrade_package_install_version.test.ts @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; + +import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server'; +import { loggerMock } from '@kbn/logging-mocks'; + +import * as kbnTestServer from '@kbn/core/test_helpers/kbn_server'; + +import { upgradePackageInstallVersion } from '../services/setup/upgrade_package_install_version'; +import { + FLEET_INSTALL_FORMAT_VERSION, + PACKAGES_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, +} from '../constants'; +import type { Installation } from '../types'; + +import { useDockerRegistry, waitForFleetSetup } from './helpers'; + +const logFilePath = Path.join(__dirname, 'logs.log'); + +const fakeRequest = { + headers: {}, + getBasePath: () => '', + path: '/', + route: { settings: {} }, + url: { + href: '/', + }, + raw: { + req: { + url: '/', + }, + }, +} as unknown as KibanaRequest; + +describe('Uprade package install version', () => { + let esServer: kbnTestServer.TestElasticsearchUtils; + let kbnServer: kbnTestServer.TestKibanaUtils; + + const registryUrl = useDockerRegistry(); + + const startServers = async () => { + const { startES } = kbnTestServer.createTestServers({ + adjustTimeout: (t) => jest.setTimeout(t), + settings: { + es: { + license: 'trial', + }, + kbn: {}, + }, + }); + + esServer = await startES(); + const startKibana = async () => { + const root = kbnTestServer.createRootWithCorePlugins( + { + xpack: { + fleet: { + registryUrl, + packages: [ + { + name: 'fleet_server', + version: 'latest', + }, + { + name: 'system', + version: 'latest', + }, + { + name: 'nginx', + version: 'latest', + }, + { + name: 'apache', + version: 'latest', + }, + ], + }, + }, + logging: { + appenders: { + file: { + type: 'file', + fileName: logFilePath, + layout: { + type: 'json', + }, + }, + }, + loggers: [ + { + name: 'root', + appenders: ['file'], + }, + { + name: 'plugins.fleet', + level: 'all', + }, + ], + }, + }, + { oss: false } + ); + + await root.preboot(); + const coreSetup = await root.setup(); + const coreStart = await root.start(); + + return { + root, + coreSetup, + coreStart, + stop: async () => await root.shutdown(), + }; + }; + kbnServer = await startKibana(); + + await waitForFleetSetup(kbnServer.root); + }; + + const stopServers = async () => { + if (kbnServer) { + await kbnServer.stop(); + } + + if (esServer) { + await esServer.stop(); + } + + await new Promise((res) => setTimeout(res, 10000)); + }; + + // Share the same servers for all the test to make test a lot faster (but test are not isolated anymore) + beforeAll(async () => { + await startServers(); + }); + + afterAll(async () => { + await stopServers(); + }); + + describe('with package installed with a previous format install version', () => { + let soClient: SavedObjectsClientContract; + + const OUTDATED_PACKAGES = ['nginx', 'apache']; + + beforeAll(async () => { + soClient = kbnServer.coreStart.savedObjects.getScopedClient(fakeRequest, { + excludedWrappers: ['security'], + }); + + const res = await soClient.find({ + type: PACKAGES_SAVED_OBJECT_TYPE, + perPage: SO_SEARCH_LIMIT, + }); + + for (const so of res.saved_objects) { + if (OUTDATED_PACKAGES.includes(so.attributes.name)) { + await soClient.update(PACKAGES_SAVED_OBJECT_TYPE, so.id, { + install_format_schema_version: '0.0.1', + }); + } + } + }); + it('should upgrade package install version for outdated packages', async () => { + const now = Date.now(); + await upgradePackageInstallVersion({ + soClient, + esClient: kbnServer.coreStart.elasticsearch.client.asInternalUser, + logger: loggerMock.create(), + }); + + const res = await soClient.find({ + type: PACKAGES_SAVED_OBJECT_TYPE, + perPage: SO_SEARCH_LIMIT, + }); + expect(res.saved_objects).toHaveLength(4); + res.saved_objects.forEach((so) => { + expect(so.attributes.install_format_schema_version).toBe(FLEET_INSTALL_FORMAT_VERSION); + if (!OUTDATED_PACKAGES.includes(so.attributes.name)) { + expect(new Date(so.updated_at as string).getTime()).toBeLessThan(now); + } else { + expect(new Date(so.updated_at as string).getTime()).toBeGreaterThan(now); + } + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 6438c042bac7c..fb64803a7e6d5 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -257,6 +257,7 @@ const getSavedObjectTypes = ( install_version: { type: 'keyword' }, install_status: { type: 'keyword' }, install_source: { type: 'keyword' }, + install_format_schema_version: { type: 'version' }, }, }, migrations: { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 0124bff41736f..57e293257d176 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -20,7 +20,7 @@ import { SO_SEARCH_LIMIT, } from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; -import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; +import { PACKAGES_SAVED_OBJECT_TYPE, FLEET_INSTALL_FORMAT_VERSION } from '../../../constants'; import type { AssetReference, Installation, InstallType } from '../../../types'; import { prepareToInstallTemplates } from '../elasticsearch/template/install'; import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy'; @@ -88,7 +88,7 @@ export async function _installPackage({ } else { // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL // (it might be stuck) update the saved object and proceed - await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { install_version: pkgVersion, install_status: 'installing', install_started_at: new Date().toISOString(), @@ -254,6 +254,7 @@ export async function _installPackage({ install_version: pkgVersion, install_status: 'installed', package_assets: packageAssetRefs, + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts index bfb09abcfaa28..15c76233e9b75 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/index.ts @@ -28,6 +28,7 @@ export { getBundledPackages } from './bundled_packages'; export type { BulkInstallResponse, IBulkInstallPackageError } from './install'; export { handleInstallPackageFailure, installPackage, ensureInstalledPackage } from './install'; +export { reinstallPackageForInstallation } from './reinstall'; export { removeInstallation } from './remove'; export class PackageNotInstalledError extends Error { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 6bbb91ada321c..90a41add5f184 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -19,6 +19,8 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import pRetry from 'p-retry'; +import { FLEET_INSTALL_FORMAT_VERSION } from '../../../constants/fleet_es_assets'; + import { generateESIndexPatterns } from '../elasticsearch/template/template'; import type { BulkInstallPackageInfo, @@ -214,6 +216,13 @@ interface InstallRegistryPackageParams { force?: boolean; ignoreConstraints?: boolean; } +interface InstallUploadedArchiveParams { + savedObjectsClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; + archiveBuffer: Buffer; + contentType: string; + spaceId: string; +} function getTelemetryEvent(pkgName: string, pkgVersion: string): PackageUpdateEvent { return { @@ -389,14 +398,6 @@ async function installPackageFromRegistry({ } } -interface InstallUploadedArchiveParams { - savedObjectsClient: SavedObjectsClientContract; - esClient: ElasticsearchClient; - archiveBuffer: Buffer; - contentType: string; - spaceId: string; -} - async function installPackageByUpload({ savedObjectsClient, esClient, @@ -612,6 +613,7 @@ export async function createInstallation(options: { install_status: 'installing', install_started_at: new Date().toISOString(), install_source: installSource, + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, keep_policies_up_to_date: defaultKeepPoliciesUpToDate, }, { id: pkgName, overwrite: true } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/reinstall.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/reinstall.test.ts new file mode 100644 index 0000000000000..f58e91cdffe3c --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/reinstall.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; + +import type { Installation } from '../../../../common'; + +import { reinstallPackageForInstallation } from './reinstall'; +import { installPackage } from './install'; + +jest.mock('./install'); + +const mockedInstallPackage = installPackage as jest.MockedFunction; + +describe('reinstallPackageForInstallation', () => { + beforeEach(() => { + mockedInstallPackage.mockReset(); + }); + it('should throw an error for uploaded package', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createInternalClient(); + await expect( + reinstallPackageForInstallation({ + soClient, + esClient, + installation: { + install_source: 'upload', + } as Installation, + }) + ).rejects.toThrow(/Cannot reinstall an uploaded package/); + }); + + it('should install registry package', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createInternalClient(); + await expect( + reinstallPackageForInstallation({ + soClient, + esClient, + installation: { + install_source: 'registry', + name: 'test', + version: '1.0.0', + } as Installation, + }) + ); + + expect(mockedInstallPackage).toHaveBeenCalledWith( + expect.objectContaining({ + installSource: 'registry', + pkgkey: 'test-1.0.0', + force: true, + }) + ); + }); + + it('should install bundled package', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createInternalClient(); + await expect( + reinstallPackageForInstallation({ + soClient, + esClient, + installation: { + install_source: 'bundled', + name: 'test', + version: '1.0.0', + } as Installation, + }) + ); + + expect(mockedInstallPackage).toHaveBeenCalledWith( + expect.objectContaining({ + installSource: 'registry', + pkgkey: 'test-1.0.0', + force: true, + }) + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/reinstall.ts b/x-pack/plugins/fleet/server/services/epm/packages/reinstall.ts new file mode 100644 index 0000000000000..6f6ed35f4974b --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/reinstall.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import type { Installation } from '../../../types'; +import { pkgToPkgKey } from '../registry'; + +import { installPackage } from './install'; + +export async function reinstallPackageForInstallation({ + soClient, + esClient, + installation, +}: { + soClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; + installation: Installation; +}) { + if (installation.install_source === 'upload') { + throw new Error('Cannot reinstall an uploaded package'); + } + return installPackage({ + // If the package is bundled reinstall from the registry will still use the bundled package. + installSource: 'registry', + savedObjectsClient: soClient, + pkgkey: pkgToPkgKey({ + name: installation.name, + version: installation.version, + }), + esClient, + spaceId: installation.installed_kibana_space_id || DEFAULT_SPACE_ID, + // Force install the package will update the index template and the datastream write indices + force: true, + }); +} diff --git a/x-pack/plugins/fleet/server/services/setup.test.ts b/x-pack/plugins/fleet/server/services/setup.test.ts index c7dd2b2657992..ba32b15533c20 100644 --- a/x-pack/plugins/fleet/server/services/setup.test.ts +++ b/x-pack/plugins/fleet/server/services/setup.test.ts @@ -24,6 +24,7 @@ jest.mock('./output'); jest.mock('./download_source'); jest.mock('./epm/packages'); jest.mock('./managed_package_policies'); +jest.mock('./setup/upgrade_package_install_version'); const mockedMethodThrowsError = (mockFn: jest.Mock) => mockFn.mockImplementation(() => { diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index ac8aa17603d92..05d88cc3aa1ce 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -6,9 +6,8 @@ */ import { compact } from 'lodash'; - +import pMap from 'p-map'; import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; - import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import { AUTO_UPDATE_PACKAGES } from '../../common'; @@ -33,12 +32,13 @@ import { settingsService } from '.'; import { awaitIfPending } from './setup_utils'; import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; -import { getInstallations, installPackage } from './epm/packages'; +import { getInstallations, reinstallPackageForInstallation } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; -import { pkgToPkgKey } from './epm/registry'; import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies'; import { upgradeManagedPackagePolicies } from './managed_package_policies'; import { getBundledPackages } from './epm/packages'; +import { upgradePackageInstallVersion } from './setup/upgrade_package_install_version'; + export interface SetupStatus { isInitialized: boolean; nonFatalErrors: Array< @@ -122,6 +122,9 @@ async function createSetupSideEffects( const nonFatalErrors = [...preconfiguredPackagesNonFatalErrors, ...packagePolicyUpgradeErrors]; + logger.debug('Upgrade Fleet package install versions'); + await upgradePackageInstallVersion({ soClient, esClient, logger }); + logger.debug('Setting up Fleet enrollment keys'); await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient); @@ -162,8 +165,9 @@ export async function ensureFleetGlobalEsAssets( (bundledPkg: BundledPackage) => bundledPkg.name === pkg.name && bundledPkg.version === pkg.version ); - await Promise.all( - installedPackages.saved_objects.map(async ({ attributes: installation }) => { + await pMap( + installedPackages.saved_objects, + async ({ attributes: installation }) => { if (installation.install_source !== 'registry') { const matchingBundledPackage = findMatchingBundledPkg(installation); if (!matchingBundledPackage) { @@ -171,36 +175,19 @@ export async function ensureFleetGlobalEsAssets( `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets` ); return; - } else { - await installPackage({ - installSource: 'upload', - savedObjectsClient: soClient, - esClient, - spaceId: DEFAULT_SPACE_ID, - contentType: 'application/zip', - archiveBuffer: matchingBundledPackage.buffer, - }).catch((err) => { - logger.error( - `Bundled package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}` - ); - }); - return; } } - await installPackage({ - installSource: installation.install_source, - savedObjectsClient: soClient, - pkgkey: pkgToPkgKey({ name: installation.name, version: installation.version }), + await reinstallPackageForInstallation({ + soClient, esClient, - spaceId: DEFAULT_SPACE_ID, - // Force install the package will update the index template and the datastream write indices - force: true, + installation, }).catch((err) => { logger.error( `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}` ); }); - }) + }, + { concurrency: 10 } ); } } diff --git a/x-pack/plugins/fleet/server/services/setup/index.ts b/x-pack/plugins/fleet/server/services/setup/index.ts new file mode 100644 index 0000000000000..1646e501c8cab --- /dev/null +++ b/x-pack/plugins/fleet/server/services/setup/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { upgradePackageInstallVersion } from './upgrade_package_install_version'; diff --git a/x-pack/plugins/fleet/server/services/setup/upgrade_package_install_version.ts b/x-pack/plugins/fleet/server/services/setup/upgrade_package_install_version.ts new file mode 100644 index 0000000000000..cceb293b165bd --- /dev/null +++ b/x-pack/plugins/fleet/server/services/setup/upgrade_package_install_version.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; +import pMap from 'p-map'; +import type { Logger } from '@kbn/logging'; + +import { PACKAGES_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../constants'; +import { FLEET_INSTALL_FORMAT_VERSION } from '../../constants/fleet_es_assets'; +import type { Installation } from '../../types'; + +import { reinstallPackageForInstallation } from '../epm/packages'; + +function findOutdatedInstallations(soClient: SavedObjectsClientContract) { + return soClient.find({ + type: PACKAGES_SAVED_OBJECT_TYPE, + perPage: SO_SEARCH_LIMIT, + filter: `${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_status:installed and (${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_format_schema_version < ${FLEET_INSTALL_FORMAT_VERSION} or not ${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_format_schema_version:*)`, + }); +} +/** + * Upgrade package install version for packages installed with an older version of Kibana + */ +export async function upgradePackageInstallVersion({ + soClient, + esClient, + logger, +}: { + soClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; + logger: Logger; +}) { + const res = await findOutdatedInstallations(soClient); + + if (res.total === 0) { + return; + } + + await pMap( + res.saved_objects, + ({ attributes: installation }) => { + // Uploaded package cannot be reinstalled + if (installation.install_source === 'upload') { + logger.warn(`Uploaded package needs to be manually reinstalled ${installation.name}.`); + return; + } + return reinstallPackageForInstallation({ + soClient, + esClient, + installation, + }).catch((err: Error) => { + logger.error( + `Package needs to be manually reinstalled ${installation.name} updating install_version failed. ${err.message}` + ); + }); + }, + { concurrency: 10 } + ); +} diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 9c257bafe87d0..d142dc6fd3515 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -4,10 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import type { Client } from '@elastic/elasticsearch'; import expect from '@kbn/expect'; import { sortBy } from 'lodash'; import { AssetReference } from '@kbn/fleet-plugin/common'; +import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; @@ -764,6 +766,7 @@ const expectAssetsInstalled = ({ install_status: 'installed', install_started_at: res.attributes.install_started_at, install_source: 'registry', + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, }); }); }; diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index fd65a5ed4af67..f31c4cd44b80d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -6,6 +6,8 @@ */ import expect from '@kbn/expect'; +import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants'; + import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; @@ -510,6 +512,7 @@ export default function (providerContext: FtrProviderContext) { install_status: 'installed', install_started_at: res.attributes.install_started_at, install_source: 'registry', + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, }); }); });