Skip to content

Commit

Permalink
[Fleet] Add install_format_version to package and reinstall on setup (e…
Browse files Browse the repository at this point in the history
  • Loading branch information
nchaulet authored and Vadim Yakhin committed Jul 5, 2022
1 parent 0fa3d2d commit f4d44c5
Show file tree
Hide file tree
Showing 17 changed files with 435 additions and 38 deletions.
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/common/types/models/epm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,7 @@ export interface Installation extends SavedObjectAttributes {
install_source: InstallSource;
installed_kibana_space_id?: string;
keep_policies_up_to_date?: boolean;
install_format_schema_version?: string;
}

export interface PackageUsageStats {
Expand Down
4 changes: 4 additions & 0 deletions x-pack/plugins/fleet/dev_docs/epm.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,7 @@ When a package is installed or upgraded, certain Kibana and Elasticsearch assets
- Index templates are generated from `YAML` files contained in the package.
- There is one index template per data stream.
- For the generation of an index template, all `yml` files contained in the package subdirectory `data_stream/DATASET_NAME/fields/` are used.

# Install format changes

It is possible that the way we install ES assets change, (adding a new metadata, ...) in this case we use an `install_format_version` attributes on the package saved object and we have a constant `FLEET_INSTALL_FORMAT_VERSION` in Kibana. You can bump the `FLEET_INSTALL_FORMAT_VERSION` so all the packages not installed with that version will be reinstalled and their assets updated.
2 changes: 2 additions & 0 deletions x-pack/plugins/fleet/server/constants/fleet_es_assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta';

const meta = getESAssetMetadata();

export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0';

export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';

export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1';
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/server/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,5 @@ export {
FLEET_FINAL_PIPELINE_ID,
FLEET_FINAL_PIPELINE_CONTENT,
FLEET_FINAL_PIPELINE_VERSION,
FLEET_INSTALL_FORMAT_VERSION,
} from './fleet_es_assets';
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import Path from 'path';

import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server';
import { loggerMock } from '@kbn/logging-mocks';

import * as kbnTestServer from '@kbn/core/test_helpers/kbn_server';

import { upgradePackageInstallVersion } from '../services/setup/upgrade_package_install_version';
import {
FLEET_INSTALL_FORMAT_VERSION,
PACKAGES_SAVED_OBJECT_TYPE,
SO_SEARCH_LIMIT,
} from '../constants';
import type { Installation } from '../types';

import { useDockerRegistry, waitForFleetSetup } from './helpers';

const logFilePath = Path.join(__dirname, 'logs.log');

const fakeRequest = {
headers: {},
getBasePath: () => '',
path: '/',
route: { settings: {} },
url: {
href: '/',
},
raw: {
req: {
url: '/',
},
},
} as unknown as KibanaRequest;

describe('Uprade package install version', () => {
let esServer: kbnTestServer.TestElasticsearchUtils;
let kbnServer: kbnTestServer.TestKibanaUtils;

const registryUrl = useDockerRegistry();

const startServers = async () => {
const { startES } = kbnTestServer.createTestServers({
adjustTimeout: (t) => jest.setTimeout(t),
settings: {
es: {
license: 'trial',
},
kbn: {},
},
});

esServer = await startES();
const startKibana = async () => {
const root = kbnTestServer.createRootWithCorePlugins(
{
xpack: {
fleet: {
registryUrl,
packages: [
{
name: 'fleet_server',
version: 'latest',
},
{
name: 'system',
version: 'latest',
},
{
name: 'nginx',
version: 'latest',
},
{
name: 'apache',
version: 'latest',
},
],
},
},
logging: {
appenders: {
file: {
type: 'file',
fileName: logFilePath,
layout: {
type: 'json',
},
},
},
loggers: [
{
name: 'root',
appenders: ['file'],
},
{
name: 'plugins.fleet',
level: 'all',
},
],
},
},
{ oss: false }
);

await root.preboot();
const coreSetup = await root.setup();
const coreStart = await root.start();

return {
root,
coreSetup,
coreStart,
stop: async () => await root.shutdown(),
};
};
kbnServer = await startKibana();

await waitForFleetSetup(kbnServer.root);
};

const stopServers = async () => {
if (kbnServer) {
await kbnServer.stop();
}

if (esServer) {
await esServer.stop();
}

await new Promise((res) => setTimeout(res, 10000));
};

// Share the same servers for all the test to make test a lot faster (but test are not isolated anymore)
beforeAll(async () => {
await startServers();
});

afterAll(async () => {
await stopServers();
});

describe('with package installed with a previous format install version', () => {
let soClient: SavedObjectsClientContract;

const OUTDATED_PACKAGES = ['nginx', 'apache'];

beforeAll(async () => {
soClient = kbnServer.coreStart.savedObjects.getScopedClient(fakeRequest, {
excludedWrappers: ['security'],
});

const res = await soClient.find<Installation>({
type: PACKAGES_SAVED_OBJECT_TYPE,
perPage: SO_SEARCH_LIMIT,
});

for (const so of res.saved_objects) {
if (OUTDATED_PACKAGES.includes(so.attributes.name)) {
await soClient.update<Installation>(PACKAGES_SAVED_OBJECT_TYPE, so.id, {
install_format_schema_version: '0.0.1',
});
}
}
});
it('should upgrade package install version for outdated packages', async () => {
const now = Date.now();
await upgradePackageInstallVersion({
soClient,
esClient: kbnServer.coreStart.elasticsearch.client.asInternalUser,
logger: loggerMock.create(),
});

const res = await soClient.find<Installation>({
type: PACKAGES_SAVED_OBJECT_TYPE,
perPage: SO_SEARCH_LIMIT,
});
expect(res.saved_objects).toHaveLength(4);
res.saved_objects.forEach((so) => {
expect(so.attributes.install_format_schema_version).toBe(FLEET_INSTALL_FORMAT_VERSION);
if (!OUTDATED_PACKAGES.includes(so.attributes.name)) {
expect(new Date(so.updated_at as string).getTime()).toBeLessThan(now);
} else {
expect(new Date(so.updated_at as string).getTime()).toBeGreaterThan(now);
}
});
});
});
});
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/server/saved_objects/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ const getSavedObjectTypes = (
install_version: { type: 'keyword' },
install_status: { type: 'keyword' },
install_source: { type: 'keyword' },
install_format_schema_version: { type: 'version' },
},
},
migrations: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
SO_SEARCH_LIMIT,
} from '../../../../common';
import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import { PACKAGES_SAVED_OBJECT_TYPE, FLEET_INSTALL_FORMAT_VERSION } from '../../../constants';
import type { AssetReference, Installation, InstallType } from '../../../types';
import { prepareToInstallTemplates } from '../elasticsearch/template/install';
import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy';
Expand Down Expand Up @@ -88,7 +88,7 @@ export async function _installPackage({
} else {
// if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL
// (it might be stuck) update the saved object and proceed
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
await savedObjectsClient.update<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_version: pkgVersion,
install_status: 'installing',
install_started_at: new Date().toISOString(),
Expand Down Expand Up @@ -254,6 +254,7 @@ export async function _installPackage({
install_version: pkgVersion,
install_status: 'installed',
package_assets: packageAssetRefs,
install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
})
);

Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/server/services/epm/packages/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export { getBundledPackages } from './bundled_packages';

export type { BulkInstallResponse, IBulkInstallPackageError } from './install';
export { handleInstallPackageFailure, installPackage, ensureInstalledPackage } from './install';
export { reinstallPackageForInstallation } from './reinstall';
export { removeInstallation } from './remove';

export class PackageNotInstalledError extends Error {
Expand Down
18 changes: 10 additions & 8 deletions x-pack/plugins/fleet/server/services/epm/packages/install.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';

import pRetry from 'p-retry';

import { FLEET_INSTALL_FORMAT_VERSION } from '../../../constants/fleet_es_assets';

import { generateESIndexPatterns } from '../elasticsearch/template/template';
import type {
BulkInstallPackageInfo,
Expand Down Expand Up @@ -214,6 +216,13 @@ interface InstallRegistryPackageParams {
force?: boolean;
ignoreConstraints?: boolean;
}
interface InstallUploadedArchiveParams {
savedObjectsClient: SavedObjectsClientContract;
esClient: ElasticsearchClient;
archiveBuffer: Buffer;
contentType: string;
spaceId: string;
}

function getTelemetryEvent(pkgName: string, pkgVersion: string): PackageUpdateEvent {
return {
Expand Down Expand Up @@ -389,14 +398,6 @@ async function installPackageFromRegistry({
}
}

interface InstallUploadedArchiveParams {
savedObjectsClient: SavedObjectsClientContract;
esClient: ElasticsearchClient;
archiveBuffer: Buffer;
contentType: string;
spaceId: string;
}

async function installPackageByUpload({
savedObjectsClient,
esClient,
Expand Down Expand Up @@ -612,6 +613,7 @@ export async function createInstallation(options: {
install_status: 'installing',
install_started_at: new Date().toISOString(),
install_source: installSource,
install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
keep_policies_up_to_date: defaultKeepPoliciesUpToDate,
},
{ id: pkgName, overwrite: true }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';

import type { Installation } from '../../../../common';

import { reinstallPackageForInstallation } from './reinstall';
import { installPackage } from './install';

jest.mock('./install');

const mockedInstallPackage = installPackage as jest.MockedFunction<typeof installPackage>;

describe('reinstallPackageForInstallation', () => {
beforeEach(() => {
mockedInstallPackage.mockReset();
});
it('should throw an error for uploaded package', async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createInternalClient();
await expect(
reinstallPackageForInstallation({
soClient,
esClient,
installation: {
install_source: 'upload',
} as Installation,
})
).rejects.toThrow(/Cannot reinstall an uploaded package/);
});

it('should install registry package', async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createInternalClient();
await expect(
reinstallPackageForInstallation({
soClient,
esClient,
installation: {
install_source: 'registry',
name: 'test',
version: '1.0.0',
} as Installation,
})
);

expect(mockedInstallPackage).toHaveBeenCalledWith(
expect.objectContaining({
installSource: 'registry',
pkgkey: 'test-1.0.0',
force: true,
})
);
});

it('should install bundled package', async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createInternalClient();
await expect(
reinstallPackageForInstallation({
soClient,
esClient,
installation: {
install_source: 'bundled',
name: 'test',
version: '1.0.0',
} as Installation,
})
);

expect(mockedInstallPackage).toHaveBeenCalledWith(
expect.objectContaining({
installSource: 'registry',
pkgkey: 'test-1.0.0',
force: true,
})
);
});
});
Loading

0 comments on commit f4d44c5

Please sign in to comment.