Skip to content

Commit

Permalink
[Fleet] Finer-grained error information from install/upgrade API (#95649
Browse files Browse the repository at this point in the history
)

* Intercept installation errors and add meta info.

* Adjust mock.

* Catch errors in all steps of install/upgrade.

* Adjust handler for direct package upload.

* Don't throw not-found errors on assets during rollback.

* Correctly catch errors from _installPackage()

* Propagate error from installResult in bulk install case.

* Add tests for rollback.

* Remove unused code.

* Skipping test that doesn't test what it says.

* Fix and reenable test.
  • Loading branch information
skh committed Apr 18, 2021
1 parent 031750d commit 424f77c
Showing 15 changed files with 327 additions and 140 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/common/types/models/epm.ts
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ export enum InstallStatus {
uninstalling = 'uninstalling',
}

export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install';
export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install' | 'unknown';
export type InstallSource = 'registry' | 'upload';

export type EpmPackageInstallStatus = 'installed' | 'installing';
7 changes: 5 additions & 2 deletions x-pack/plugins/fleet/common/types/rest_spec/epm.ts
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ import type {
RegistrySearchResult,
PackageInfo,
PackageUsageStats,
InstallType,
} from '../models/epm';

export interface GetCategoriesRequest {
@@ -83,8 +84,10 @@ export interface IBulkInstallPackageHTTPError {
}

export interface InstallResult {
assets: AssetReference[];
status: 'installed' | 'already_installed';
assets?: AssetReference[];
status?: 'installed' | 'already_installed';
error?: Error;
installType: InstallType;
}

export interface BulkInstallPackageInfo {
46 changes: 24 additions & 22 deletions x-pack/plugins/fleet/server/routes/epm/handlers.ts
Original file line number Diff line number Diff line change
@@ -226,20 +226,21 @@ export const installPackageFromRegistryHandler: RequestHandler<
const savedObjectsClient = context.core.savedObjects.client;
const esClient = context.core.elasticsearch.client.asCurrentUser;
const { pkgkey } = request.params;
try {
const res = await installPackage({
installSource: 'registry',
savedObjectsClient,
pkgkey,
esClient,
force: request.body?.force,
});

const res = await installPackage({
installSource: 'registry',
savedObjectsClient,
pkgkey,
esClient,
force: request.body?.force,
});
if (!res.error) {
const body: InstallPackageResponse = {
response: res.assets,
response: res.assets || [],
};
return response.ok({ body });
} catch (e) {
return await defaultIngestErrorHandler({ error: e, response });
} else {
return await defaultIngestErrorHandler({ error: res.error, response });
}
};

@@ -292,20 +293,21 @@ export const installPackageByUploadHandler: RequestHandler<
const esClient = context.core.elasticsearch.client.asCurrentUser;
const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later
const archiveBuffer = Buffer.from(request.body);
try {
const res = await installPackage({
installSource: 'upload',
savedObjectsClient,
esClient,
archiveBuffer,
contentType,
});

const res = await installPackage({
installSource: 'upload',
savedObjectsClient,
esClient,
archiveBuffer,
contentType,
});
if (!res.error) {
const body: InstallPackageResponse = {
response: res.assets,
response: res.assets || [],
};
return response.ok({ body });
} catch (error) {
return defaultIngestErrorHandler({ error, response });
} else {
return defaultIngestErrorHandler({ error: res.error, response });
}
};

Original file line number Diff line number Diff line change
@@ -32,22 +32,27 @@ export async function bulkInstallPackages({
);

logger.debug(`kicking off bulk install of ${packagesToInstall.join(', ')} from registry`);
const installResults = await Promise.allSettled(
const bulkInstallResults = await Promise.allSettled(
latestPackagesResults.map(async (result, index) => {
const packageName = packagesToInstall[index];
if (result.status === 'fulfilled') {
const latestPackage = result.value;
return {
name: packageName,
version: latestPackage.version,
result: await installPackage({
savedObjectsClient,
esClient,
pkgkey: Registry.pkgToPkgKey(latestPackage),
installSource,
skipPostInstall: true,
}),
};
const installResult = await installPackage({
savedObjectsClient,
esClient,
pkgkey: Registry.pkgToPkgKey(latestPackage),
installSource,
skipPostInstall: true,
});
if (installResult.error) {
return { name: packageName, error: installResult.error };
} else {
return {
name: packageName,
version: latestPackage.version,
result: installResult,
};
}
}
return { name: packageName, error: result.reason };
})
@@ -56,18 +61,27 @@ export async function bulkInstallPackages({
// only install index patterns if we completed install for any package-version for the
// first time, aka fresh installs or upgrades
if (
installResults.find(
(result) => result.status === 'fulfilled' && result.value.result?.status === 'installed'
bulkInstallResults.find(
(result) =>
result.status === 'fulfilled' &&
!result.value.result?.error &&
result.value.result?.status === 'installed'
)
) {
await installIndexPatterns({ savedObjectsClient, esClient, installSource });
}

return installResults.map((result, index) => {
return bulkInstallResults.map((result, index) => {
const packageName = packagesToInstall[index];
return result.status === 'fulfilled'
? result.value
: { name: packageName, error: result.reason };
if (result.status === 'fulfilled') {
if (result.value && result.value.error) {
return { name: packageName, error: result.value.error };
} else {
return result.value;
}
} else {
return { name: packageName, error: result.reason };
}
});
}

Original file line number Diff line number Diff line change
@@ -77,7 +77,7 @@ describe('ensureInstalledDefaultPackages', () => {
return [
{
name: mockInstallation.attributes.name,
result: { assets: [], status: 'installed' },
result: { assets: [], status: 'installed', installType: 'install' },
version: '',
statusCode: 200,
},
@@ -95,13 +95,13 @@ describe('ensureInstalledDefaultPackages', () => {
return [
{
name: 'success one',
result: { assets: [], status: 'installed' },
result: { assets: [], status: 'installed', installType: 'install' },
version: '',
statusCode: 200,
},
{
name: 'success two',
result: { assets: [], status: 'installed' },
result: { assets: [], status: 'installed', installType: 'install' },
version: '',
statusCode: 200,
},
@@ -111,7 +111,7 @@ describe('ensureInstalledDefaultPackages', () => {
},
{
name: 'success three',
result: { assets: [], status: 'installed' },
result: { assets: [], status: 'installed', installType: 'install' },
version: '',
statusCode: 200,
},
@@ -134,7 +134,7 @@ describe('ensureInstalledDefaultPackages', () => {
return [
{
name: 'undefined package',
result: { assets: [], status: 'installed' },
result: { assets: [], status: 'installed', installType: 'install' },
version: '',
statusCode: 200,
},
208 changes: 117 additions & 91 deletions x-pack/plugins/fleet/server/services/epm/packages/install.ts
Original file line number Diff line number Diff line change
@@ -201,54 +201,62 @@ async function installPackageFromRegistry({
// TODO: change epm API to /packageName/version so we don't need to do this
const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey);

// get the currently installed package
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
const installType = getInstallType({ pkgVersion, installedPkg });

// get latest package version
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);

// let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
const installOutOfDateVersionOk =
force || ['reinstall', 'reupdate', 'rollback'].includes(installType);
// if an error happens during getInstallType, report that we don't know
let installType: InstallType = 'unknown';

// if the requested version is the same as installed version, check if we allow it based on
// current installed package status and force flag, if we don't allow it,
// just return the asset references from the existing installation
if (
installedPkg?.attributes.version === pkgVersion &&
installedPkg?.attributes.install_status === 'installed'
) {
if (!force) {
logger.debug(`${pkgkey} is already installed, skipping installation`);
return {
assets: [
...installedPkg.attributes.installed_es,
...installedPkg.attributes.installed_kibana,
],
status: 'already_installed',
};
try {
// get the currently installed package
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
installType = getInstallType({ pkgVersion, installedPkg });

// get latest package version
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);

// let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
const installOutOfDateVersionOk =
force || ['reinstall', 'reupdate', 'rollback'].includes(installType);

// if the requested version is the same as installed version, check if we allow it based on
// current installed package status and force flag, if we don't allow it,
// just return the asset references from the existing installation
if (
installedPkg?.attributes.version === pkgVersion &&
installedPkg?.attributes.install_status === 'installed'
) {
if (!force) {
logger.debug(`${pkgkey} is already installed, skipping installation`);
return {
assets: [
...installedPkg.attributes.installed_es,
...installedPkg.attributes.installed_kibana,
],
status: 'already_installed',
installType,
};
}
}
}

// if the requested version is out-of-date of the latest package version, check if we allow it
// if we don't allow it, return an error
if (semverLt(pkgVersion, latestPackage.version)) {
if (!installOutOfDateVersionOk) {
throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
// if the requested version is out-of-date of the latest package version, check if we allow it
// if we don't allow it, return an error
if (semverLt(pkgVersion, latestPackage.version)) {
if (!installOutOfDateVersionOk) {
throw new PackageOutdatedError(
`${pkgkey} is out-of-date and cannot be installed or updated`
);
}
logger.debug(
`${pkgkey} is out-of-date, installing anyway due to ${
force ? 'force flag' : `install type ${installType}`
}`
);
}
logger.debug(
`${pkgkey} is out-of-date, installing anyway due to ${
force ? 'force flag' : `install type ${installType}`
}`
);
}

// get package info
const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
// get package info
const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);

// try installing the package, if there was an error, call error handler and rethrow
try {
// try installing the package, if there was an error, call error handler and rethrow
// TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status
// @ts-ignore
return _installPackage({
savedObjectsClient,
esClient,
@@ -257,19 +265,26 @@ async function installPackageFromRegistry({
packageInfo,
installType,
installSource: 'registry',
}).then((assets) => {
return { assets, status: 'installed' };
});
})
.then((assets) => {
return { assets, status: 'installed', installType };
})
.catch(async (err: Error) => {
await handleInstallPackageFailure({
savedObjectsClient,
error: err,
pkgName,
pkgVersion,
installedPkg,
esClient,
});
return { error: err, installType };
});
} catch (e) {
await handleInstallPackageFailure({
savedObjectsClient,
return {
error: e,
pkgName,
pkgVersion,
installedPkg,
esClient,
});
throw e;
installType,
};
}
}

@@ -286,46 +301,57 @@ async function installPackageByUpload({
archiveBuffer,
contentType,
}: InstallUploadedArchiveParams): Promise<InstallResult> {
const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType);

const installedPkg = await getInstallationObject({
savedObjectsClient,
pkgName: packageInfo.name,
});
// if an error happens during getInstallType, report that we don't know
let installType: InstallType = 'unknown';
try {
const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType);

const installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg });
if (installType !== 'install') {
throw new PackageOperationNotSupportedError(
`Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.`
);
}
const installedPkg = await getInstallationObject({
savedObjectsClient,
pkgName: packageInfo.name,
});

const installSource = 'upload';
const paths = await unpackBufferToCache({
name: packageInfo.name,
version: packageInfo.version,
installSource,
archiveBuffer,
contentType,
});
installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg });
if (installType !== 'install') {
throw new PackageOperationNotSupportedError(
`Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.`
);
}

setPackageInfo({
name: packageInfo.name,
version: packageInfo.version,
packageInfo,
});
const installSource = 'upload';
const paths = await unpackBufferToCache({
name: packageInfo.name,
version: packageInfo.version,
installSource,
archiveBuffer,
contentType,
});

return _installPackage({
savedObjectsClient,
esClient,
installedPkg,
paths,
packageInfo,
installType,
installSource,
}).then((assets) => {
return { assets, status: 'installed' };
});
setPackageInfo({
name: packageInfo.name,
version: packageInfo.version,
packageInfo,
});
// TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status
// @ts-ignore
return _installPackage({
savedObjectsClient,
esClient,
installedPkg,
paths,
packageInfo,
installType,
installSource,
})
.then((assets) => {
return { assets, status: 'installed', installType };
})
.catch(async (err: Error) => {
return { error: err, installType };
});
} catch (e) {
return { error: e, installType };
}
}

export type InstallPackageParams = {
@@ -352,7 +378,7 @@ export async function installPackage(args: InstallPackageParams) {
esClient,
force,
}).then(async (installResult) => {
if (skipPostInstall) {
if (skipPostInstall || installResult.error) {
return installResult;
}
logger.debug(`install of ${pkgkey} finished, running post-install`);
@@ -374,7 +400,7 @@ export async function installPackage(args: InstallPackageParams) {
archiveBuffer,
contentType,
}).then(async (installResult) => {
if (skipPostInstall) {
if (skipPostInstall || installResult.error) {
return installResult;
}
logger.debug(`install of uploaded package finished, running post-install`);
7 changes: 6 additions & 1 deletion x-pack/plugins/fleet/server/services/epm/packages/remove.ts
Original file line number Diff line number Diff line change
@@ -79,6 +79,7 @@ export async function removeInstallation(options: {
return installedAssets;
}

// TODO: this is very much like deleteKibanaSavedObjectsAssets below
function deleteKibanaAssets(
installedObjects: KibanaAssetReference[],
savedObjectsClient: SavedObjectsClientContract
@@ -136,6 +137,7 @@ async function deleteTemplate(esClient: ElasticsearchClient, name: string): Prom
}
}

// TODO: this is very much like deleteKibanaAssets above
export async function deleteKibanaSavedObjectsAssets(
savedObjectsClient: SavedObjectsClientContract,
installedRefs: AssetReference[]
@@ -153,6 +155,9 @@ export async function deleteKibanaSavedObjectsAssets(
try {
await Promise.all(deletePromises);
} catch (err) {
logger.warn(err);
// in the rollback case, partial installs are likely, so missing assets are not an error
if (!savedObjectsClient.errors.isNotFoundError(err)) {
logger.error(err);
}
}
}
1 change: 1 addition & 0 deletions x-pack/test/fleet_api_integration/apis/epm/index.js
Original file line number Diff line number Diff line change
@@ -24,5 +24,6 @@ export default function loadTests({ loadTestFile }) {
loadTestFile(require.resolve('./update_assets'));
loadTestFile(require.resolve('./data_stream'));
loadTestFile(require.resolve('./package_install_complete'));
loadTestFile(require.resolve('./install_error_rollback'));
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { skipIfNoDockerRegistry } from '../../helpers';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const goodPackage = 'error_handling-0.1.0';
const badPackage = 'error_handling-0.2.0';

const installPackage = async (pkgkey: string) => {
await supertest
.post(`/api/fleet/epm/packages/${pkgkey}`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true });
};

const getPackageInfo = async (pkgkey: string) => {
return await supertest.get(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx');
};

describe('package installation error handling and rollback', async () => {
skipIfNoDockerRegistry(providerContext);
beforeEach(async () => {
await esArchiver.load('empty_kibana');
});
afterEach(async () => {
await esArchiver.unload('empty_kibana');
});

it('on a fresh install, it should uninstall a broken package during rollback', async function () {
await supertest
.post(`/api/fleet/epm/packages/${badPackage}`)
.set('kbn-xsrf', 'xxxx')
.expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana

const pkgInfoResponse = await getPackageInfo(badPackage);
expect(JSON.parse(pkgInfoResponse.text).response.status).to.be('not_installed');
});

it('on an upgrade, it should fall back to the previous good version during rollback', async function () {
await installPackage(goodPackage);
await supertest
.post(`/api/fleet/epm/packages/${badPackage}`)
.set('kbn-xsrf', 'xxxx')
.expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana

const goodPkgInfoResponse = await getPackageInfo(goodPackage);
expect(JSON.parse(goodPkgInfoResponse.text).response.status).to.be('installed');
expect(JSON.parse(goodPkgInfoResponse.text).response.version).to.be('0.1.0');
});
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
This package should install without errors.

Version 0.2.0 of this package should fail during installation. We need this good version to test rollback.
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"attributes": {
"description": "sample visualization",
"title": "sample vis title",
"uiStateJSON": "{}",
"version": 1,
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}"
},
"id": "sample_visualization",
"type": "visualization",
"migrationVersion": {
"visualization": "7.7.0"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
format_version: 1.0.0
name: error_handling
title: Error handling
description: tests error handling and rollback
version: 0.1.0
categories: []
release: beta
type: integration
license: basic

requirement:
elasticsearch:
versions: '>7.7.0'
kibana:
versions: '>7.7.0'

icons:
- src: '/img/logo_overrides_64_color.svg'
size: '16x16'
type: 'image/svg+xml'
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
This package should fail during installation.

Version 0.1.0 of this package should install without errors, and be rolled back to without errors.

This package contains one Kibana visualization that requires a non-existent version of Kibana in order to trigger an error during installation.
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"attributes": {
"description": "sample visualization",
"title": "sample vis title",
"uiStateJSON": "{}",
"version": 1,
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}"
},
"id": "sample_visualization",
"type": "visualization",
"migrationVersion": {
"visualization": "12.7.0"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
format_version: 1.0.0
name: error_handling
title: Error handling
description: tests error handling and rollback
version: 0.2.0
categories: []
release: beta
type: integration
license: basic

requirement:
elasticsearch:
versions: '>7.7.0'
kibana:
versions: '>7.7.0'

icons:
- src: '/img/logo_overrides_64_color.svg'
size: '16x16'

0 comments on commit 424f77c

Please sign in to comment.