Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fleet] Fix loading package from ES cache #151655

Merged
merged 5 commits into from
Feb 21, 2023
Merged

[Fleet] Fix loading package from ES cache #151655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
aada764
[Fleet] Fix loading package from ES cache
nchaulet Feb 20, 2023
0efd576
[CI] Auto-commit changed files from 'node scripts/precommit_hook.js -…
kibanamachine Feb 20, 2023
923da0f
Fix tests
nchaulet Feb 21, 2023
9333014
Merge branch 'fix-loading-package-from-es-cache' of github.com:nchaul…
nchaulet Feb 21, 2023
4fe3cce
Fix after review
nchaulet Feb 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/server/services/epm/archive/parse.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import { pkgToPkgKey } from '../registry';
import { unpackBufferEntries } from '.';

const readFileAsync = promisify(readFile);
const MANIFEST_NAME = 'manifest.yml';
export const MANIFEST_NAME = 'manifest.yml';

const DEFAULT_RELEASE_VALUE = 'ga';

Expand Down
89 changes: 9 additions & 80 deletions x-pack/plugins/fleet/server/services/epm/archive/storage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@

import { extname } from 'path';

import { uniq } from 'lodash';
import { safeLoad } from 'js-yaml';
import { isBinaryFile } from 'isbinaryfile';
import mime from 'mime-types';
import { v5 as uuidv5 } from 'uuid';
Expand All @@ -20,19 +18,13 @@ import type {
InstallablePackage,
InstallSource,
PackageAssetReference,
RegistryDataStream,
} from '../../../../common/types';
import { pkgToPkgKey } from '../registry';

import { appContextService } from '../../app_context';

import { getArchiveEntry, setArchiveEntry, setArchiveFilelist, setPackageInfo } from '.';
import type { ArchiveEntry } from '.';
import {
parseAndVerifyPolicyTemplates,
parseAndVerifyStreams,
parseTopLevelElasticsearchEntry,
} from './parse';
import { MANIFEST_NAME, parseAndVerifyArchive } from './parse';

const ONE_BYTE = 1024 * 1024;
// could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17
Expand Down Expand Up @@ -188,7 +180,6 @@ export const getEsPackage = async (
savedObjectsClient: SavedObjectsClientContract
) => {
const logger = appContextService.getLogger();
const pkgKey = pkgToPkgKey({ name: pkgName, version: pkgVersion });
const bulkRes = await savedObjectsClient.bulkGet<PackageAsset>(
references.map((reference) => ({
...reference,
Expand Down Expand Up @@ -216,87 +207,25 @@ export const getEsPackage = async (
return undefined;
}

const paths: string[] = [];
const manifests: Record<string, Buffer> = {};
const entries: ArchiveEntry[] = assets.map(packageAssetToArchiveEntry);
const paths: string[] = [];
entries.forEach(({ path, buffer }) => {
if (path && buffer) {
setArchiveEntry(path, buffer);
paths.push(path);
}
paths.push(path);
if (path.endsWith(MANIFEST_NAME) && buffer) manifests[path] = buffer;
});

// create the packageInfo
// TODO: this is mostly copied from validtion.ts, needed in case package does not exist in storage yet or is missing from cache
// we don't want to reach out to the registry again so recreate it here. should check whether it exists in packageInfoCache first
const manifestPath = `${pkgName}-${pkgVersion}/manifest.yml`;
const soResManifest = await savedObjectsClient.get<PackageAsset>(
ASSETS_SAVED_OBJECT_TYPE,
assetPathToObjectId(manifestPath)
);
const packageInfo = safeLoad(soResManifest.attributes.data_utf8);
if (packageInfo.elasticsearch) {
packageInfo.elasticsearch = parseTopLevelElasticsearchEntry(packageInfo.elasticsearch);
}
try {
const readmePath = `docs/README.md`;
await savedObjectsClient.get<PackageAsset>(
ASSETS_SAVED_OBJECT_TYPE,
assetPathToObjectId(`${pkgName}-${pkgVersion}/${readmePath}`)
);
packageInfo.readme = `/package/${pkgName}/${pkgVersion}/${readmePath}`;
} catch (err) {
// read me doesn't exist
}

let dataStreamPaths: string[] = [];
const dataStreams: RegistryDataStream[] = [];
paths
.filter((path) => path.startsWith(`${pkgKey}/data_stream/`))
.forEach((path) => {
const parts = path.split('/');
if (parts.length > 2 && parts[2]) dataStreamPaths.push(parts[2]);
});

dataStreamPaths = uniq(dataStreamPaths);

await Promise.all(
dataStreamPaths.map(async (dataStreamPath) => {
const dataStreamManifestPath = `${pkgKey}/data_stream/${dataStreamPath}/manifest.yml`;
const soResDataStreamManifest = await savedObjectsClient.get<PackageAsset>(
ASSETS_SAVED_OBJECT_TYPE,
assetPathToObjectId(dataStreamManifestPath)
);
const dataStreamManifest = safeLoad(soResDataStreamManifest.attributes.data_utf8);
const {
ingest_pipeline: ingestPipeline,
dataset,
streams: manifestStreams,
...dataStreamManifestProps
} = dataStreamManifest;
const streams = parseAndVerifyStreams(manifestStreams, dataStreamPath);

dataStreams.push({
dataset: dataset || `${pkgName}.${dataStreamPath}`,
package: pkgName,
ingest_pipeline: ingestPipeline,
path: dataStreamPath,
streams,
...dataStreamManifestProps,
});
})
);
packageInfo.policy_templates = parseAndVerifyPolicyTemplates(packageInfo);
packageInfo.data_streams = dataStreams;
packageInfo.assets = paths.map((path) => {
return path.replace(`${pkgName}-${pkgVersion}`, `/package/${pkgName}/${pkgVersion}`);
});

// Add asset references to cache
// // Add asset references to cache
setArchiveFilelist({ name: pkgName, version: pkgVersion }, paths);

const packageInfo = parseAndVerifyArchive(paths, manifests);
setPackageInfo({ name: pkgName, version: pkgVersion, packageInfo });

return {
paths,
packageInfo,
paths,
};
};
25 changes: 22 additions & 3 deletions x-pack/plugins/fleet/server/services/epm/packages/get.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { getSettings } from '../../settings';

import { getPackageInfo, getPackages, getPackageUsageStats } from './get';

const MockRegistry = Registry as jest.Mocked<typeof Registry>;
const MockRegistry = jest.mocked(Registry);

jest.mock('../../settings');

Expand Down Expand Up @@ -200,6 +200,7 @@ describe('When using EPM `get` services', () => {
]);
MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined);
MockRegistry.fetchInfo.mockResolvedValue({} as any);
MockRegistry.pkgToPkgKey.mockImplementation(({ name, version }) => `${name}-${version}`);
});

it('should return installed package that is not in registry with package info', async () => {
Expand Down Expand Up @@ -246,9 +247,27 @@ description: Elasticsearch description`,
}
});
soClient.bulkGet.mockResolvedValue({
saved_objects: [],
saved_objects: [
{
id: 'test',
references: [],
type: 'epm-package-assets',
attributes: {
asset_path: 'elasticsearch-0.0.1/manifest.yml',
data_utf8: `
name: elasticsearch
version: 0.0.1
title: Elastic
description: Elasticsearch description
format_version: 0.0.1
owner: elastic`,
},
},
],
});
await getPackages({
savedObjectsClient: soClient,
});

await expect(
getPackages({
savedObjectsClient: soClient,
Expand Down