diff --git a/fleet_packages.json b/fleet_packages.json index 3657057ad3431..69fd83f12037c 100644 --- a/fleet_packages.json +++ b/fleet_packages.json @@ -12,4 +12,25 @@ in order to verify package integrity. */ -[] +[ + { + "name": "apm", + "version": "8.1.0" + }, + { + "name": "elastic_agent", + "version": "1.3.0" + }, + { + "name": "endpoint", + "version": "1.5.0" + }, + { + "name": "fleet_server", + "version": "1.1.0" + }, + { + "name": "synthetics", + "version": "0.9.2" + } +] diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index fe34b36f7781f..d04855480dd62 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -35,6 +35,7 @@ export interface FleetConfigType { developer?: { disableRegistryVersionCheck?: boolean; allowAgentUpgradeSourceUri?: boolean; + bundledPackageLocation?: string; }; } diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 1c7e09a51c5da..39650bfed5da8 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -305,6 +305,7 @@ export enum RegistryDataStreamKeys { } export interface RegistryDataStream { + [key: string]: any; [RegistryDataStreamKeys.type]: string; [RegistryDataStreamKeys.ilm_policy]?: string; [RegistryDataStreamKeys.hidden]?: boolean; @@ -323,6 +324,7 @@ export interface RegistryElasticsearch { privileges?: RegistryDataStreamPrivileges; 'index_template.settings'?: estypes.IndicesIndexSettings; 'index_template.mappings'?: estypes.MappingTypeMapping; + 'ingest_pipeline.name'?: string; } export interface RegistryDataStreamPrivileges { diff --git a/x-pack/plugins/fleet/common/types/models/package_spec.ts b/x-pack/plugins/fleet/common/types/models/package_spec.ts index 57aaa0230e647..f4021b087912a 100644 --- a/x-pack/plugins/fleet/common/types/models/package_spec.ts +++ b/x-pack/plugins/fleet/common/types/models/package_spec.ts @@ -70,4 +70,5 @@ export interface PackageSpecScreenshot { title: string; size?: string; type?: string; + path?: string; } diff --git a/x-pack/plugins/fleet/server/index.ts b/x-pack/plugins/fleet/server/index.ts index 550d6acaffc7e..47b94103c65fa 100644 --- a/x-pack/plugins/fleet/server/index.ts +++ b/x-pack/plugins/fleet/server/index.ts @@ -5,6 +5,8 @@ * 2.0. */ +import path from 'path'; + import { schema } from '@kbn/config-schema'; import type { TypeOf } from '@kbn/config-schema'; import type { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server'; @@ -40,6 +42,8 @@ export type { } from './types'; export { AgentNotFoundError, FleetUnauthorizedError } from './errors'; +const DEFAULT_BUNDLED_PACKAGE_LOCATION = path.join(__dirname, '../target/bundled_packages'); + export const config: PluginConfigDescriptor = { exposeToBrowser: { epm: true, @@ -130,6 +134,7 @@ export const config: PluginConfigDescriptor = { developer: schema.object({ disableRegistryVersionCheck: schema.boolean({ defaultValue: false }), allowAgentUpgradeSourceUri: schema.boolean({ defaultValue: false }), + bundledPackageLocation: schema.string({ defaultValue: DEFAULT_BUNDLED_PACKAGE_LOCATION }), }), }), }; diff --git a/x-pack/plugins/fleet/server/integration_tests/__snapshots__/cloud_preconfiguration.test.ts.snap b/x-pack/plugins/fleet/server/integration_tests/__snapshots__/cloud_preconfiguration.test.ts.snap index 80f2c39abe988..a4ba6b0c04c8a 100644 --- a/x-pack/plugins/fleet/server/integration_tests/__snapshots__/cloud_preconfiguration.test.ts.snap +++ b/x-pack/plugins/fleet/server/integration_tests/__snapshots__/cloud_preconfiguration.test.ts.snap @@ -115,7 +115,7 @@ Object { "meta": Object { "package": Object { "name": "apm", - "version": "8.2.0-dev3", + "version": "8.2.0-dev4", }, }, "name": "Elastic APM", diff --git a/x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts b/x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts index 7dd58a546aef0..04d75f15c7f81 100644 --- a/x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts +++ b/x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts @@ -24,7 +24,7 @@ export function useDockerRegistry() { let dockerProcess: ChildProcess | undefined; async function startDockerRegistryServer() { - const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:8b4ce36ecdf86e6cfdf781d9df8d564a014add9afc9aec21cf2c5a68ff82d3ab`; + const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:b3dfc6a11ff7dce82ba8689ea9eeb54e353c6b4bfd2d28127b20ef72fd8883e9`; const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage]; diff --git a/x-pack/plugins/fleet/server/integration_tests/validate_bundled_packages.test.ts b/x-pack/plugins/fleet/server/integration_tests/validate_bundled_packages.test.ts new file mode 100644 index 0000000000000..fa7f8b2275dad --- /dev/null +++ b/x-pack/plugins/fleet/server/integration_tests/validate_bundled_packages.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; +import fs from 'fs/promises'; + +import JSON5 from 'json5'; +import { REPO_ROOT } from '@kbn/utils'; + +import * as Registry from '../services/epm/registry'; +import { generatePackageInfoFromArchiveBuffer } from '../services/epm/archive'; + +import { createAppContextStartContractMock } from '../mocks'; +import { appContextService } from '../services'; + +import { useDockerRegistry } from './helpers'; + +describe('validate bundled packages', () => { + const registryUrl = useDockerRegistry(); + let mockContract: ReturnType; + + beforeEach(() => { + mockContract = createAppContextStartContractMock({ registryUrl }); + appContextService.start(mockContract); + }); + + async function getBundledPackageEntries() { + const configFilePath = path.resolve(REPO_ROOT, 'fleet_packages.json'); + const configFile = await fs.readFile(configFilePath, 'utf8'); + const bundledPackages = JSON5.parse(configFile); + + return bundledPackages as Array<{ name: string; version: string }>; + } + + async function setupPackageObjects() { + const bundledPackages = await getBundledPackageEntries(); + + const packageObjects = await Promise.all( + bundledPackages.map(async (bundledPackage) => { + const registryPackage = await Registry.getRegistryPackage( + bundledPackage.name, + bundledPackage.version + ); + + const packageArchive = await Registry.fetchArchiveBuffer( + bundledPackage.name, + bundledPackage.version + ); + + return { registryPackage, packageArchive }; + }) + ); + + return packageObjects; + } + + it('generates matching package info objects for uploaded and registry packages', async () => { + const packageObjects = await setupPackageObjects(); + + for (const packageObject of packageObjects) { + const { registryPackage, packageArchive } = packageObject; + + const archivePackageInfo = await generatePackageInfoFromArchiveBuffer( + packageArchive.archiveBuffer, + 'application/zip' + ); + + expect(archivePackageInfo.packageInfo.data_streams).toEqual( + registryPackage.packageInfo.data_streams + ); + } + }); +}); diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 8e6155e6bf7c2..b229677a4ded9 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -21,6 +21,7 @@ import type { PackagePolicyServiceInterface } from '../services/package_policy'; import type { AgentPolicyServiceInterface } from '../services'; import type { FleetAppContext } from '../plugin'; import { createMockTelemetryEventsSender } from '../telemetry/__mocks__'; +import type { FleetConfigType } from '../../common'; import { createFleetAuthzMock } from '../../common'; import { agentServiceMock } from '../services/agents/agent_service.mock'; import type { FleetRequestHandlerContext } from '../types'; @@ -39,11 +40,14 @@ export interface MockedFleetAppContext extends FleetAppContext { logger: ReturnType['get']>; } -export const createAppContextStartContractMock = (): MockedFleetAppContext => { +export const createAppContextStartContractMock = ( + configOverrides: Partial = {} +): MockedFleetAppContext => { const config = { agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + ...configOverrides, }; const config$ = of(config); diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 7c590eb1bcd39..e7e43d3d932b7 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -23,7 +23,7 @@ import { getBufferExtractor } from './extract'; export * from './cache'; export { getBufferExtractor, untarBuffer, unzipBuffer } from './extract'; -export { parseAndVerifyArchiveBuffer as parseAndVerifyArchiveEntries } from './validation'; +export { generatePackageInfoFromArchiveBuffer } from './parse'; export interface ArchiveEntry { path: string; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts similarity index 69% rename from x-pack/plugins/fleet/server/services/epm/archive/validation.ts rename to x-pack/plugins/fleet/server/services/epm/archive/parse.ts index 5fdbdc5b17332..828a31b3ab03e 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { merge } from '@kbn/std'; import yaml from 'js-yaml'; import { pick, uniq } from 'lodash'; @@ -32,6 +33,45 @@ import { unpackBufferEntries } from './index'; const MANIFESTS: Record = {}; const MANIFEST_NAME = 'manifest.yml'; +const DEFAULT_RELEASE_VALUE = 'ga'; + +// Ingest pipelines are specified in a `data_stream//elasticsearch/ingest_pipeline/` directory where a `default` +// ingest pipeline should be specified by one of these filenames. +const DEFAULT_INGEST_PIPELINE_VALUE = 'default'; +const DEFAULT_INGEST_PIPELINE_FILE_NAME_YML = 'default.yml'; +const DEFAULT_INGEST_PIPELINE_FILE_NAME_JSON = 'default.json'; + +// Borrowed from https://github.com/elastic/kibana/blob/main/x-pack/plugins/security_solution/common/utils/expand_dotted.ts +// with some alterations around non-object values. The package registry service expands some dotted fields from manifest files, +// so we need to do the same here. +const expandDottedField = (dottedFieldName: string, val: unknown): object => { + const parts = dottedFieldName.split('.'); + + if (parts.length === 1) { + return { [parts[0]]: val }; + } else { + return { [parts[0]]: expandDottedField(parts.slice(1).join('.'), val) }; + } +}; + +export const expandDottedObject = (dottedObj: object) => { + if (typeof dottedObj !== 'object' || Array.isArray(dottedObj)) { + return dottedObj; + } + return Object.entries(dottedObj).reduce( + (acc, [key, val]) => merge(acc, expandDottedField(key, val)), + {} + ); +}; + +export const expandDottedEntries = (obj: object) => { + return Object.entries(obj).reduce((acc, [key, value]) => { + acc[key] = expandDottedObject(value); + + return acc; + }, {} as Record); +}; + // not sure these are 100% correct but they do the job here // keeping them local until others need them type OptionalPropertyOf = Exclude< @@ -76,10 +116,19 @@ const registryPolicyTemplateProps = Object.values(RegistryPolicyTemplateKeys); const registryStreamProps = Object.values(RegistryStreamKeys); const registryDataStreamProps = Object.values(RegistryDataStreamKeys); -// TODO: everything below performs verification of manifest.yml files, and hence duplicates functionality already implemented in the -// package registry. At some point this should probably be replaced (or enhanced) with verification based on -// https://github.com/elastic/package-spec/ -export async function parseAndVerifyArchiveBuffer( +/* + This function generates a package info object (see type `ArchivePackage`) by parsing and verifying the `manifest.yml` file as well + as the directory structure for the given package archive and other files adhering to the package spec: https://github.com/elastic/package-spec. + + Currently, this process is duplicative of logic that's already implemented in the Package Registry codebase, + e.g. https://github.com/elastic/package-registry/blob/main/packages/package.go. Because of this duplication, it's likely for our parsing/verification + logic to fall out of sync with the registry codebase's implementation. + + This should be addressed in https://github.com/elastic/kibana/issues/115032 + where we'll no longer use the package registry endpoint as a source of truth for package info objects, and instead Fleet will _always_ generate + them in the manner implemented below. +*/ +export async function generatePackageInfoFromArchiveBuffer( archiveBuffer: Buffer, contentType: string ): Promise<{ paths: string[]; packageInfo: ArchivePackage }> { @@ -144,8 +193,13 @@ function parseAndVerifyArchive(paths: string[]): ArchivePackage { ); } - parsed.data_streams = parseAndVerifyDataStreams(paths, parsed.name, parsed.version); + const parsedDataStreams = parseAndVerifyDataStreams(paths, parsed.name, parsed.version); + if (parsedDataStreams.length) { + parsed.data_streams = parsedDataStreams; + } + parsed.policy_templates = parseAndVerifyPolicyTemplates(manifest); + // add readme if exists const readme = parseAndVerifyReadme(paths, parsed.name, parsed.version); if (readme) { @@ -202,11 +256,11 @@ export function parseAndVerifyDataStreams( const { title: dataStreamTitle, - release, + release = DEFAULT_RELEASE_VALUE, type, dataset, - ingest_pipeline: ingestPipeline, streams: manifestStreams, + elasticsearch, ...restOfProps } = manifest; if (!(dataStreamTitle && type)) { @@ -214,28 +268,75 @@ export function parseAndVerifyDataStreams( `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'type'` ); } + + let ingestPipeline; + const ingestPipelinePaths = paths.filter((path) => + path.startsWith(`${pkgKey}/data_stream/${dataStreamPath}/elasticsearch/ingest_pipeline`) + ); + + if ( + ingestPipelinePaths.length && + (ingestPipelinePaths.some((ingestPipelinePath) => + ingestPipelinePath.endsWith(DEFAULT_INGEST_PIPELINE_FILE_NAME_YML) + ) || + ingestPipelinePaths.some((ingestPipelinePath) => + ingestPipelinePath.endsWith(DEFAULT_INGEST_PIPELINE_FILE_NAME_JSON) + )) + ) { + ingestPipeline = DEFAULT_INGEST_PIPELINE_VALUE; + } + const streams = parseAndVerifyStreams(manifestStreams, dataStreamPath); + const parsedElasticsearchEntry: Record = {}; + + if (ingestPipeline) { + parsedElasticsearchEntry['ingest_pipeline.name'] = DEFAULT_INGEST_PIPELINE_VALUE; + } + + if (elasticsearch?.privileges) { + parsedElasticsearchEntry.privileges = elasticsearch.privileges; + } + + if (elasticsearch?.index_template?.mappings) { + parsedElasticsearchEntry['index_template.mappings'] = expandDottedEntries( + elasticsearch.index_template.mappings + ); + } + + if (elasticsearch?.index_template?.settings) { + parsedElasticsearchEntry['index_template.settings'] = expandDottedEntries( + elasticsearch.index_template.settings + ); + } + + // Build up the stream object here so we can conditionally insert nullable fields. The package registry omits undefined + // fields, so we're mimicking that behavior here. + const dataStreamObject: RegistryDataStream = { + title: dataStreamTitle, + release, + type, + package: pkgName, + dataset: dataset || `${pkgName}.${dataStreamPath}`, + path: dataStreamPath, + elasticsearch: parsedElasticsearchEntry, + }; + + if (ingestPipeline) { + dataStreamObject.ingest_pipeline = ingestPipeline; + } + + if (streams.length) { + dataStreamObject.streams = streams; + } + dataStreams.push( - Object.entries(restOfProps).reduce( - (validatedDataStream, [key, value]) => { - if (registryDataStreamProps.includes(key as RegistryDataStreamKeys)) { - // @ts-expect-error - validatedDataStream[key] = value; - } - return validatedDataStream; - }, - { - title: dataStreamTitle, - release, - type, - package: pkgName, - dataset: dataset || `${pkgName}.${dataStreamPath}`, - ingest_pipeline: ingestPipeline, - path: dataStreamPath, - streams, + Object.entries(restOfProps).reduce((validatedDataStream, [key, value]) => { + if (registryDataStreamProps.includes(key as RegistryDataStreamKeys)) { + validatedDataStream[key] = value; } - ) + return validatedDataStream; + }, dataStreamObject) ); }); @@ -261,25 +362,27 @@ export function parseAndVerifyStreams( `Invalid manifest for data stream ${dataStreamPath}: stream is missing one or more fields of: input, title` ); } + const vars = parseAndVerifyVars(manifestVars, `data stream ${dataStreamPath}`); - // default template path name see https://github.com/elastic/package-registry/blob/master/util/dataset.go#L143 + const streamObject: RegistryStream = { + input, + title: streamTitle, + template_path: templatePath || 'stream.yml.hbs', + }; + + if (vars.length) { + streamObject.vars = vars; + } + streams.push( - Object.entries(restOfProps).reduce( - (validatedStream, [key, value]) => { - if (registryStreamProps.includes(key as RegistryStreamKeys)) { - // @ts-expect-error - validatedStream[key] = value; - } - return validatedStream; - }, - { - input, - title: streamTitle, - vars, - template_path: templatePath || 'stream.yml.hbs', - } as RegistryStream - ) + Object.entries(restOfProps).reduce((validatedStream, [key, value]) => { + if (registryStreamProps.includes(key as RegistryStreamKeys)) { + // @ts-expect-error + validatedStream[key] = value; + } + return validatedStream; + }, streamObject) ); }); } diff --git a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts index bc26388c990f3..d2a96921bf033 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts @@ -27,7 +27,7 @@ import { appContextService } from '../../app_context'; import { getArchiveEntry, setArchiveEntry, setArchiveFilelist, setPackageInfo } from './index'; import type { ArchiveEntry } from './index'; -import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './validation'; +import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './parse'; const ONE_BYTE = 1024 * 1024; // could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17 diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts index 77ece9e1d7787..6934134c34aca 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts @@ -9,19 +9,26 @@ import fs from 'fs/promises'; import path from 'path'; import type { BundledPackage } from '../../../types'; +import { IngestManagerError } from '../../../errors'; import { appContextService } from '../../app_context'; import { splitPkgKey } from '../registry'; -const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../../target/bundled_packages'); - export async function getBundledPackages(): Promise { + const config = appContextService.getConfig(); + + const bundledPackageLocation = config?.developer?.bundledPackageLocation; + + if (!bundledPackageLocation) { + throw new IngestManagerError('xpack.fleet.developer.bundledPackageLocation is not configured'); + } + try { - const dirContents = await fs.readdir(BUNDLED_PACKAGE_DIRECTORY); + const dirContents = await fs.readdir(bundledPackageLocation); const zipFiles = dirContents.filter((file) => file.endsWith('.zip')); const result = await Promise.all( zipFiles.map(async (zipFile) => { - const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile)); + const file = await fs.readFile(path.join(bundledPackageLocation, zipFile)); const { pkgName, pkgVersion } = splitPkgKey(zipFile.replace(/\.zip$/, '')); @@ -36,7 +43,7 @@ export async function getBundledPackages(): Promise { return result; } catch (err) { const logger = appContextService.getLogger(); - logger.debug(`Unable to read bundled packages from ${BUNDLED_PACKAGE_DIRECTORY}`); + logger.debug(`Unable to read bundled packages from ${bundledPackageLocation}`); return []; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index c803b0ff18a44..973e2624af5d9 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -54,7 +54,7 @@ jest.mock('../kibana/index_pattern/install', () => { }); jest.mock('../archive', () => { return { - parseAndVerifyArchiveEntries: jest.fn(() => + generatePackageInfoFromArchiveBuffer: jest.fn(() => Promise.resolve({ packageInfo: { name: 'apache', version: '1.3.0' } }) ), unpackBufferToCache: jest.fn(), diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 23883f90d4248..8a4271c20be65 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -32,7 +32,11 @@ import type { } from '../../../types'; import { appContextService } from '../../app_context'; import * as Registry from '../registry'; -import { setPackageInfo, parseAndVerifyArchiveEntries, unpackBufferToCache } from '../archive'; +import { + setPackageInfo, + generatePackageInfoFromArchiveBuffer, + unpackBufferToCache, +} from '../archive'; import { toAssetReference } from '../kibana/assets/install'; import type { ArchiveAsset } from '../kibana/assets/install'; @@ -391,7 +395,7 @@ async function installPackageByUpload({ let installType: InstallType = 'unknown'; const telemetryEvent: PackageUpdateEvent = getTelemetryEvent('', ''); try { - const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType); + const { packageInfo } = await generatePackageInfoFromArchiveBuffer(archiveBuffer, contentType); const installedPkg = await getInstallationObject({ savedObjectsClient, diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index db6a324352ca1..182f20297afd5 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -261,7 +261,7 @@ export async function ensureCachedArchiveInfo( } } -async function fetchArchiveBuffer( +export async function fetchArchiveBuffer( pkgName: string, pkgVersion: string ): Promise<{ archiveBuffer: Buffer; archivePath: string }> { diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index 4247be10e0792..6364b1de59fc8 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -14,7 +14,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./file')); loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); - // loadTestFile(require.resolve('./install_bundled')); + loadTestFile(require.resolve('./install_bundled')); loadTestFile(require.resolve('./install_by_upload')); loadTestFile(require.resolve('./install_endpoint')); loadTestFile(require.resolve('./install_overrides')); diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts b/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts index c70495ea7809d..2fb227c5afc1a 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts @@ -9,6 +9,7 @@ import expect from '@kbn/expect'; import fs from 'fs/promises'; import path from 'path'; +import { BUNDLED_PACKAGE_DIR } from '../../config'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { setupFleetAndAgents } from '../agents/services'; @@ -22,30 +23,31 @@ export default function (providerContext: FtrProviderContext) { path.dirname(__filename), '../fixtures/bundled_packages' ); - const BUNDLED_PACKAGES_DIR = path.join( - path.dirname(__filename), - '../../../../plugins/fleet/target/bundled_packages' - ); const bundlePackage = async (name: string) => { try { - await fs.access(BUNDLED_PACKAGES_DIR); + await fs.access(BUNDLED_PACKAGE_DIR); } catch (error) { - await fs.mkdir(BUNDLED_PACKAGES_DIR); + await fs.mkdir(BUNDLED_PACKAGE_DIR); } await fs.copyFile( path.join(BUNDLED_PACKAGE_FIXTURES_DIR, `${name}.zip`), - path.join(BUNDLED_PACKAGES_DIR, `${name}.zip`) + path.join(BUNDLED_PACKAGE_DIR, `${name}.zip`) ); }; const removeBundledPackages = async () => { try { - const files = await fs.readdir(BUNDLED_PACKAGES_DIR); + const files = await fs.readdir(BUNDLED_PACKAGE_DIR); for (const file of files) { - await fs.unlink(path.join(BUNDLED_PACKAGES_DIR, file)); + const isFixtureFile = !!(await fs.readFile(path.join(BUNDLED_PACKAGE_FIXTURES_DIR, file))); + + // Only remove fixture files - leave normal bundled packages in place + if (isFixtureFile) { + await fs.unlink(path.join(BUNDLED_PACKAGE_DIR, file)); + } } } catch (error) { log.error('Error removing bundled packages'); @@ -63,10 +65,11 @@ export default function (providerContext: FtrProviderContext) { describe('without registry', () => { it('installs from bundled source via api', async () => { - await bundlePackage('elastic_agent-1.2.0'); + // Need to bundle a package that doesn't conflict with those listed in `fleet_packages.json + await bundlePackage('nginx-1.2.1'); const response = await supertest - .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) + .post(`/api/fleet/epm/packages/nginx/1.2.1`) .set('kbn-xsrf', 'xxxx') .type('application/json') .send({ force: true }) @@ -76,11 +79,11 @@ export default function (providerContext: FtrProviderContext) { }); it('allows for upgrading from newer bundled source when outdated package was installed from bundled source', async () => { - await bundlePackage('elastic_agent-1.0.0'); - await bundlePackage('elastic_agent-1.2.0'); + await bundlePackage('nginx-1.1.0'); + await bundlePackage('nginx-1.2.1'); const installResponse = await supertest - .post(`/api/fleet/epm/packages/elastic_agent/1.0.0`) + .post(`/api/fleet/epm/packages/nginx/1.1.0`) .set('kbn-xsrf', 'xxxx') .type('application/json') .send({ force: true }) @@ -89,7 +92,7 @@ export default function (providerContext: FtrProviderContext) { expect(installResponse.body._meta.install_source).to.be('bundled'); const updateResponse = await supertest - .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) + .post(`/api/fleet/epm/packages/nginx/1.2.1`) .set('kbn-xsrf', 'xxxx') .type('application/json') .send({ force: true }) @@ -101,10 +104,10 @@ export default function (providerContext: FtrProviderContext) { describe('with registry', () => { it('allows for updating from registry when outdated package is installed from bundled source', async () => { - await bundlePackage('elastic_agent-1.2.0'); + await bundlePackage('nginx-1.1.0'); const bundledInstallResponse = await supertest - .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) + .post(`/api/fleet/epm/packages/nginx/1.1.0`) .set('kbn-xsrf', 'xxxx') .type('application/json') .send({ force: true }) @@ -112,8 +115,9 @@ export default function (providerContext: FtrProviderContext) { expect(bundledInstallResponse.body._meta.install_source).to.be('bundled'); + // Update to one version prior to the bundled version of nginx const registryUpdateResponse = await supertest - .post(`/api/fleet/epm/packages/elastic_agent/1.3.0`) + .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) .set('kbn-xsrf', 'xxxx') .type('application/json') .send({ force: true }) diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 97f0bce458791..28b68609ce15e 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -75,7 +75,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/gzip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(27); + expect(res.body.items.length).to.be(29); }); it('should install a zip archive correctly and package info should return correctly after validation', async function () { @@ -86,7 +86,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.items.length).to.be(27); + expect(res.body.items.length).to.be(29); }); it('should throw an error if the archive is zip but content type is gzip', async function () { diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.0.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.0.0.zip deleted file mode 100644 index 77961d6b14c53..0000000000000 Binary files a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.0.0.zip and /dev/null differ diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.2.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.2.0.zip deleted file mode 100644 index a41061b7212ae..0000000000000 Binary files a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.2.0.zip and /dev/null differ diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/nginx-1.1.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/nginx-1.1.0.zip new file mode 100644 index 0000000000000..8beb16d00edb5 Binary files /dev/null and b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/nginx-1.1.0.zip differ diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/nginx-1.2.1.zip b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/nginx-1.2.1.zip new file mode 100644 index 0000000000000..977cd4318a384 Binary files /dev/null and b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/nginx-1.2.1.zip differ diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index 8c91b5ce8c279..60b57dc400cce 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -15,7 +15,9 @@ import { defineDockerServersConfig } from '@kbn/test'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:8b4ce36ecdf86e6cfdf781d9df8d564a014add9afc9aec21cf2c5a68ff82d3ab'; + 'docker.elastic.co/package-registry/distribution@sha256:b3dfc6a11ff7dce82ba8689ea9eeb54e353c6b4bfd2d28127b20ef72fd8883e9'; + +export const BUNDLED_PACKAGE_DIR = '/tmp/fleet_bundled_packages'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); @@ -63,6 +65,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `--xpack.fleet.packages.0.name=endpoint`, `--xpack.fleet.packages.0.version=latest`, ...(registryPort ? [`--xpack.fleet.registryUrl=http://localhost:${registryPort}`] : []), + `--xpack.fleet.developer.bundledPackageLocation=${BUNDLED_PACKAGE_DIR}`, ], }, }; diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 360caeeda2868..71c82b05961f0 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -15,7 +15,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:8b4ce36ecdf86e6cfdf781d9df8d564a014add9afc9aec21cf2c5a68ff82d3ab'; + 'docker.elastic.co/package-registry/distribution@sha256:b3dfc6a11ff7dce82ba8689ea9eeb54e353c6b4bfd2d28127b20ef72fd8883e9'; // the default export of config files must be a config provider // that returns an object with the projects config values diff --git a/x-pack/test/functional_synthetics/config.js b/x-pack/test/functional_synthetics/config.js index 35ff84fffc06b..58a6f85196ba0 100644 --- a/x-pack/test/functional_synthetics/config.js +++ b/x-pack/test/functional_synthetics/config.js @@ -17,7 +17,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry that updates Synthetics. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:8b4ce36ecdf86e6cfdf781d9df8d564a014add9afc9aec21cf2c5a68ff82d3ab'; + 'docker.elastic.co/package-registry/distribution@sha256:b3dfc6a11ff7dce82ba8689ea9eeb54e353c6b4bfd2d28127b20ef72fd8883e9'; // the default export of config files must be a config provider // that returns an object with the projects config values