[Fleet] Add support for forcing stack aligned versions on bundled packages

fleet_packages.json

@@ -4,6 +4,12 @@
   on disk rather than only in the configured package registry service. This allows Kibana to install
   "stack-aligned" packages or those that are installed by default in an airgapped or offline environment.
 
+  The `forceAlignStackVersion` option is available for packages who wish to opt into having their
+  versions forcibly rewritten to the current version of Kibana. This is useful for packages that
+  deploy multiple, version-aligned stack components like APM. When this option is enabled, Kibana
+  will fetch the latest available version of the package from EPR (including prerelease versions),
+  download that version, and rewrite its version to align with Kibana's.
+
   Packages will be fetched from https://epr-snapshot.elastic.co by default. This can be overridden
   via the `--epr-registry=production` command line argument when building Kibana. Fetching from the
   snapshot registry allows Kibana to bundle packages that have yet to be published to production in
@@ -14,7 +20,8 @@
 [
   {
     "name": "apm",
-    "version": "8.4.0"
+    "version": "8.4.0",
+    "forceAlignStackVersion": true
   },
   {
     "name": "elastic_agent",

package.json

@@ -613,7 +613,8 @@
     "xml2js": "^0.4.22",
     "xterm": "^4.18.0",
     "xterm-addon-fit": "^0.5.0",
-    "yauzl": "^2.10.0"
+    "yauzl": "^2.10.0",
+    "yazl": "^2.5.1"
   },
   "devDependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
@@ -1164,6 +1165,7 @@
     "@types/xml2js": "^0.4.5",
     "@types/yargs": "^15.0.0",
     "@types/yauzl": "^2.9.1",
+    "@types/yazl": "^2.4.2",
     "@types/zen-observable": "^0.8.0",
     "@typescript-eslint/eslint-plugin": "^5.20.0",
     "@typescript-eslint/parser": "^5.20.0",

src/dev/build/lib/archive_utils.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import fs from 'fs';
+import yauzl from 'yauzl';
+import yazl from 'yazl';
+
+// The utilities in this file are largely copied with minor modifications from
+// `x-pack/plugins/fleet/server/services/epm/extract.ts`. We can't import them directly
+// in the bundled package task due to tsconfig limitation, so they're copied here instead.
+
+export interface ZipEntry {
+  path: string;
+  buffer?: Buffer;
+}
+
+export async function unzipBuffer(buffer: Buffer): Promise<ZipEntry[]> {
+  const zipEntries: ZipEntry[] = [];
+  const zipfile = await yauzlFromBuffer(buffer, { lazyEntries: true });
+
+  zipfile.readEntry();
+  zipfile.on('entry', async (entry: yauzl.Entry) => {
+    const path = entry.fileName;
+
+    // Only include files, not directories
+    if (path.endsWith('/')) {
+      return zipfile.readEntry();
+    }
+
+    const entryBuffer = await getZipReadStream(zipfile, entry).then(streamToBuffer);
+    zipEntries.push({ buffer: entryBuffer, path });
+
+    zipfile.readEntry();
+  });
+
+  await new Promise((resolve, reject) => zipfile.on('end', resolve).on('error', reject));
+
+  return zipEntries;
+}
+
+export async function createZipFile(entries: ZipEntry[], destination: string): Promise<Buffer> {
+  const zipfile = new yazl.ZipFile();
+
+  for (const entry of entries) {
+    zipfile.addBuffer(entry.buffer || Buffer.from(''), entry.path);
+  }
+
+  return new Promise((resolve, reject) => {
+    zipfile.outputStream.on('error', reject);
+
+    zipfile.end();
+
+    zipfile.outputStream
+      .pipe(fs.createWriteStream(destination))
+      .on('close', resolve)
+      .on('error', reject);
+  });
+}
+
+// Copied over some utilities from x-pack/plugins/fleet/server/services/epm/archive/extract.ts since we can't
+// import them directly due to `tsconfig` limitations in the `kibana/src/` directory.
+function yauzlFromBuffer(buffer: Buffer, opts: yauzl.Options): Promise<yauzl.ZipFile> {
+  return new Promise((resolve, reject) =>
+    yauzl.fromBuffer(buffer, opts, (err?: Error, handle?: yauzl.ZipFile) =>
+      err ? reject(err) : resolve(handle!)
+    )
+  );
+}
+
+function getZipReadStream(
+  zipfile: yauzl.ZipFile,
+  entry: yauzl.Entry
+): Promise<NodeJS.ReadableStream> {
+  return new Promise((resolve, reject) =>
+    zipfile.openReadStream(entry, (err?: Error, readStream?: NodeJS.ReadableStream) =>
+      err ? reject(err) : resolve(readStream!)
+    )
+  );
+}
+
+function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+    stream.on('end', () => resolve(Buffer.concat(chunks)));
+    stream.on('error', reject);
+  });
+}

src/dev/build/lib/index.ts

@@ -17,3 +17,4 @@ export * from './scan_delete';
 export * from './scan_copy';
 export * from './platform';
 export * from './scan';
+export * from './archive_utils';

src/dev/build/tasks/bundle_fleet_packages.ts

@@ -7,15 +7,18 @@
  */
 
 import JSON5 from 'json5';
+import fs from 'fs/promises';
+import { safeLoad, safeDump } from 'js-yaml';
 
 import { readCliArgs } from '../args';
-import { Task, read, downloadToDisk } from '../lib';
+import { Task, read, downloadToDisk, unzipBuffer, createZipFile } from '../lib';
 
 const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/target/bundled_packages';
 
 interface FleetPackage {
   name: string;
   version: string;
+  forceAlignStackVersion?: boolean;
 }
 
 export const BundleFleetPackages: Task = {
@@ -25,8 +28,7 @@ export const BundleFleetPackages: Task = {
     log.info('Fetching fleet packages from package registry');
     log.indent(4);
 
-    // Support the `--use-snapshot-epr` command line argument to fetch from the snapshot registry
-    // in development or test environments
+    // Support the `--epr-registry` command line argument to fetch from the snapshot or production registry
     const { buildOptions } = readCliArgs(process.argv);
     const eprUrl =
       buildOptions?.eprRegistry === 'snapshot'
@@ -40,13 +42,28 @@ export const BundleFleetPackages: Task = {
 
     log.debug(
       `Found configured bundled packages: ${parsedFleetPackages
-        .map((fleetPackage) => `${fleetPackage.name}-${fleetPackage.version}`)
+        .map((fleetPackage) => `${fleetPackage.name}-${fleetPackage.version || 'latest'}`)
         .join(', ')}`
     );
 
     await Promise.all(
       parsedFleetPackages.map(async (fleetPackage) => {
-        const archivePath = `${fleetPackage.name}-${fleetPackage.version}.zip`;
+        const stackVersion = config.getBuildVersion();
+
+        let versionToWrite = fleetPackage.version;
+
+        // If `forceAlignStackVersion` is set, we will rewrite the version specified in the config
+        // to the version of the stack when writing the bundled package to disk. This allows us
+        // to support some unique package development workflows, e.g. APM.
+        if (fleetPackage.forceAlignStackVersion) {
+          versionToWrite = stackVersion;
+
+          log.debug(
+            `Bundling ${fleetPackage.name}-${fleetPackage.version} as ${fleetPackage.name}-${stackVersion} to align with stack version`
+          );
+        }
+
+        const archivePath = `${fleetPackage.name}-${versionToWrite}.zip`;
         const archiveUrl = `${eprUrl}/epr/${fleetPackage.name}/${fleetPackage.name}-${fleetPackage.version}.zip`;
 
         const destination = build.resolvePath(BUNDLED_PACKAGES_DIR, archivePath);
@@ -61,9 +78,42 @@ export const BundleFleetPackages: Task = {
             skipChecksumCheck: true,
             maxAttempts: 3,
           });
+
+          // If we're force aligning the version, we need to
+          // 1. Unzip the downloaded archive
+          // 2. Edit the `manifest.yml` file to include the updated `version` value
+          // 3. Re-zip the archive and replace it on disk
+          if (fleetPackage.forceAlignStackVersion) {
+            const buffer = await fs.readFile(destination);
+            const zipEntries = await unzipBuffer(buffer);
+
+            const manifestPath = `${fleetPackage.name}-${fleetPackage.version}/manifest.yml`;
+            const manifestEntry = zipEntries.find((entry) => entry.path === manifestPath);
+
+            if (!manifestEntry || !manifestEntry.buffer) {
+              log.debug(
+                `Unable to find manifest.yml for stack aligned package ${fleetPackage.name}`
+              );
+
+              return;
+            }
+
+            const manifestYml = await safeLoad(manifestEntry.buffer.toString('utf8'));
+            manifestYml.version = stackVersion;
+
+            const newManifestYml = safeDump(manifestYml);
+            manifestEntry.buffer = Buffer.from(newManifestYml, 'utf8');
+
+            // Update all paths to use the new version
+            zipEntries.forEach(
+              (entry) => (entry.path = entry.path.replace(fleetPackage.version, versionToWrite!))
+            );
+
+            await createZipFile(zipEntries, destination);
+          }
         } catch (error) {
-          log.warning(`Failed to download bundled package archive ${archivePath}`);
-          log.warning(error);
+          log.error(`Failed to download bundled package archive ${archivePath}`);
+          throw error;
         }
       })
     );

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -241,6 +241,7 @@ interface InstallUploadedArchiveParams {
   archiveBuffer: Buffer;
   contentType: string;
   spaceId: string;
+  version?: string;
 }
 
 function getTelemetryEvent(pkgName: string, pkgVersion: string): PackageUpdateEvent {
@@ -437,6 +438,7 @@ async function installPackageByUpload({
   archiveBuffer,
   contentType,
   spaceId,
+  version,
 }: InstallUploadedArchiveParams): Promise<InstallResult> {
   // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611
   await Promise.resolve();
@@ -449,21 +451,27 @@ async function installPackageByUpload({
   try {
     const { packageInfo } = await generatePackageInfoFromArchiveBuffer(archiveBuffer, contentType);
 
+    // Allow for overriding the version in the manifest for cases where we install
+    // stack-aligned bundled packages. In general, the version pulled from a bundled package
+    // filename and the version in the manifest will match, except for special cases around the
+    // `forceAlignStackVersion` flag in `fleet_packages.json`.
+    const pkgVersion = version || packageInfo.version;
+
     const installedPkg = await getInstallationObject({
       savedObjectsClient,
       pkgName: packageInfo.name,
     });
 
-    installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg });
+    installType = getInstallType({ pkgVersion, installedPkg });
 
     span?.addLabels({
       packageName: packageInfo.name,
-      packageVersion: packageInfo.version,
+      packageVersion: pkgVersion,
       installType,
     });
 
     telemetryEvent.packageName = packageInfo.name;
-    telemetryEvent.newVersion = packageInfo.version;
+    telemetryEvent.newVersion = pkgVersion;
     telemetryEvent.installType = installType;
     telemetryEvent.currentVersion = installedPkg?.attributes.version || 'not_installed';
 
@@ -472,14 +480,14 @@ async function installPackageByUpload({
     deleteVerificationResult(packageInfo);
     const paths = await unpackBufferToCache({
       name: packageInfo.name,
-      version: packageInfo.version,
+      version: pkgVersion,
       archiveBuffer,
       contentType,
     });
 
     setPackageInfo({
       name: packageInfo.name,
-      version: packageInfo.version,
+      version: pkgVersion,
       packageInfo,
     });
 
@@ -505,7 +513,7 @@ async function installPackageByUpload({
       logger,
       installedPkg,
       paths,
-      packageInfo,
+      packageInfo: { ...packageInfo, version: pkgVersion },
       installType,
       installSource,
       spaceId,
@@ -572,6 +580,7 @@ export async function installPackage(args: InstallPackageParams): Promise<Instal
         archiveBuffer: matchingBundledPackage.buffer,
         contentType: 'application/zip',
         spaceId,
+        version: matchingBundledPackage.version,
       });
 
       return { ...response, installSource: 'bundled' };

yarn.lock

@@ -8738,6 +8738,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/yazl@^2.4.2":
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/@types/yazl/-/yazl-2.4.2.tgz#d5f8a4752261badbf1a36e8b49e042dc18ec84bc"
+  integrity sha512-T+9JH8O2guEjXNxqmybzQ92mJUh2oCwDDMSSimZSe1P+pceZiFROZLYmcbqkzV5EUwz6VwcKXCO2S2yUpra6XQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/zen-observable@^0.8.0":
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/@types/zen-observable/-/zen-observable-0.8.0.tgz#8b63ab7f1aa5321248aad5ac890a485656dcea4d"