[Fleet] Fix transforms with new specs not reinstalled when version is…

… same (#155453)

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/fleet/server/services/api_keys/transform_api_keys.ts

@@ -96,7 +96,6 @@ export async function generateTransformSecondaryAuthHeaders({
         role_descriptors: {},
       }
     );
-
     logger.debug(`Created api_key name: ${name}`);
     let encodedApiKey: TransformAPIKey['encoded'] | null = null;
 

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts

@@ -146,6 +146,7 @@ const processTransformAssetsPerModule = (
   installNameSuffix: string,
   transformPaths: string[],
   previousInstalledTransformEsAssets: EsAssetReference[] = [],
+  force?: boolean,
   username?: string
 ) => {
   const transformsSpecifications = new Map();
@@ -244,8 +245,14 @@ const processTransformAssetsPerModule = (
         `default-${transformVersion}`
       );
 
-      const currentTransformSameAsPrev =
-        previousInstalledTransformEsAssets.find((t) => t.id === installationName) !== undefined;
+      // Here, we track if fleet_transform_version (not package version) has changed based on installation name
+      // if version has changed, install transform and update es assets
+      // else, don't delete the dest index and install transform as it can be an expensive operation
+      const matchingTransformFromPrevInstall = previousInstalledTransformEsAssets.find(
+        (t) => t.id === installationName
+      );
+
+      const currentTransformSameAsPrev = matchingTransformFromPrevInstall !== undefined;
       if (previousInstalledTransformEsAssets.length === 0) {
         aliasesRefs.push(...aliasNames);
         transforms.push({
@@ -258,30 +265,36 @@ const processTransformAssetsPerModule = (
         });
         transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', true);
       } else {
-        if (!currentTransformSameAsPrev) {
-          // If upgrading from old json schema to new yml schema
-          // We need to make sure to delete those transforms by matching the legacy naming convention
-          const versionFromOldJsonSchema = previousInstalledTransformEsAssets.find((t) =>
-            t.id.startsWith(
-              getLegacyTransformNameForInstallation(
-                installablePackage,
-                `${transformModuleId}/default.json`
+        if (force || !currentTransformSameAsPrev) {
+          // If we are reinstalling the package (i.e. force = true),
+          // force delete old transforms so we can reinstall the same transforms again
+          if (force && matchingTransformFromPrevInstall) {
+            transformsToRemoveWithDestIndex.push(matchingTransformFromPrevInstall);
+          } else {
+            // If upgrading from old json schema to new yml schema
+            // We need to make sure to delete those transforms by matching the legacy naming convention
+            const versionFromOldJsonSchema = previousInstalledTransformEsAssets.find((t) =>
+              t.id.startsWith(
+                getLegacyTransformNameForInstallation(
+                  installablePackage,
+                  `${transformModuleId}/default.json`
+                )
               )
-            )
-          );
-
-          if (versionFromOldJsonSchema !== undefined) {
-            transformsToRemoveWithDestIndex.push(versionFromOldJsonSchema);
-          }
-
-          // If upgrading from yml to newer version of yaml
-          // Match using new naming convention
-          const installNameWithoutVersion = installationName.split(transformVersion)[0];
-          const prevVersion = previousInstalledTransformEsAssets.find((t) =>
-            t.id.startsWith(installNameWithoutVersion)
-          );
-          if (prevVersion !== undefined) {
-            transformsToRemove.push(prevVersion);
+            );
+
+            if (versionFromOldJsonSchema !== undefined) {
+              transformsToRemoveWithDestIndex.push(versionFromOldJsonSchema);
+            }
+
+            // If upgrading from yml to newer version of yaml
+            // Match using new naming convention
+            const installNameWithoutVersion = installationName.split(transformVersion)[0];
+            const prevVersion = previousInstalledTransformEsAssets.find((t) =>
+              t.id.startsWith(installNameWithoutVersion)
+            );
+            if (prevVersion !== undefined) {
+              transformsToRemove.push(prevVersion);
+            }
           }
           transforms.push({
             transformModuleId,
@@ -387,6 +400,7 @@ const installTransformsAssets = async (
   logger: Logger,
   esReferences: EsAssetReference[] = [],
   previousInstalledTransformEsAssets: EsAssetReference[] = [],
+  force?: boolean,
   authorizationHeader?: HTTPAuthorizationHeader | null
 ) => {
   let installedTransforms: EsAssetReference[] = [];
@@ -408,20 +422,24 @@ const installTransformsAssets = async (
       installNameSuffix,
       transformPaths,
       previousInstalledTransformEsAssets,
+      force,
       username
     );
 
     // By default, for internal Elastic packages that touch system indices, we want to run as internal user
     // so we set runAsKibanaSystem: true by default (e.g. when run_as_kibana_system set to true/not defined in yml file).
     // If package should be installed as the logged in user, set run_as_kibana_system: false,
-    // and pass es-secondary-authorization in header when creating the transforms.
-    const secondaryAuth = await generateTransformSecondaryAuthHeaders({
-      authorizationHeader,
-      logger,
-      pkgName: installablePackage.name,
-      pkgVersion: installablePackage.version,
-      username,
-    });
+    // generate api key, and pass es-secondary-authorization in header when creating the transforms.
+    const secondaryAuth = transforms.some((t) => t.runAsKibanaSystem === false)
+      ? await generateTransformSecondaryAuthHeaders({
+          authorizationHeader,
+          logger,
+          pkgName: installablePackage.name,
+          pkgVersion: installablePackage.version,
+          username,
+        })
+      : // No need to generate api key/secondary auth if all transforms are run as kibana_system user
+        undefined;
 
     // delete all previous transform
     await Promise.all([
@@ -567,15 +585,34 @@ const installTransformsAssets = async (
   return { installedTransforms, esReferences };
 };
 
-export const installTransforms = async (
-  installablePackage: InstallablePackage,
-  paths: string[],
-  esClient: ElasticsearchClient,
-  savedObjectsClient: SavedObjectsClientContract,
-  logger: Logger,
-  esReferences?: EsAssetReference[],
-  authorizationHeader?: HTTPAuthorizationHeader | null
-) => {
+interface InstallTransformsParams {
+  installablePackage: InstallablePackage;
+  paths: string[];
+  esClient: ElasticsearchClient;
+  savedObjectsClient: SavedObjectsClientContract;
+  logger: Logger;
+  esReferences?: EsAssetReference[];
+  /**
+   * Force transforms to install again even though fleet_transform_version might be same
+   * Should be true when package is re-installing
+   */
+  force?: boolean;
+  /**
+   * Authorization header parsed from original Kibana request, used to generate API key from user
+   * to pass in secondary authorization info to transform
+   */
+  authorizationHeader?: HTTPAuthorizationHeader | null;
+}
+export const installTransforms = async ({
+  installablePackage,
+  paths,
+  esClient,
+  savedObjectsClient,
+  logger,
+  force,
+  esReferences,
+  authorizationHeader,
+}: InstallTransformsParams) => {
   const transformPaths = paths.filter((path) => isTransform(path));
 
   const installation = await getInstallation({
@@ -613,6 +650,7 @@ export const installTransforms = async (
     );
   }
 
+  // If package contains yml transform specifications
   return await installTransformsAssets(
     installablePackage,
     installNameSuffix,
@@ -622,6 +660,7 @@ export const installTransforms = async (
     logger,
     esReferences,
     previousInstalledTransformEsAssets,
+    force,
     authorizationHeader
   );
 };

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts

@@ -122,8 +122,8 @@ describe('test transform install with legacy schema', () => {
       ],
     });
 
-    await installTransforms(
-      {
+    await installTransforms({
+      installablePackage: {
         name: 'endpoint',
         version: '0.16.0-dev.0',
         data_streams: [
@@ -157,16 +157,16 @@ describe('test transform install with legacy schema', () => {
           },
         ],
       } as unknown as RegistryPackage,
-      [
+      paths: [
         'endpoint-0.16.0-dev.0/data_stream/policy/elasticsearch/ingest_pipeline/default.json',
         'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json',
         'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json',
       ],
       esClient,
       savedObjectsClient,
-      loggerMock.create(),
-      previousInstallation.installed_es
-    );
+      logger: loggerMock.create(),
+      esReferences: previousInstallation.installed_es,
+    });
 
     expect(esClient.transform.getTransform.mock.calls).toEqual([
       [
@@ -320,8 +320,8 @@ describe('test transform install with legacy schema', () => {
       } as unknown as SavedObject<Installation>)
     );
 
-    await installTransforms(
-      {
+    await installTransforms({
+      installablePackage: {
         name: 'endpoint',
         version: '0.16.0-dev.0',
         data_streams: [
@@ -341,12 +341,12 @@ describe('test transform install with legacy schema', () => {
           },
         ],
       } as unknown as RegistryPackage,
-      ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
+      paths: ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
       esClient,
       savedObjectsClient,
-      loggerMock.create(),
-      previousInstallation.installed_es
-    );
+      logger: loggerMock.create(),
+      esReferences: previousInstallation.installed_es,
+    });
 
     const meta = getESAssetMetadata({ packageName: 'endpoint' });
 
@@ -422,8 +422,8 @@ describe('test transform install with legacy schema', () => {
       ],
     });
 
-    await installTransforms(
-      {
+    await installTransforms({
+      installablePackage: {
         name: 'endpoint',
         version: '0.16.0-dev.0',
         data_streams: [
@@ -457,12 +457,12 @@ describe('test transform install with legacy schema', () => {
           },
         ],
       } as unknown as RegistryPackage,
-      [],
+      paths: [],
       esClient,
       savedObjectsClient,
-      loggerMock.create(),
-      previousInstallation.installed_es
-    );
+      logger: loggerMock.create(),
+      esReferences: previousInstallation.installed_es,
+    });
 
     expect(esClient.transform.getTransform.mock.calls).toEqual([
       [
@@ -556,8 +556,8 @@ describe('test transform install with legacy schema', () => {
       )
     );
 
-    await installTransforms(
-      {
+    await installTransforms({
+      installablePackage: {
         name: 'endpoint',
         version: '0.16.0-dev.0',
         data_streams: [
@@ -577,12 +577,12 @@ describe('test transform install with legacy schema', () => {
           },
         ],
       } as unknown as RegistryPackage,
-      ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
+      paths: ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
       esClient,
       savedObjectsClient,
-      loggerMock.create(),
-      previousInstallation.installed_es
-    );
+      logger: loggerMock.create(),
+      esReferences: previousInstallation.installed_es,
+    });
 
     const meta = getESAssetMetadata({ packageName: 'endpoint' });
 

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/reauthorize.ts

@@ -26,6 +26,7 @@ interface FleetTransformMetadata {
   managed_by?: string;
   installed_by?: string;
   last_authorized_by?: string;
+  run_as_kibana_system?: boolean;
   transformId: string;
 }
 
@@ -114,10 +115,14 @@ export async function handleTransformReauthorizeAndStart({
       )
     )
   );
-  const transformsMetadata: FleetTransformMetadata[] = transformInfos.flat().map((t) => {
-    const transform = t.transforms?.[0];
-    return { ...transform._meta, transformId: transform?.id };
-  });
+
+  const transformsMetadata: FleetTransformMetadata[] = transformInfos
+    .flat()
+    .map<FleetTransformMetadata>((t) => {
+      const transform = t.transforms?.[0];
+      return { ...transform._meta, transformId: transform?.id };
+    })
+    .filter((t) => t?.run_as_kibana_system === false);
 
   const shouldInstallSequentially =
     uniqBy(transformsMetadata, 'order').length === transforms.length;