[Security Solution][Detections] Fix fetching package info from regist…

…ry for installed integrations (#134732) (#134784) **Fixes:** #134639 ## Summary In Cloud, `Elastic APM` and `Fleet Server` integrations are installed by default. However, attempts to fetch their packages from Elastic Package Registry via Fleet services on the server-side fail with the following errors: ```json { "message": "[email protected] not found", "status_code": 500 } ``` ```json { "message": "[email protected] not found", "status_code": 500 } ``` <img width="797" alt="Screenshot 2022-06-20 at 11 28 18" src="https://user-images.githubusercontent.com/7359339/174571610-4c24e777-c49a-49e0-addf-54c6301cc8ca.png"> This behavior happens in some Cloud environments (like the one in the related ticket). It seems to not happen in Cloud CI environments and locally. This PR adds error handling for this edge case to `GET /internal/detection_engine/fleet/integrations/installed?packages=` endpoint. - It logs fetching errors to the console logs of Kibana. - It uses a "best-effort" approach for returning data from the endpoint. If we could successfully read existing integration policies, we already have all of the needed data except correct integration titles. So, if after that any request to EPR results in an error, we: - Still return 200 with a list of installed integrations - Include correct titles for those packages that were successfully fetched - Include "best guess" titles for those packages that failed ``` [2022-06-20T12:57:10.270+02:00][ERROR][plugins.securitySolution] Error fetching package info from registry for [email protected]. Boom! [2022-06-20T12:57:10.270+02:00][ERROR][plugins.securitySolution] Error fetching package info from registry for [email protected]. Boom! ``` <img width="1085" alt="Screenshot 2022-06-20 at 13 05 08" src="https://user-images.githubusercontent.com/7359339/174588468-d28c1383-3a25-4f16-8905-bad3ca73e63e.png"> ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios (cherry picked from commit cdcb272) Co-authored-by: Georgii Gorbachev <[email protected]>
elastic · Jun 20, 2022 · a973ba8 · a973ba8
1 parent bc1c8f4
commit a973ba8
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 16 deletions.
diff --git a/...ection_engine/routes/fleet/get_installed_integrations/get_installed_integrations_route.ts b/...ection_engine/routes/fleet/get_installed_integrations/get_installed_integrations_route.ts
@@ -5,18 +5,25 @@
  * 2.0.
  */
 
+import { Logger } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';
-
+import { initPromisePool } from '../../../../../utils/promise_pool';
+import { buildSiemResponse } from '../../utils';
 import type { SecuritySolutionPluginRouter } from '../../../../../types';
+
 import { DETECTION_ENGINE_INSTALLED_INTEGRATIONS_URL } from '../../../../../../common/constants';
 import { GetInstalledIntegrationsResponse } from '../../../../../../common/detection_engine/schemas/response/get_installed_integrations_response_schema';
-import { buildSiemResponse } from '../../utils';
 import { createInstalledIntegrationSet } from './installed_integration_set';
 
+const MAX_CONCURRENT_REQUESTS_TO_PACKAGE_REGISTRY = 5;
+
 /**
  * Returns an array of installed Fleet integrations and their packages.
  */
-export const getInstalledIntegrationsRoute = (router: SecuritySolutionPluginRouter) => {
+export const getInstalledIntegrationsRoute = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger
+) => {
   router.get(
     {
       path: DETECTION_ENGINE_INSTALLED_INTEGRATIONS_URL,
@@ -39,17 +46,40 @@ export const getInstalledIntegrationsRoute = (router: SecuritySolutionPluginRout
           set.addPackagePolicy(policy);
         });
 
-        const registryPackages = await Promise.all(
-          set.getPackages().map((packageInfo) => {
-            return fleet.packages.getRegistryPackage(
+        const registryPackages = await initPromisePool({
+          concurrency: MAX_CONCURRENT_REQUESTS_TO_PACKAGE_REGISTRY,
+          items: set.getPackages(),
+          executor: async (packageInfo) => {
+            const registryPackage = await fleet.packages.getRegistryPackage(
               packageInfo.package_name,
               packageInfo.package_version
             );
-          })
-        );
+            return registryPackage;
+          },
+        });
+
+        if (registryPackages.errors.length > 0) {
+          const errors = registryPackages.errors.map(({ error, item }) => {
+            return {
+              error,
+              packageId: `${item.package_name}@${item.package_version}`,
+            };
+          });
+
+          const packages = errors.map((e) => e.packageId).join(', ');
+          logger.error(
+            `Unable to retrieve installed integrations. Error fetching packages from registry: ${packages}.`
+          );
+
+          errors.forEach(({ error, packageId }) => {
+            const logMessage = `Error fetching package info from registry for ${packageId}`;
+            const logReason = error instanceof Error ? error.message : String(error);
+            logger.debug(`${logMessage}. ${logReason}`);
+          });
+        }
 
-        registryPackages.forEach((registryPackage) => {
-          set.addRegistryPackage(registryPackage.packageInfo);
+        registryPackages.results.forEach(({ result }) => {
+          set.addRegistryPackage(result.packageInfo);
         });
 
         const installedIntegrations = set.getIntegrations();

diff --git a/...lib/detection_engine/routes/fleet/get_installed_integrations/installed_integration_set.ts b/...lib/detection_engine/routes/fleet/get_installed_integrations/installed_integration_set.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { flatten } from 'lodash';
+import { capitalize, flatten } from 'lodash';
 import { PackagePolicy, RegistryPackage } from '@kbn/fleet-plugin/common';
 import {
   InstalledIntegration,
@@ -35,7 +35,7 @@ export const createInstalledIntegrationSet = (): IInstalledIntegrationSet => {
 
   const addPackagePolicy = (policy: PackagePolicy): void => {
     const packageInfo = getPackageInfoFromPolicy(policy);
-    const integrationsInfo = getIntegrationsInfoFromPolicy(policy);
+    const integrationsInfo = getIntegrationsInfoFromPolicy(policy, packageInfo);
     const packageKey = `${packageInfo.package_name}:${packageInfo.package_version}`;
     const existingPackageInfo = packageMap.get(packageKey);
 
@@ -121,11 +121,16 @@ const getPackageInfoFromPolicy = (policy: PackagePolicy): InstalledPackageBasicI
   };
 };
 
-const getIntegrationsInfoFromPolicy = (policy: PackagePolicy): InstalledIntegrationBasicInfo[] => {
+const getIntegrationsInfoFromPolicy = (
+  policy: PackagePolicy,
+  packageInfo: InstalledPackageBasicInfo
+): InstalledIntegrationBasicInfo[] => {
   return policy.inputs.map((input) => {
+    const integrationName = normalizeString(input.policy_template); // e.g. 'cloudtrail'
+    const integrationTitle = `${packageInfo.package_title} ${capitalize(integrationName)}`; // e.g. 'AWS Cloudtrail'
     return {
-      integration_name: normalizeString(input.policy_template),
-      integration_title: '', // this gets initialized later in addRegistryPackage()
+      integration_name: integrationName,
+      integration_title: integrationTitle, // title gets re-initialized later in addRegistryPackage()
       is_enabled: input.enabled,
     };
   });

diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts
@@ -119,7 +119,7 @@ export const initRoutes = (
 
   getRuleExecutionEventsRoute(router);
 
-  getInstalledIntegrationsRoute(router);
+  getInstalledIntegrationsRoute(router, logger);
 
   createTimelinesRoute(router, config, security);
   patchTimelinesRoute(router, config, security);