[Ingest Manager] Support limiting integrations on an agent config (#7…

…0542)

* Add API endpoint and hook for retrieving restricted packages

* Filter out restricted packages already in use from list of integrations available for an agent config

* Allow list agent configs to optionally return expanded package configs, re

* Filter out agent configs which already use the restricted package already from list of agent configs available for an integration

* Allow more than 20 agent configs to be shown

* Rename restricted to limited; add some common methods to DRY

* Add limited package check on server side

* Adjust copy wording

* Fix typings

* Add some package config api integration tests, update es archive mappings

* Move test to dockerized integation tests directory; move existing epm tests to their own directory

* Remove extra assignPackageConfigs() - already handled in packageConfigService.create()

* Review fixes

* Fix type, reenabled skipped test

* Move new EPM integration test file

x-pack/plugins/ingest_manager/common/constants/routes.ts

@@ -17,6 +17,7 @@ const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`;
 const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`;
 export const EPM_API_ROUTES = {
   LIST_PATTERN: EPM_PACKAGES_MANY,
+  LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`,
   INFO_PATTERN: EPM_PACKAGES_ONE,
   INSTALL_PATTERN: EPM_PACKAGES_ONE,
   DELETE_PATTERN: EPM_PACKAGES_ONE,

x-pack/plugins/ingest_manager/common/services/index.ts

@@ -3,11 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import * as AgentStatusKueryHelper from './agent_status';
-
 export * from './routes';
+export * as AgentStatusKueryHelper from './agent_status';
 export { packageToPackageConfigInputs, packageToPackageConfig } from './package_to_config';
 export { storedPackageConfigsToAgentInputs } from './package_configs_to_agent_inputs';
 export { configToYaml } from './config_to_yaml';
-export { AgentStatusKueryHelper };
+export { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from './limited_package';
 export { decodeCloudId } from './decode_cloud_id';

x-pack/plugins/ingest_manager/common/services/limited_package.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { PackageInfo, AgentConfig, PackageConfig } from '../types';
+
+// Assume packages only ever include 1 config template for now
+export const isPackageLimited = (packageInfo: PackageInfo): boolean => {
+  return packageInfo.config_templates?.[0]?.multiple === false;
+};
+
+export const doesAgentConfigAlreadyIncludePackage = (
+  agentConfig: AgentConfig,
+  packageName: string
+): boolean => {
+  if (agentConfig.package_configs.length && typeof agentConfig.package_configs[0] === 'string') {
+    throw new Error('Unable to read full package config information');
+  }
+  return (agentConfig.package_configs as PackageConfig[])
+    .map((packageConfig) => packageConfig.package?.name || '')
+    .includes(packageName);
+};

x-pack/plugins/ingest_manager/common/services/routes.ts

@@ -27,6 +27,10 @@ export const epmRouteService = {
     return EPM_API_ROUTES.LIST_PATTERN;
   },
 
+  getListLimitedPath: () => {
+    return EPM_API_ROUTES.LIMITED_LIST_PATTERN;
+  },
+
   getInfoPath: (pkgkey: string) => {
     return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey);
   },

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -79,6 +79,7 @@ export interface RegistryConfigTemplate {
   title: string;
   description: string;
   inputs: RegistryInput[];
+  multiple?: boolean;
 }
 
 export interface RegistryInput {

x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts

@@ -7,7 +7,9 @@ import { AgentConfig, NewAgentConfig, FullAgentConfig } from '../models';
 import { ListWithKuery } from './common';
 
 export interface GetAgentConfigsRequest {
-  query: ListWithKuery;
+  query: ListWithKuery & {
+    full?: boolean;
+  };
 }
 
 export type GetAgentConfigsResponseItem = AgentConfig & { agents?: number };

x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts

@@ -3,8 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { HttpFetchQuery } from 'src/core/public';
 
-export interface ListWithKuery {
+export interface ListWithKuery extends HttpFetchQuery {
   page?: number;
   perPage?: number;
   sortField?: string;

x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts

@@ -34,6 +34,11 @@ export interface GetPackagesResponse {
   success: boolean;
 }
 
+export interface GetLimitedPackagesResponse {
+  response: string[];
+  success: boolean;
+}
+
 export interface GetFileRequest {
   params: {
     pkgkey: string;

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts

@@ -3,7 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { HttpFetchQuery } from 'src/core/public';
 import {
   useRequest,
   sendRequest,
@@ -12,6 +11,7 @@ import {
 } from './use_request';
 import { agentConfigRouteService } from '../../services';
 import {
+  GetAgentConfigsRequest,
   GetAgentConfigsResponse,
   GetOneAgentConfigResponse,
   GetFullAgentConfigResponse,
@@ -25,7 +25,7 @@ import {
   DeleteAgentConfigResponse,
 } from '../../types';
 
-export const useGetAgentConfigs = (query: HttpFetchQuery = {}) => {
+export const useGetAgentConfigs = (query?: GetAgentConfigsRequest['query']) => {
   return useRequest<GetAgentConfigsResponse>({
     path: agentConfigRouteService.getListPath(),
     method: 'get',

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts

@@ -10,6 +10,7 @@ import { epmRouteService } from '../../services';
 import {
   GetCategoriesResponse,
   GetPackagesResponse,
+  GetLimitedPackagesResponse,
   GetInfoResponse,
   InstallPackageResponse,
   DeletePackageResponse,
@@ -30,6 +31,13 @@ export const useGetPackages = (query: HttpFetchQuery = {}) => {
   });
 };
 
+export const useGetLimitedPackages = () => {
+  return useRequest<GetLimitedPackagesResponse>({
+    path: epmRouteService.getListLimitedPath(),
+    method: 'get',
+  });
+};
+
 export const useGetPackageInfoByKey = (pkgkey: string) => {
   return useRequest<GetInfoResponse>({
     path: epmRouteService.getInfoPath(pkgkey),

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx

@@ -9,6 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer, EuiTextColor } from '@elastic/eui';
 import { Error } from '../../../components';
 import { AgentConfig, PackageInfo, GetAgentConfigsResponseItem } from '../../../types';
+import { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from '../../../services';
 import { useGetPackageInfoByKey, useGetAgentConfigs, sendGetOneAgentConfig } from '../../../hooks';
 
 export const StepSelectConfig: React.FunctionComponent<{
@@ -24,7 +25,12 @@ export const StepSelectConfig: React.FunctionComponent<{
   const [selectedConfigError, setSelectedConfigError] = useState<Error>();
 
   // Fetch package info
-  const { data: packageInfoData, error: packageInfoError } = useGetPackageInfoByKey(pkgkey);
+  const {
+    data: packageInfoData,
+    error: packageInfoError,
+    isLoading: packageInfoLoading,
+  } = useGetPackageInfoByKey(pkgkey);
+  const isLimitedPackage = (packageInfoData && isPackageLimited(packageInfoData.response)) || false;
 
   // Fetch agent configs info
   const {
@@ -36,6 +42,7 @@ export const StepSelectConfig: React.FunctionComponent<{
     perPage: 1000,
     sortField: 'name',
     sortOrder: 'asc',
+    full: true,
   });
   const agentConfigs = agentConfigsData?.items || [];
   const agentConfigsById = agentConfigs.reduce(
@@ -112,12 +119,18 @@ export const StepSelectConfig: React.FunctionComponent<{
           searchable
           allowExclusions={false}
           singleSelection={true}
-          isLoading={isAgentConfigsLoading}
-          options={agentConfigs.map(({ id, name, description }) => {
+          isLoading={isAgentConfigsLoading || packageInfoLoading}
+          options={agentConfigs.map((agentConf) => {
+            const alreadyHasLimitedPackage =
+              (isLimitedPackage &&
+                packageInfoData &&
+                doesAgentConfigAlreadyIncludePackage(agentConf, packageInfoData.response.name)) ||
+              false;
             return {
-              label: name,
-              key: id,
-              checked: selectedConfigId === id ? 'on' : undefined,
+              label: agentConf.name,
+              key: agentConf.id,
+              checked: selectedConfigId === agentConf.id ? 'on' : undefined,
+              disabled: alreadyHasLimitedPackage,
               'data-test-subj': 'agentConfigItem',
             };
           })}

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx

@@ -8,8 +8,13 @@ import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer } from '@elastic/eui';
 import { Error } from '../../../components';
-import { AgentConfig, PackageInfo } from '../../../types';
-import { useGetOneAgentConfig, useGetPackages, sendGetPackageInfoByKey } from '../../../hooks';
+import { AgentConfig, PackageInfo, PackageConfig, GetPackagesResponse } from '../../../types';
+import {
+  useGetOneAgentConfig,
+  useGetPackages,
+  useGetLimitedPackages,
+  sendGetPackageInfoByKey,
+} from '../../../hooks';
 import { PackageIcon } from '../../../components/package_icon';
 
 export const StepSelectPackage: React.FunctionComponent<{
@@ -28,12 +33,27 @@ export const StepSelectPackage: React.FunctionComponent<{
   const { data: agentConfigData, error: agentConfigError } = useGetOneAgentConfig(agentConfigId);
 
   // Fetch packages info
+  // Filter out limited packages already part of selected agent config
+  const [packages, setPackages] = useState<GetPackagesResponse['response']>([]);
   const {
     data: packagesData,
     error: packagesError,
     isLoading: isPackagesLoading,
   } = useGetPackages();
-  const packages = packagesData?.response || [];
+  const {
+    data: limitedPackagesData,
+    isLoading: isLimitedPackagesLoading,
+  } = useGetLimitedPackages();
+  useEffect(() => {
+    if (packagesData?.response && limitedPackagesData?.response && agentConfigData?.item) {
+      const allPackages = packagesData.response;
+      const limitedPackages = limitedPackagesData.response;
+      const usedLimitedPackages = (agentConfigData.item.package_configs as PackageConfig[])
+        .map((packageConfig) => packageConfig.package?.name || '')
+        .filter((pkgName) => limitedPackages.includes(pkgName));
+      setPackages(allPackages.filter((pkg) => !usedLimitedPackages.includes(pkg.name)));
+    }
+  }, [packagesData, limitedPackagesData, agentConfigData]);
 
   // Update parent agent config state
   useEffect(() => {
@@ -101,7 +121,7 @@ export const StepSelectPackage: React.FunctionComponent<{
           searchable
           allowExclusions={false}
           singleSelection={true}
-          isLoading={isPackagesLoading}
+          isLoading={isPackagesLoading || isLimitedPackagesLoading}
           options={packages.map(({ title, name, version, icons }) => {
             const pkgkey = `${name}-${version}`;
             return {

x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts

@@ -7,6 +7,7 @@
 export { getFlattenedObject } from '../../../../../../../src/core/public';
 
 export {
+  AgentStatusKueryHelper,
   agentConfigRouteService,
   packageConfigRouteService,
   dataStreamRouteService,
@@ -21,5 +22,6 @@ export {
   packageToPackageConfigInputs,
   storedPackageConfigsToAgentInputs,
   configToYaml,
-  AgentStatusKueryHelper,
+  isPackageLimited,
+  doesAgentConfigAlreadyIncludePackage,
 } from '../../../../common';

x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts

@@ -24,6 +24,7 @@ export {
   // API schema - misc setup, status
   GetFleetStatusResponse,
   // API schemas - Agent Config
+  GetAgentConfigsRequest,
   GetAgentConfigsResponse,
   GetAgentConfigsResponseItem,
   GetOneAgentConfigResponse,
@@ -92,6 +93,7 @@ export {
   ServiceName,
   GetCategoriesResponse,
   GetPackagesResponse,
+  GetLimitedPackagesResponse,
   GetInfoResponse,
   InstallPackageResponse,
   DeletePackageResponse,

x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts

@@ -38,8 +38,12 @@ export const getAgentConfigsHandler: RequestHandler<
   TypeOf<typeof GetAgentConfigsRequestSchema.query>
 > = async (context, request, response) => {
   const soClient = context.core.savedObjects.client;
+  const { full: withPackageConfigs = false, ...restOfQuery } = request.query;
   try {
-    const { items, total, page, perPage } = await agentConfigService.list(soClient, request.query);
+    const { items, total, page, perPage } = await agentConfigService.list(soClient, {
+      withPackageConfigs,
+      ...restOfQuery,
+    });
     const body: GetAgentConfigsResponse = {
       items,
       total,
@@ -103,6 +107,7 @@ export const createAgentConfigHandler: RequestHandler<
   TypeOf<typeof CreateAgentConfigRequestSchema.body>
 > = async (context, request, response) => {
   const soClient = context.core.savedObjects.client;
+  const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
   const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
   const withSysMonitoring = request.query.sys_monitoring ?? false;
   try {
@@ -128,15 +133,9 @@ export const createAgentConfigHandler: RequestHandler<
     if (withSysMonitoring && newSysPackageConfig !== undefined && agentConfig !== undefined) {
       newSysPackageConfig.config_id = agentConfig.id;
       newSysPackageConfig.namespace = agentConfig.namespace;
-      const sysPackageConfig = await packageConfigService.create(soClient, newSysPackageConfig, {
+      await packageConfigService.create(soClient, callCluster, newSysPackageConfig, {
         user,
       });
-
-      if (sysPackageConfig) {
-        agentConfig = await agentConfigService.assignPackageConfigs(soClient, agentConfig.id, [
-          sysPackageConfig.id,
-        ]);
-      }
     }
 
     const body: CreateAgentConfigResponse = {

x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts

@@ -5,27 +5,29 @@
  */
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server';
-import {
-  GetPackagesRequestSchema,
-  GetFileRequestSchema,
-  GetInfoRequestSchema,
-  InstallPackageRequestSchema,
-  DeletePackageRequestSchema,
-} from '../../types';
 import {
   GetInfoResponse,
   InstallPackageResponse,
   DeletePackageResponse,
   GetCategoriesResponse,
   GetPackagesResponse,
+  GetLimitedPackagesResponse,
 } from '../../../common';
+import {
+  GetPackagesRequestSchema,
+  GetFileRequestSchema,
+  GetInfoRequestSchema,
+  InstallPackageRequestSchema,
+  DeletePackageRequestSchema,
+} from '../../types';
 import {
   getCategories,
   getPackages,
   getFile,
   getPackageInfo,
   installPackage,
   removeInstallation,
+  getLimitedPackages,
 } from '../../services/epm/packages';
 
 export const getCategoriesHandler: RequestHandler = async (context, request, response) => {
@@ -69,6 +71,25 @@ export const getListHandler: RequestHandler<
   }
 };
 
+export const getLimitedListHandler: RequestHandler = async (context, request, response) => {
+  try {
+    const savedObjectsClient = context.core.savedObjects.client;
+    const res = await getLimitedPackages({ savedObjectsClient });
+    const body: GetLimitedPackagesResponse = {
+      response: res,
+      success: true,
+    };
+    return response.ok({
+      body,
+    });
+  } catch (e) {
+    return response.customError({
+      statusCode: 500,
+      body: { message: e.message },
+    });
+  }
+};
+
 export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.params>> = async (
   context,
   request,