[Fleet] Add preconfiguration to kibana config

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -200,6 +200,8 @@ kibana_vars=(
     xpack.fleet.agents.elasticsearch.host
     xpack.fleet.agents.kibana.host
     xpack.fleet.agents.tlsCheckDisabled
+    xpack.fleet.agentPolicies
+    xpack.fleet.packages
     xpack.fleet.registryUrl
     xpack.graph.canEditDrillDownUrls
     xpack.graph.enabled

x-pack/plugins/fleet/common/constants/index.ts

@@ -15,6 +15,7 @@ export * from './epm';
 export * from './output';
 export * from './enrollment_api_key';
 export * from './settings';
+export * from './preconfiguration';
 
 // TODO: This is the default `index.max_result_window` ES setting, which dictates
 // the maximum amount of results allowed to be returned from a search. It's possible

x-pack/plugins/fleet/common/constants/preconfiguration.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const PRECONFIGURATION_METADATA_INDEX = '.fleet-preconfiguration';

x-pack/plugins/fleet/common/types/index.ts

@@ -7,6 +7,7 @@
 
 export * from './models';
 export * from './rest_spec';
+import type { PreconfiguredAgentPolicy, PreconfiguredPackage } from './models/preconfiguration';
 
 export interface FleetConfigType {
   enabled: boolean;
@@ -32,6 +33,8 @@ export interface FleetConfigType {
     agentPolicyRolloutRateLimitIntervalMs: number;
     agentPolicyRolloutRateLimitRequestPerInterval: number;
   };
+  agentPolicies?: PreconfiguredAgentPolicy[];
+  packages?: PreconfiguredPackage[];
 }
 
 // Calling Object.entries(PackagesGroupedByStatus) gave `status: string`

x-pack/plugins/fleet/server/constants/index.ts

@@ -52,4 +52,5 @@ export {
   // Fleet Server index
   ENROLLMENT_API_KEYS_INDEX,
   AGENTS_INDEX,
+  PRECONFIGURATION_METADATA_INDEX,
 } from '../../common';

x-pack/plugins/fleet/server/index.ts

@@ -15,6 +15,8 @@ import {
   AGENT_POLLING_REQUEST_TIMEOUT_MS,
 } from '../common';
 
+import { PreconfiguredPackagesSchema, PreconfiguredAgentPoliciesSchema } from './types';
+
 import { FleetPlugin } from './plugin';
 
 export { default as apm } from 'elastic-apm-node';
@@ -77,6 +79,8 @@ export const config: PluginConfigDescriptor = {
         defaultValue: AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL,
       }),
     }),
+    packages: schema.maybe(PreconfiguredPackagesSchema),
+    agentPolicies: schema.maybe(PreconfiguredAgentPoliciesSchema),
   }),
 };
 

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -19,6 +19,7 @@ import {
   DEFAULT_AGENT_POLICY,
   AGENT_POLICY_SAVED_OBJECT_TYPE,
   AGENT_SAVED_OBJECT_TYPE,
+  PRECONFIGURATION_METADATA_INDEX,
 } from '../constants';
 import type {
   PackagePolicy,
@@ -150,7 +151,8 @@ class AgentPolicyService {
     config: PreconfiguredAgentPolicy
   ): Promise<{
     created: boolean;
-    policy: AgentPolicy;
+    policy?: AgentPolicy;
+    deleted?: string;
   }> {
     const { id, ...preconfiguredAgentPolicy } = omit(config, 'package_policies');
     const preconfigurationId = String(id);
@@ -159,6 +161,29 @@ class AgentPolicyService {
       search: escapeSearchQueryPhrase(preconfigurationId),
     };
 
+    // Check to see if a preconfigured policy with te same preconfigurationId was already deleted by the user
+    try {
+      const deletionRecord = await esClient.search({
+        index: PRECONFIGURATION_METADATA_INDEX,
+        body: {
+          query: {
+            match: {
+              deleted_preconfiguration_id: preconfigurationId,
+            },
+          },
+        },
+      });
+
+      const { total } = deletionRecord.body.hits;
+      const wasDeleted = (typeof total === 'number' ? total : total.value) > 0;
+      if (wasDeleted) {
+        return { created: false, deleted: preconfigurationId };
+      }
+    } catch (e) {
+      // If ES failed on an index not found error, ignore it. This means nothing has been deleted yet.
+      if (e.body.status !== 404) throw e;
+    }
+
     const newAgentPolicyDefaults: Partial<NewAgentPolicy> = {
       namespace: 'default',
       monitoring_enabled: ['logs', 'metrics'],
@@ -580,6 +605,16 @@ class AgentPolicyService {
         }
       );
     }
+
+    if (agentPolicy.preconfiguration_id) {
+      await esClient.index({
+        index: PRECONFIGURATION_METADATA_INDEX,
+        body: {
+          deleted_preconfiguration_id: String(agentPolicy.preconfiguration_id),
+        },
+      });
+    }
+
     await soClient.delete(SAVED_OBJECT_TYPE, id);
     await this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'deleted', id);
     return {
@@ -817,5 +852,6 @@ export async function addPackageToAgentPolicy(
 
   await packagePolicyService.create(soClient, esClient, newPackagePolicy, {
     bumpRevision: false,
+    skipEnsureInstalled: true,
   });
 }

x-pack/plugins/fleet/server/services/package_policy.ts

@@ -60,7 +60,12 @@ class PackagePolicyService {
     soClient: SavedObjectsClientContract,
     esClient: ElasticsearchClient,
     packagePolicy: NewPackagePolicy,
-    options?: { id?: string; user?: AuthenticatedUser; bumpRevision?: boolean }
+    options?: {
+      id?: string;
+      user?: AuthenticatedUser;
+      bumpRevision?: boolean;
+      skipEnsureInstalled?: boolean;
+    }
   ): Promise<PackagePolicy> {
     // Check that its agent policy does not have a package policy with the same name
     const parentAgentPolicy = await agentPolicyService.get(soClient, packagePolicy.policy_id);
@@ -88,18 +93,25 @@ class PackagePolicyService {
 
     // Make sure the associated package is installed
     if (packagePolicy.package?.name) {
-      const [, pkgInfo] = await Promise.all([
-        ensureInstalledPackage({
-          savedObjectsClient: soClient,
-          pkgName: packagePolicy.package.name,
-          esClient,
-        }),
-        getPackageInfo({
-          savedObjectsClient: soClient,
-          pkgName: packagePolicy.package.name,
-          pkgVersion: packagePolicy.package.version,
-        }),
-      ]);
+      const pkgInfoPromise = getPackageInfo({
+        savedObjectsClient: soClient,
+        pkgName: packagePolicy.package.name,
+        pkgVersion: packagePolicy.package.version,
+      });
+
+      let pkgInfo;
+      if (options?.skipEnsureInstalled) pkgInfo = await pkgInfoPromise;
+      else {
+        const [, packageInfo] = await Promise.all([
+          ensureInstalledPackage({
+            savedObjectsClient: soClient,
+            pkgName: packagePolicy.package.name,
+            esClient,
+          }),
+          pkgInfoPromise,
+        ]);
+        pkgInfo = packageInfo;
+      }
 
       // Check if it is a limited package, and if so, check that the corresponding agent policy does not
       // already contain a package policy for this package

x-pack/plugins/fleet/server/services/preconfiguration.test.ts

@@ -24,6 +24,19 @@ const mockDefaultOutput: Output = {
   hosts: ['http://127.0.0.1:9201'],
 };
 
+const createMockESClient = () => {
+  const esClient = elasticsearchServiceMock.createInternalClient();
+  esClient.search.mockResolvedValue({
+    body: {
+      hits: {
+        // @ts-ignore
+        total: 0,
+      },
+    },
+  });
+  return esClient;
+};
+
 function getPutPreconfiguredPackagesMock() {
   const soClient = savedObjectsClientMock.create();
   soClient.find.mockImplementation(async ({ type, search }) => {
@@ -126,7 +139,7 @@ describe('policy preconfiguration', () => {
 
   it('should perform a no-op when passed no policies or packages', async () => {
     const soClient = getPutPreconfiguredPackagesMock();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const esClient = createMockESClient();
 
     const { policies, packages } = await ensurePreconfiguredPackagesAndPolicies(
       soClient,
@@ -142,7 +155,7 @@ describe('policy preconfiguration', () => {
 
   it('should install packages successfully', async () => {
     const soClient = getPutPreconfiguredPackagesMock();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const esClient = createMockESClient();
 
     const { policies, packages } = await ensurePreconfiguredPackagesAndPolicies(
       soClient,
@@ -158,7 +171,7 @@ describe('policy preconfiguration', () => {
 
   it('should install packages and configure agent policies successfully', async () => {
     const soClient = getPutPreconfiguredPackagesMock();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const esClient = createMockESClient();
 
     const { policies, packages } = await ensurePreconfiguredPackagesAndPolicies(
       soClient,
@@ -187,7 +200,7 @@ describe('policy preconfiguration', () => {
 
   it('should throw an error when trying to install duplicate packages', async () => {
     const soClient = getPutPreconfiguredPackagesMock();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const esClient = createMockESClient();
 
     await expect(
       ensurePreconfiguredPackagesAndPolicies(
@@ -207,7 +220,7 @@ describe('policy preconfiguration', () => {
 
   it('should not attempt to recreate or modify an agent policy if its ID is unchanged', async () => {
     const soClient = getPutPreconfiguredPackagesMock();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const esClient = createMockESClient();
 
     const { policies: policiesA } = await ensurePreconfiguredPackagesAndPolicies(
       soClient,

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -69,13 +69,13 @@ export async function ensurePreconfiguredPackagesAndPolicies(
   // Create policies specified in Kibana config
   const preconfiguredPolicies = await Promise.all(
     policies.map(async (preconfiguredAgentPolicy) => {
-      const { created, policy } = await agentPolicyService.ensurePreconfiguredAgentPolicy(
+      const { created, policy, deleted } = await agentPolicyService.ensurePreconfiguredAgentPolicy(
         soClient,
         esClient,
         omit(preconfiguredAgentPolicy, 'is_managed') // Don't add `is_managed` until the policy has been fully configured
       );
 
-      if (!created) return { created, policy };
+      if (!created) return { created, policy, deleted };
       const { package_policies: packagePolicies } = preconfiguredAgentPolicy;
 
       const installedPackagePolicies = await Promise.all(
@@ -122,22 +122,32 @@ export async function ensurePreconfiguredPackagesAndPolicies(
       await addPreconfiguredPolicyPackages(
         soClient,
         esClient,
-        policy,
+        policy!,
         installedPackagePolicies!,
         defaultOutput
       );
       // Add the is_managed flag after configuring package policies to avoid errors
       if (shouldAddIsManagedFlag) {
-        agentPolicyService.update(soClient, esClient, policy.id, { is_managed: true });
+        agentPolicyService.update(soClient, esClient, policy!.id, { is_managed: true });
       }
     }
   }
 
   return {
-    policies: preconfiguredPolicies.map((p) => ({
-      id: p.policy.id,
-      updated_at: p.policy.updated_at,
-    })),
+    policies: preconfiguredPolicies.map((p) =>
+      p.policy
+        ? {
+            id: p.policy.id,
+            updated_at: p.policy.updated_at,
+          }
+        : {
+            id: p.deleted,
+            updated_at: i18n.translate('xpack.fleet.preconfiguration.policyDeleted', {
+              defaultMessage: 'Preconfigured policy {id} was deleted; skipping creation',
+              values: { id: p.deleted },
+            }),
+          }
+    ),
     packages: preconfiguredPackages.map((pkg) => pkgToPkgKey(pkg)),
   };
 }
@@ -155,20 +165,19 @@ async function addPreconfiguredPolicyPackages(
   >,
   defaultOutput: Output
 ) {
-  return await Promise.all(
-    installedPackagePolicies.map(async ({ installedPackage, name, description, inputs }) =>
-      addPackageToAgentPolicy(
-        soClient,
-        esClient,
-        installedPackage,
-        agentPolicy,
-        defaultOutput,
-        name,
-        description,
-        (policy) => overridePackageInputs(policy, inputs)
-      )
-    )
-  );
+  // Add packages synchronously to avoid overwriting
+  for (const { installedPackage, name, description, inputs } of installedPackagePolicies) {
+    await addPackageToAgentPolicy(
+      soClient,
+      esClient,
+      installedPackage,
+      agentPolicy,
+      defaultOutput,
+      name,
+      description,
+      (policy) => overridePackageInputs(policy, inputs)
+    );
+  }
 }
 
 async function ensureInstalledPreconfiguredPackage(

x-pack/plugins/fleet/server/services/setup.ts

@@ -15,7 +15,9 @@ import type { PackagePolicy } from '../../common';
 
 import { SO_SEARCH_LIMIT } from '../constants';
 
+import { appContextService } from './app_context';
 import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy';
+import { ensurePreconfiguredPackagesAndPolicies } from './preconfiguration';
 import { outputService } from './output';
 import {
   ensureInstalledDefaultPackages,
@@ -151,6 +153,20 @@ async function createSetupSideEffects(
 
   await ensureAgentActionPolicyChangeExists(soClient);
 
+  const { agentPolicies: policiesOrUndefined, packages: packagesOrUndefined } =
+    appContextService.getConfig() ?? {};
+
+  const policies = policiesOrUndefined ?? [];
+  const packages = packagesOrUndefined ?? [];
+
+  await ensurePreconfiguredPackagesAndPolicies(
+    soClient,
+    esClient,
+    policies,
+    packages,
+    defaultOutput
+  );
+
   return { isIntialized: true };
 }