elastic · juliaElastic · Oct 18, 2023 · Oct 10, 2023 · Oct 10, 2023 · Oct 10, 2023
@@ -50,6 +50,9 @@ xpack.fleet.internal.registry.excludePackages: [
 xpack.fleet.packages:
   - name: apm
     version: latest
+# fleet_server package installed to publish agent metrics
+  - name: fleet_server
+    version: latest  
 ## Disable APM UI components and API calls
 xpack.apm.featureFlags.agentConfigurationAvailable: false
 xpack.apm.featureFlags.configurableIndicesAvailable: false

@@ -50,6 +50,10 @@ xpack.fleet.internal.registry.excludePackages: [
     'symantec',
     'cyberark',
   ]
+# fleet_server package installed to publish agent metrics
+xpack.fleet.packages:
+  - name: fleet_server
+    version: latest  
 
 xpack.ml.ad.enabled: true
 xpack.ml.dfa.enabled: true

@@ -34,6 +34,9 @@ export default async function ({ readConfigFile }) {
 
         // to be re-enabled once kibana/issues/102552 is completed
         '--xpack.reporting.enabled=false',
+
+        // disable fleet task that writes to metrics.fleet_server.* data streams, impacting functional tests
+        `--xpack.task_manager.unsafe.exclude_task_types=${JSON.stringify(['Fleet-Metrics-Task'])}`,
       ],
     },
 

@@ -58,13 +58,13 @@ export const getAgentUsage = async (
   };
 };
 
+export interface AgentPerVersion {
+  version: string;
+  count: number;
+}
+
 export interface AgentData {
-  agents_per_version: Array<
-    {
-      version: string;
-      count: number;
-    } & AgentStatus
-  >;
+  agents_per_version: Array<AgentPerVersion & AgentStatus>;
   agent_checkin_status: {
     error: number;
     degraded: number;

@@ -84,6 +84,7 @@ export const createAppContextStartContractMock = (
     config$,
     kibanaVersion: '8.99.0', // Fake version :)
     kibanaBranch: 'main',
+    kibanaInstanceId: '1',
     telemetryEventsSender: createMockTelemetryEventsSender(),
     bulkActionsResolver: {} as any,
     messageSigningService: createMessageSigningServiceMock(),

@@ -130,6 +130,8 @@ import { FleetActionsClient, type FleetActionsClientInterface } from './services
 import type { FilesClientFactory } from './services/files/types';
 import { PolicyWatcher } from './services/agent_policy_watch';
 import { getPackageSpecTagId } from './services/epm/kibana/assets/tag_assets';
+import { FleetMetricsTask } from './services/metrics/fleet_metrics_task';
+import { fetchAgentMetrics } from './services/metrics/fetch_agent_metrics';
 
 export interface FleetSetupDeps {
   security: SecurityPluginSetup;
@@ -167,6 +169,7 @@ export interface FleetAppContext {
   isProductionMode: PluginInitializerContext['env']['mode']['prod'];
   kibanaVersion: PluginInitializerContext['env']['packageInfo']['version'];
   kibanaBranch: PluginInitializerContext['env']['packageInfo']['branch'];
+  kibanaInstanceId: PluginInitializerContext['env']['instanceUuid'];
   cloud?: CloudSetup;
   logger?: Logger;
   httpSetup?: HttpServiceSetup;
@@ -251,6 +254,7 @@ export class FleetPlugin
   private isProductionMode: FleetAppContext['isProductionMode'];
   private kibanaVersion: FleetAppContext['kibanaVersion'];
   private kibanaBranch: FleetAppContext['kibanaBranch'];
+  private kibanaInstanceId: FleetAppContext['kibanaInstanceId'];
   private httpSetup?: HttpServiceSetup;
   private securitySetup!: SecurityPluginSetup;
   private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup;
@@ -259,6 +263,7 @@ export class FleetPlugin
   private bulkActionsResolver?: BulkActionsResolver;
   private fleetUsageSender?: FleetUsageSender;
   private checkDeletedFilesTask?: CheckDeletedFilesTask;
+  private fleetMetricsTask?: FleetMetricsTask;
 
   private agentService?: AgentService;
   private packageService?: PackageService;
@@ -270,6 +275,7 @@ export class FleetPlugin
     this.isProductionMode = this.initializerContext.env.mode.prod;
     this.kibanaVersion = this.initializerContext.env.packageInfo.version;
     this.kibanaBranch = this.initializerContext.env.packageInfo.branch;
+    this.kibanaInstanceId = this.initializerContext.env.instanceUuid;
     this.logger = this.initializerContext.logger.get();
     this.configInitialValue = this.initializerContext.config.get();
     this.telemetryEventsSender = new TelemetryEventsSender(this.logger.get('telemetry_events'));
@@ -440,6 +446,10 @@ export class FleetPlugin
     this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch);
     registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config));
 
+    const fetchAgents = async (abortController: AbortController) =>
+      await fetchAgentMetrics(core, abortController);
+    this.fleetMetricsTask = new FleetMetricsTask(deps.taskManager, fetchAgents);
+
     const router: FleetRouter = core.http.createRouter<FleetRequestHandlerContext>();
     // Allow read-only users access to endpoints necessary for Integrations UI
     // Only some endpoints require superuser so we pass a raw IRouter here
@@ -490,6 +500,7 @@ export class FleetPlugin
       isProductionMode: this.isProductionMode,
       kibanaVersion: this.kibanaVersion,
       kibanaBranch: this.kibanaBranch,
+      kibanaInstanceId: this.kibanaInstanceId,
       httpSetup: this.httpSetup,
       cloud: this.cloud,
       logger: this.logger,
@@ -504,6 +515,7 @@ export class FleetPlugin
     this.fleetUsageSender?.start(plugins.taskManager);
     this.checkDeletedFilesTask?.start({ taskManager: plugins.taskManager });
     startFleetUsageLogger(plugins.taskManager);
+    this.fleetMetricsTask?.start(plugins.taskManager, core.elasticsearch.client.asInternalUser);
 
     const logger = appContextService.getLogger();
 

@@ -62,6 +62,7 @@ class AppContextService {
   private isProductionMode: FleetAppContext['isProductionMode'] = false;
   private kibanaVersion: FleetAppContext['kibanaVersion'] = kibanaPackageJson.version;
   private kibanaBranch: FleetAppContext['kibanaBranch'] = kibanaPackageJson.branch;
+  private kibanaInstanceId: FleetAppContext['kibanaInstanceId'] = '';
   private cloud?: CloudSetup;
   private logger: Logger | undefined;
   private httpSetup?: HttpServiceSetup;
@@ -86,6 +87,7 @@ class AppContextService {
     this.logger = appContext.logger;
     this.kibanaVersion = appContext.kibanaVersion;
     this.kibanaBranch = appContext.kibanaBranch;
+    this.kibanaInstanceId = appContext.kibanaInstanceId;
     this.httpSetup = appContext.httpSetup;
     this.telemetryEventsSender = appContext.telemetryEventsSender;
     this.savedObjectsTagging = appContext.savedObjectsTagging;
@@ -209,6 +211,10 @@ class AppContextService {
     return this.kibanaBranch;
   }
 
+  public getKibanaInstanceId() {
+    return this.kibanaInstanceId;
+  }
+
   public addExternalCallback(type: ExternalCallback[0], callback: ExternalCallback[1]) {
     if (!this.externalCallbacks.has(type)) {
       this.externalCallbacks.set(type, new Set());

@@ -22,12 +22,8 @@ export function getFilteredSearchPackages() {
 }
 
 export function getFilteredInstallPackages() {
-  const shouldFilterFleetServer = appContextService.getConfig()?.internal?.fleetServerStandalone;
   const filtered: string[] = [];
-  // Do not allow to install Fleet server integration if configured to use standalone fleet server
-  if (shouldFilterFleetServer) {
-    filtered.push(FLEET_SERVER_PACKAGE);
-  }
+
   const excludePackages = appContextService.getConfig()?.internal?.registry?.excludePackages ?? [];
 
   return filtered.concat(excludePackages);

@@ -332,7 +332,7 @@ describe('install', () => {
       expect(response.status).toEqual('already_installed');
     });
 
-    it('should not allow to install fleet_server if internal.fleetServerStandalone is configured', async () => {
+    it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => {
       jest.mocked(appContextService.getConfig).mockReturnValueOnce({
         internal: {
           fleetServerStandalone: true,
@@ -347,8 +347,7 @@ describe('install', () => {
         esClient: {} as ElasticsearchClient,
       });
 
-      expect(response.error).toBeDefined();
-      expect(response.error?.message).toMatch(/fleet_server installation is not authorized/);
+      expect(response.status).toEqual('installed');
     });
   });
 

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
+import { coreMock } from '@kbn/core/server/mocks';
+import type { CoreSetup } from '@kbn/core/server';
+
+import { fetchAgentMetrics } from './fetch_agent_metrics';
+
+jest.mock('../../collectors/agent_collectors', () => {
+  return {
+    getAgentUsage: jest.fn().mockResolvedValue({}),
+  };
+});
+
+describe('fetchAgentMetrics', () => {
+  const { createSetup: coreSetupMock } = coreMock;
+  const abortController = new AbortController();
+  let mockCore: CoreSetup;
+  let esClient: ElasticsearchClientMock;
+
+  beforeEach(async () => {
+    mockCore = coreSetupMock();
+    const [{ elasticsearch }] = await mockCore.getStartServices();
+    esClient = elasticsearch.client.asInternalUser as ElasticsearchClientMock;
+  });
+
+  it('should fetch agent metrics', async () => {
+    esClient.search.mockResolvedValue({
+      took: 5,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: {
+          value: 0,
+          relation: 'eq',
+        },
+        hits: [],
+      },
+      aggregations: {
+        versions: {
+          buckets: [
+            {
+              key: '8.12.0',
+              doc_count: 1,
+            },
+          ],
+        },
+        upgrade_details: {
+          buckets: [
+            {
+              key: 'UPG_REQUESTED',
+              doc_count: 1,
+            },
+          ],
+        },
+      },
+    });
+
+    const result = await fetchAgentMetrics(mockCore, abortController);
+
+    expect(result).toEqual({
+      agents: {},
+      agents_per_version: [
+        {
+          version: '8.12.0',
+          count: 1,
+        },
+      ],
+      upgrading_step: {
+        downloading: 0,
+        extracting: 0,
+        failed: 0,
+        replacing: 0,
+        requested: 1,
+        restarting: 0,
+        rollback: 0,
+        scheduled: 0,
+        watching: 0,
+      },
+    });
+  });
+});