From e4a088c7eb09fdb603ec165b77759d919c048cab Mon Sep 17 00:00:00 2001
From: Gerard Soldevila <gerard.soldevila@elastic.co>
Date: Mon, 25 Sep 2023 13:24:42 +0200
Subject: [PATCH 01/27] [Migrations] Ensure individual migrator failures do not
 break consistency (#166924)

## Summary

Tackles https://github.com/elastic/kibana/issues/158818

The goal of the PR is to introduce failures in single migrators at some
of the crucial steps of the migration, testing that consistency is
maintained, and that subsequent migration attempts can successfully
complete the upgrade.

This is done by _proxying_ the `Client` class, which uses the
elasticsearch-js library underneath to perform all calls to ES.
Inspired on https://github.com/elastic/kibana/pull/158995.
---
 .../elasticsearch_client_wrapper.ts           |  81 +++++++
 .../group5/dot_kibana_split.test.ts           | 205 ++++++++++++++++++
 .../migrations/kibana_migrator_test_kit.ts    |   6 +-
 3 files changed, 291 insertions(+), 1 deletion(-)
 create mode 100644 src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts

diff --git a/src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts b/src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts
new file mode 100644
index 0000000000000..66975bbba4409
--- /dev/null
+++ b/src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+
+export type ElasticsearchClientWrapperFactory = (client: Client) => Client;
+
+interface GetElasticsearchClientWrapperFactoryParams {
+  failOn: (methodName: string, methodArguments: any[]) => boolean;
+  errorDelaySeconds?: number;
+}
+
+export const getElasticsearchClientWrapperFactory = ({
+  failOn,
+  errorDelaySeconds,
+}: GetElasticsearchClientWrapperFactoryParams): ElasticsearchClientWrapperFactory => {
+  const interceptClientMethod = (methodName: string, method: any): any => {
+    return new Proxy(method, {
+      apply: (applyTarget, thisArg, methodArguments) => {
+        if (failOn(methodName, methodArguments)) {
+          return new Promise((_, reject) =>
+            setTimeout(
+              () => reject(`Error: esClient.${methodName}() failed unexpectedly`),
+              (errorDelaySeconds || 0) * 1000
+            )
+          );
+        }
+        return Reflect.apply(applyTarget, thisArg, methodArguments);
+      },
+    });
+  };
+
+  const interceptClientApi = (apiName: string, api: any): any =>
+    new Proxy(api, {
+      get(target, prop) {
+        return typeof target[prop] === 'function'
+          ? interceptClientMethod(`${apiName}.${prop.toString()}`, target[prop])
+          : target[prop];
+      },
+    });
+
+  const wrapClient = (client: Client): any =>
+    new Proxy(client, {
+      get(target, prop, receiver) {
+        switch (prop) {
+          // intercept top level esClient methods
+          case 'bulk':
+          case 'deleteByQuery':
+          case 'info':
+          case 'search':
+          case 'updateByQuery':
+            const clientMethod = Reflect.get(target, prop, receiver);
+            return interceptClientMethod(prop, clientMethod);
+          // intercept esClient APIs
+          case 'cluster':
+          case 'indices':
+          case 'tasks':
+            const clientApi = Reflect.get(target, prop, receiver);
+            return interceptClientApi(prop, clientApi);
+          // proxy child Clients too
+          case 'child':
+            return new Proxy(target[prop], {
+              apply(applyTarget, thisArg, argArray) {
+                const childClient = Reflect.apply(applyTarget, thisArg, argArray);
+                return wrapClient(childClient);
+              },
+            });
+          // do NOT intercept the rest of properties and symbols
+          default:
+            return Reflect.get(target, prop, receiver);
+        }
+      },
+    });
+
+  return wrapClient;
+};
diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts
index 636e70cd9dd9e..25dc5a46a6793 100644
--- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts
+++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts
@@ -29,6 +29,7 @@ import {
 } from '../kibana_migrator_test_kit';
 import { delay, parseLogFile } from '../test_utils';
 import '../jest_matchers';
+import { getElasticsearchClientWrapperFactory } from '../elasticsearch_client_wrapper';
 
 // define a type => index distribution
 const RELOCATE_TYPES: Record<string, string> = {
@@ -58,6 +59,210 @@ describe('split .kibana index into multiple system indices', () => {
     await clearLog(logFilePathSecondRun);
   });
 
+  describe('failure cases', () => {
+    const getFailingKibanaMigratorTestKit = async ({
+      logFilePath,
+      failOn,
+      delaySeconds,
+    }: {
+      logFilePath: string;
+      failOn: (methodName: string, methodArgs: any[]) => boolean;
+      delaySeconds?: number;
+    }) => {
+      const clientWrapperFactory = getElasticsearchClientWrapperFactory({
+        failOn,
+        errorDelaySeconds: delaySeconds,
+      });
+
+      return await getKibanaMigratorTestKit({
+        types: typeRegistry.getAllTypes(),
+        kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
+        defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP,
+        logFilePath,
+        clientWrapperFactory,
+      });
+    };
+
+    beforeEach(async () => {
+      esServer = await startElasticsearch({
+        dataArchive: Path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'),
+      });
+    });
+
+    describe('when the .kibana_task_manager migrator fails on the TRANSFORMED_DOCUMENTS_BULK_INDEX state, after the other ones have finished', () => {
+      it('is capable of completing the .kibana_task_manager migration in subsequent restart', async () => {
+        const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({
+          logFilePath: logFilePathFirstRun,
+          failOn: (methodName, methodArgs) => {
+            // fail on esClient.bulk({ index: '.kibana_task_manager_1' }) which supposedly causes
+            // the .kibana_task_manager migrator to fail on the TRANSFORMED_DOCUMENTS_BULK_INDEX state
+            return methodName === 'bulk' && methodArgs[0].index === '.kibana_task_manager_1';
+          },
+          delaySeconds: 90, // give the other migrators enough time to finish before failing
+        });
+
+        try {
+          await firstRun();
+          throw new Error('First run should have thrown an error but it did not');
+        } catch (error) {
+          expect(error.message).toEqual(
+            'Unable to complete saved object migrations for the [.kibana_task_manager] index. Error: esClient.bulk() failed unexpectedly'
+          );
+        }
+      });
+    });
+
+    describe('when the .kibana migrator fails on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK state', () => {
+      it('is capable of successfully performing the split migration in subsequent restart', async () => {
+        const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({
+          logFilePath: logFilePathFirstRun,
+          failOn: (methodName, methodArgs) => {
+            // fail on esClient.bulk({ index: '.kibana_8.11.0_reindex_temp_alias' }) which supposedly causes
+            // the .kibana migrator to fail on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK
+            return (
+              methodName === 'bulk' &&
+              methodArgs[0].index === `.kibana_${currentVersion}_reindex_temp_alias`
+            );
+          },
+          delaySeconds: 10, // give the .kibana_task_manager migrator enough time to finish before failing
+        });
+
+        try {
+          await firstRun();
+          throw new Error('First run should have thrown an error but it did not');
+        } catch (error) {
+          expect(error.message).toEqual(
+            'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.bulk() failed unexpectedly'
+          );
+        }
+      });
+    });
+
+    describe('when the .kibana migrator fails on the CLONE_TEMP_TO_TARGET state', () => {
+      it('is capable of successfully performing the split migration in subsequent restart', async () => {
+        const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({
+          logFilePath: logFilePathFirstRun,
+          failOn: (methodName, methodArgs) => {
+            // fail on esClient.indices.clone({ index: '.kibana_8.11.0_reindex_temp', target: ... }) which supposedly causes
+            // the .kibana migrator to fail on the CLONE_TEMP_TO_TARGET
+            return (
+              methodName === 'indices.clone' &&
+              methodArgs[0].index === `.kibana_${currentVersion}_reindex_temp` &&
+              methodArgs[0].target === `.kibana_${currentVersion}_001`
+            );
+          },
+          delaySeconds: 15, // give the other migrators enough time to finish before failing
+        });
+
+        try {
+          await firstRun();
+          throw new Error('First run should have thrown an error but it did not');
+        } catch (error) {
+          expect(error.message).toEqual(
+            'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.indices.clone() failed unexpectedly'
+          );
+        }
+      });
+    });
+
+    describe('when the .kibana migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state', () => {
+      it('is capable of successfully performing the split migration in subsequent restart', async () => {
+        const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({
+          logFilePath: logFilePathFirstRun,
+          failOn: (methodName, methodArgs) => {
+            // fail on esClient.updateByQuery({ index: '.kibana_8.11.0_001' }) which supposedly causes
+            // the .kibana migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes)
+            return (
+              methodName === 'updateByQuery' &&
+              methodArgs[0].index === `.kibana_${currentVersion}_001`
+            );
+          },
+          delaySeconds: 10, // give the other migrators enough time to finish before failing
+        });
+
+        try {
+          await firstRun();
+          throw new Error('First run should have thrown an error but it did not');
+        } catch (error) {
+          expect(error.message).toEqual(
+            'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.updateByQuery() failed unexpectedly'
+          );
+        }
+      });
+    });
+
+    describe('when the .kibana_analytics migrator fails on the CLONE_TEMP_TO_TARGET state', () => {
+      it('is capable of successfully performing the split migration in subsequent restart', async () => {
+        const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({
+          logFilePath: logFilePathFirstRun,
+          failOn: (methodName, methodArgs) => {
+            // fail on esClient.indices.clone({ index: '.kibana_8.11.0_reindex_temp', target: ... }) which supposedly causes
+            // the .kibana migrator to fail on the CLONE_TEMP_TO_TARGET
+            return (
+              methodName === 'indices.clone' &&
+              methodArgs[0].index === `.kibana_analytics_${currentVersion}_reindex_temp` &&
+              methodArgs[0].target === `.kibana_analytics_${currentVersion}_001`
+            );
+          },
+          delaySeconds: 15, // give the other migrators enough time to finish before failing
+        });
+
+        try {
+          await firstRun();
+          throw new Error('First run should have thrown an error but it did not');
+        } catch (error) {
+          expect(error.message).toEqual(
+            'Unable to complete saved object migrations for the [.kibana_analytics] index. Error: esClient.indices.clone() failed unexpectedly'
+          );
+        }
+      });
+    });
+
+    describe('when the .kibana_analytics migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state', () => {
+      it('is capable of successfully performing the split migration in subsequent restart', async () => {
+        const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({
+          logFilePath: logFilePathFirstRun,
+          failOn: (methodName, methodArgs) => {
+            // fail on esClient.updateByQuery({ index: '.kibana_8.11.0_001' }) which supposedly causes
+            // the .kibana migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes)
+            return (
+              methodName === 'updateByQuery' &&
+              methodArgs[0].index === `.kibana_analytics_${currentVersion}_001`
+            );
+          },
+          delaySeconds: 10, // give the other migrators enough time to finish before failing
+        });
+
+        try {
+          await firstRun();
+          throw new Error('First run should have thrown an error but it did not');
+        } catch (error) {
+          expect(error.message).toEqual(
+            'Unable to complete saved object migrations for the [.kibana_analytics] index. Error: esClient.updateByQuery() failed unexpectedly'
+          );
+        }
+      });
+    });
+
+    afterEach(async () => {
+      const { runMigrations: secondRun } = await getKibanaMigratorTestKit({
+        types: typeRegistry.getAllTypes(),
+        logFilePath: logFilePathSecondRun,
+        kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
+        defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP,
+      });
+      const results = await secondRun();
+      expect(
+        results
+          .flat()
+          .every((result) => result.status === 'migrated' || result.status === 'patched')
+      ).toEqual(true);
+
+      await esServer?.stop();
+      await delay(2);
+    });
+  });
+
   describe('when migrating from a legacy version', () => {
     let migratorTestKitFactory: (logFilePath: string) => Promise<KibanaMigratorTestKit>;
 
diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts
index a911fcdbdead5..1f6e9a7a58c77 100644
--- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts
+++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts
@@ -50,6 +50,7 @@ import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
 import type { NodeRoles } from '@kbn/core-node-server';
 import { baselineDocuments, baselineTypes } from './kibana_migrator_test_kit.fixtures';
 import { delay } from './test_utils';
+import type { ElasticsearchClientWrapperFactory } from './elasticsearch_client_wrapper';
 
 export const defaultLogFilePath = Path.join(__dirname, 'kibana_migrator_test_kit.log');
 
@@ -76,6 +77,7 @@ export interface KibanaMigratorTestKitParams {
   types?: Array<SavedObjectsType<any>>;
   defaultIndexTypesMap?: IndexTypesMap;
   logFilePath?: string;
+  clientWrapperFactory?: ElasticsearchClientWrapperFactory;
 }
 
 export interface KibanaMigratorTestKit {
@@ -134,6 +136,7 @@ export const getKibanaMigratorTestKit = async ({
   types = [],
   logFilePath = defaultLogFilePath,
   nodeRoles = defaultNodeRoles,
+  clientWrapperFactory,
 }: KibanaMigratorTestKitParams = {}): Promise<KibanaMigratorTestKit> => {
   let hasRun = false;
   const loggingSystem = new LoggingSystem();
@@ -145,7 +148,8 @@ export const getKibanaMigratorTestKit = async ({
   const loggingConf = await firstValueFrom(configService.atPath<LoggingConfigType>('logging'));
   loggingSystem.upgrade(loggingConf);
 
-  const client = await getElasticsearchClient(configService, loggerFactory, kibanaVersion);
+  const rawClient = await getElasticsearchClient(configService, loggerFactory, kibanaVersion);
+  const client = clientWrapperFactory ? clientWrapperFactory(rawClient) : rawClient;
 
   const typeRegistry = new SavedObjectTypeRegistry();
 

From 258e9848d16ccc0a6d184983cbf531c2e1b033d5 Mon Sep 17 00:00:00 2001
From: Maryam Saeidi <maryam.saeidi@elastic.co>
Date: Mon, 25 Sep 2023 15:49:12 +0200
Subject: [PATCH 02/27] Fix Errors rules link on observability alert page
 (#167027)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #167004

## Summary

This PR fixes the Errors stat link and improves the code using the Rules
locator.


![image](https://github.com/elastic/kibana/assets/12370520/bb836360-3ee9-4b12-b079-7fa9ad26bbf7)

## 🧪 How to test
- Please check all the statuses (Disabled, Snoozed, Errors) to make sure
links work as expected.
---
 .../public/pages/alerts/alerts.tsx            | 12 ++++-
 .../alerts/components/rule_stats.test.tsx     | 53 +++++++++++++------
 .../pages/alerts/components/rule_stats.tsx    | 31 +++++------
 .../observability/public/plugin.mock.tsx      |  9 ++--
 4 files changed, 70 insertions(+), 35 deletions(-)

diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts.tsx
index 27ae07cbf768d..08b89dad773ce 100644
--- a/x-pack/plugins/observability/public/pages/alerts/alerts.tsx
+++ b/x-pack/plugins/observability/public/pages/alerts/alerts.tsx
@@ -15,6 +15,8 @@ import { AlertConsumers } from '@kbn/rule-data-utils';
 import { useBreadcrumbs } from '@kbn/observability-shared-plugin/public';
 import { MaintenanceWindowCallout } from '@kbn/alerts-ui-shared';
 
+import { rulesLocatorID } from '../../../common';
+import { RulesParams } from '../../locators/rules';
 import { useKibana } from '../../utils/kibana_react';
 import { useHasData } from '../../hooks/use_has_data';
 import { usePluginContext } from '../../hooks/use_plugin_context';
@@ -53,6 +55,9 @@ function InternalAlertsPage() {
     },
     http,
     notifications: { toasts },
+    share: {
+      url: { locators },
+    },
     triggersActionsUi: {
       alertsTableConfigurationRegistry,
       getAlertsSearchBar: AlertsSearchBar,
@@ -179,7 +184,12 @@ function InternalAlertsPage() {
           pageTitle: (
             <>{i18n.translate('xpack.observability.alertsTitle', { defaultMessage: 'Alerts' })} </>
           ),
-          rightSideItems: renderRuleStats(ruleStats, manageRulesHref, ruleStatsLoading),
+          rightSideItems: renderRuleStats(
+            ruleStats,
+            manageRulesHref,
+            ruleStatsLoading,
+            locators.get<RulesParams>(rulesLocatorID)
+          ),
         }}
       >
         <HeaderMenu />
diff --git a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx
index aaeab5d7855f7..758df7224be2d 100644
--- a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx
+++ b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx
@@ -6,7 +6,9 @@
  */
 
 import { renderRuleStats } from './rule_stats';
-import { render, screen } from '@testing-library/react';
+import { render, screen, fireEvent } from '@testing-library/react';
+import { LocatorPublic } from '@kbn/share-plugin/common';
+import { RulesParams } from '../../../locators/rules';
 
 const RULES_PAGE_LINK = '/app/observability/alerts/rules';
 const STAT_CLASS = 'euiStat';
@@ -14,6 +16,14 @@ const STAT_TITLE_PRIMARY_SELECTOR = '[class*="euiStat__title-primary"]';
 const STAT_BUTTON_CLASS = 'euiButtonEmpty';
 
 describe('Rule stats', () => {
+  const mockedLocator = {
+    navigate: jest.fn(),
+  } as any as LocatorPublic<RulesParams>;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
   test('renders all rule stats', async () => {
     const stats = renderRuleStats(
       {
@@ -58,14 +68,17 @@ describe('Rule stats', () => {
         snoozed: 0,
       },
       RULES_PAGE_LINK,
-      false
+      false,
+      mockedLocator
     );
     const { container } = render(stats[4]);
-    expect(screen.getByText('Disabled').closest('a')).toHaveAttribute(
-      'href',
-      `${RULES_PAGE_LINK}?_a=(lastResponse:!(),status:!(disabled))`
-    );
 
+    fireEvent.click(screen.getByText('Disabled'));
+
+    expect(mockedLocator.navigate).toHaveBeenCalledWith(
+      { status: ['disabled'] },
+      { replace: false }
+    );
     expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1);
   });
 
@@ -115,14 +128,18 @@ describe('Rule stats', () => {
         snoozed: 1,
       },
       RULES_PAGE_LINK,
-      false
+      false,
+      mockedLocator
     );
     const { container } = render(stats[3]);
-    expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1);
-    expect(screen.getByText('Snoozed').closest('a')).toHaveAttribute(
-      'href',
-      `${RULES_PAGE_LINK}?_a=(lastResponse:!(),status:!(snoozed))`
+
+    fireEvent.click(screen.getByText('Snoozed'));
+
+    expect(mockedLocator.navigate).toHaveBeenCalledWith(
+      { status: ['snoozed'] },
+      { replace: false }
     );
+    expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1);
   });
 
   test('snoozed stat count is link-colored, when there are snoozed rules', async () => {
@@ -171,14 +188,18 @@ describe('Rule stats', () => {
         snoozed: 0,
       },
       RULES_PAGE_LINK,
-      false
+      false,
+      mockedLocator
     );
     const { container } = render(stats[2]);
-    expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1);
-    expect(screen.getByText('Errors').closest('a')).toHaveAttribute(
-      'href',
-      `${RULES_PAGE_LINK}?_a=(lastResponse:!(error),status:!())`
+
+    fireEvent.click(screen.getByText('Errors'));
+
+    expect(mockedLocator.navigate).toHaveBeenCalledWith(
+      { lastResponse: ['failed'] },
+      { replace: false }
     );
+    expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1);
   });
 
   test('errors stat count is link-colored, when there are error rules', () => {
diff --git a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx
index f85a6a766b17c..005ba7ccaec82 100644
--- a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx
+++ b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx
@@ -8,8 +8,10 @@
 import React from 'react';
 import { EuiButtonEmpty, EuiStat } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
+import { LocatorPublic } from '@kbn/share-plugin/common';
 import { euiThemeVars } from '@kbn/ui-theme';
 import { euiStyled } from '@kbn/kibana-react-plugin/common';
+import { RulesParams } from '../../../locators/rules';
 
 export interface RuleStatsState {
   total: number;
@@ -18,7 +20,7 @@ export interface RuleStatsState {
   error: number;
   snoozed: number;
 }
-type StatType = 'disabled' | 'snoozed' | 'error';
+type Status = 'disabled' | 'snoozed' | 'error';
 
 const Divider = euiStyled.div`
   border-right: 1px solid ${euiThemeVars.euiColorLightShade};
@@ -41,33 +43,32 @@ const ConditionalWrap = ({
   children: JSX.Element;
 }): JSX.Element => (condition ? wrap(children) : children);
 
-const getStatCount = (stats: RuleStatsState, statType: StatType) => {
-  if (statType === 'snoozed') return stats.snoozed + stats.muted;
-  return stats[statType];
+const getStatCount = (stats: RuleStatsState, status: Status) => {
+  if (status === 'snoozed') return stats.snoozed + stats.muted;
+  return stats[status];
 };
 
 export const renderRuleStats = (
   ruleStats: RuleStatsState,
   manageRulesHref: string,
-  ruleStatsLoading: boolean
+  ruleStatsLoading: boolean,
+  rulesLocator?: LocatorPublic<RulesParams>
 ) => {
-  const createRuleStatsLink = (stats: RuleStatsState, statType: StatType) => {
-    const count = getStatCount(stats, statType);
-    let statsLink = `${manageRulesHref}?_a=(lastResponse:!(),status:!())`;
+  const handleNavigateToRules = async (stats: RuleStatsState, status: Status) => {
+    const count = getStatCount(stats, status);
     if (count > 0) {
-      switch (statType) {
+      switch (status) {
         case 'error':
-          statsLink = `${manageRulesHref}?_a=(lastResponse:!(error),status:!())`;
+          await rulesLocator?.navigate({ lastResponse: ['failed'] }, { replace: false });
           break;
         case 'snoozed':
         case 'disabled':
-          statsLink = `${manageRulesHref}?_a=(lastResponse:!(),status:!(${statType}))`;
+          await rulesLocator?.navigate({ status: [status] }, { replace: false });
           break;
         default:
           break;
       }
     }
-    return statsLink;
   };
 
   const disabledStatsComponent = (
@@ -76,7 +77,7 @@ export const renderRuleStats = (
       wrap={(wrappedChildren) => (
         <EuiButtonEmpty
           data-test-subj="o11yDisabledStatsComponentButton"
-          href={createRuleStatsLink(ruleStats, 'disabled')}
+          onClick={() => handleNavigateToRules(ruleStats, 'disabled')}
         >
           {wrappedChildren}
         </EuiButtonEmpty>
@@ -102,7 +103,7 @@ export const renderRuleStats = (
       wrap={(wrappedChildren) => (
         <EuiButtonEmpty
           data-test-subj="o11ySnoozedStatsComponentButton"
-          href={createRuleStatsLink(ruleStats, 'snoozed')}
+          onClick={() => handleNavigateToRules(ruleStats, 'snoozed')}
         >
           {wrappedChildren}
         </EuiButtonEmpty>
@@ -128,7 +129,7 @@ export const renderRuleStats = (
       wrap={(wrappedChildren) => (
         <EuiButtonEmpty
           data-test-subj="o11yErrorStatsComponentButton"
-          href={createRuleStatsLink(ruleStats, 'error')}
+          onClick={() => handleNavigateToRules(ruleStats, 'error')}
         >
           {wrappedChildren}
         </EuiButtonEmpty>
diff --git a/x-pack/plugins/observability/public/plugin.mock.tsx b/x-pack/plugins/observability/public/plugin.mock.tsx
index be663d15e444d..9732f67aac35f 100644
--- a/x-pack/plugins/observability/public/plugin.mock.tsx
+++ b/x-pack/plugins/observability/public/plugin.mock.tsx
@@ -4,11 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import React from 'react';
 import { mockCasesContract } from '@kbn/cases-plugin/public/mocks';
 import { chartPluginMock } from '@kbn/charts-plugin/public/mocks';
 import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
 import { contentManagementMock } from '@kbn/content-management-plugin/public/mocks';
+import { sharePluginMock } from '@kbn/share-plugin/public/mocks';
 import { unifiedSearchPluginMock } from '@kbn/unified-search-plugin/public/mocks';
 
 const triggersActionsUiStartMock = {
@@ -97,12 +99,13 @@ export const observabilityPublicPluginsStartMock = {
       cases: mockCasesContract(),
       charts: chartPluginMock.createStartContract(),
       contentManagement: contentManagementMock.createStartContract(),
-      triggersActionsUi: triggersActionsUiStartMock.createStart(),
       data: dataPluginMock.createStartContract(),
-      dataViews: dataViews.createStart(),
       dataViewEditor: dataViewEditor.createStart(),
-      lens: null,
+      dataViews: dataViews.createStart(),
       discover: null,
+      lens: null,
+      share: sharePluginMock.createStartContract(),
+      triggersActionsUi: triggersActionsUiStartMock.createStart(),
       unifiedSearch: unifiedSearchPluginMock.createStartContract(),
     };
   },

From ab2881feef74effce344dfa4480dd91d50d4c7b5 Mon Sep 17 00:00:00 2001
From: Alex Szabo <alex.szabo@elastic.co>
Date: Mon, 25 Sep 2023 15:58:28 +0200
Subject: [PATCH 03/27] Further test failures (#167148)

## Summary
---
 src/dev/build/tasks/fleet/download_elastic_gpg_key.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts b/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts
index 483a342ba300e..6cd0b351c4d31 100644
--- a/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts
+++ b/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts
@@ -13,9 +13,9 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { downloadToDisk } from '../../lib';
 
 const ARTIFACTS_URL = 'https://artifacts.elastic.co/';
-const GPG_KEY_NAME = 'GPG-KEY-elasticsearch';
+const GPG_KEY_NAME = 'GPG-KEY-elasticsearch.sha1';
 const GPG_KEY_SHA512 =
-  '62a567354286deb02baf5fc6b82ddf6c7067898723463da9ae65b132b8c6d6f064b2874e390885682376228eed166c1c82fe7f11f6c9a69f0c157029c548fa3d';
+  '84ee193cc337344d9a7da9021daf3f5ede83f5f1ab049d169f3634921529dcd096abf7a91eec7f26f3a6913e5e38f88f69a5e2ce79ad155d46edc75705a648c6';
 
 export async function downloadElasticGpgKey(pkgDir: string, log: ToolingLog) {
   const gpgKeyUrl = ARTIFACTS_URL + GPG_KEY_NAME;

From a71242686acd0a43f446af7ce606b36f161819fd Mon Sep 17 00:00:00 2001
From: Alexi Doak <109488926+doakalexi@users.noreply.github.com>
Date: Mon, 25 Sep 2023 07:02:23 -0700
Subject: [PATCH 04/27] =?UTF-8?q?[ResponseOps]=20[Alerting]=20Removing=20s?=
 =?UTF-8?q?kip=20on=20x-pack/test/alerting=5Fapi=5Fintegration/security=5F?=
 =?UTF-8?q?and=5Fspaces/group2/tests/telemetry/alerting=5Fand=5Factions=5F?=
 =?UTF-8?q?telemetry=C2=B7ts=20(#166985)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Resolves https://github.com/elastic/kibana/issues/140973

## Summary

Failure were caused by
```
Error: read ECONNRESET
    at TCP.onStreamRead (node:internal/stream_base_commons:217:20) {
  errno: -104,
  code: 'ECONNRESET',
  syscall: 'read',
  response: undefined
}
```
Flaky test runner x 250
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3210
---
 .../group2/tests/telemetry/alerting_and_actions_telemetry.ts   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts
index ddc90ec10e9b6..02a41b6de7afa 100644
--- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts
+++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts
@@ -26,8 +26,7 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F
   const esTestIndexTool = new ESTestIndexTool(es, retry);
   const supertestWithoutAuth = getService('supertestWithoutAuth');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/140973
-  describe.skip('telemetry', () => {
+  describe('telemetry', () => {
     const objectRemover = new ObjectRemover(supertest);
     const alwaysFiringRuleId: { [key: string]: string } = {};
 

From dfcf60ae33f9482465e5c549023a6ec7b7604acd Mon Sep 17 00:00:00 2001
From: Vadim Kibana <82822460+vadimkibana@users.noreply.github.com>
Date: Mon, 25 Sep 2023 16:04:28 +0200
Subject: [PATCH 05/27] Remove deprecated EUI components in kibana_react
 (#166225)

## Summary

Closes https://github.com/elastic/kibana/issues/161422

In this PR:

- [x] Removes `KibanaPageTemplate` utilities from `kibana_react`
- [x] Migrates remaining users of `KibanaPageTemplate` from
`kibana_react` to the one from `@kbn/shared-ux-page-kibana-template`
- [x]
`src/plugins/home/public/application/components/tutorial_directory.js`
- [x]
`src/plugins/home/public/application/components/tutorial/tutorial.js`
  - [x] `x-pack/plugins/osquery/public/components/empty_state.tsx`

## Visual Changes

Below are the main visual differences this PR introduces.

Sample data page before:

<img width="1419" alt="Screenshot 2023-09-13 at 12 12 11"
src="https://github.com/elastic/kibana/assets/82822460/cbad5bd2-9c51-486c-a48d-ac44a80cdbf7">

Sample data page now:

<img width="1456" alt="Screenshot 2023-09-13 at 12 12 18"
src="https://github.com/elastic/kibana/assets/82822460/4118db78-89e8-41ed-a3d3-51a221dde857">

Tutorials page before:

<img width="1402" alt="Screenshot 2023-09-13 at 13 09 58"
src="https://github.com/elastic/kibana/assets/82822460/b51af548-211a-4a59-9143-b82f2ff55678">

Tutorials page now:

<img width="1439" alt="Screenshot 2023-09-13 at 13 10 04"
src="https://github.com/elastic/kibana/assets/82822460/f71590a3-7c3d-4f60-a460-47aacdfcbf09">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
---
 .../__snapshots__/page_template.test.tsx.snap |   2 +-
 .../page_template_inner.test.tsx.snap         |   3 +-
 .../impl/src/page_template_inner.tsx          |   3 +-
 .../__snapshots__/tutorial.test.js.snap       | 276 +++++------
 .../components/tutorial/tutorial.js           |  10 +-
 .../components/tutorial_directory.js          |   4 +-
 src/plugins/kibana_react/public/index.ts      |   4 -
 .../__snapshots__/page_template.test.tsx.snap | 440 ------------------
 .../public/page_template/index.ts             |   4 -
 .../page_template/no_data_page/index.ts       |   1 -
 .../no_data_config_page/index.tsx             |   9 -
 .../no_data_config_page.tsx                   |  38 --
 .../public/page_template/page_template.scss   |  22 -
 .../page_template/page_template.test.tsx      | 173 -------
 .../public/page_template/page_template.tsx    |  98 ----
 .../page_template/page_template_inner.tsx     |  61 ---
 .../public/page_template/util/constants.ts    |  20 -
 .../public/page_template/util/index.ts        |   1 -
 .../page_template/with_solution_nav.tsx       |  77 ---
 .../osquery/public/components/empty_state.tsx |   2 +-
 x-pack/plugins/osquery/tsconfig.json          |   1 +
 21 files changed, 157 insertions(+), 1092 deletions(-)
 delete mode 100644 src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap
 delete mode 100644 src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx
 delete mode 100644 src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx
 delete mode 100644 src/plugins/kibana_react/public/page_template/page_template.scss
 delete mode 100644 src/plugins/kibana_react/public/page_template/page_template.test.tsx
 delete mode 100644 src/plugins/kibana_react/public/page_template/page_template.tsx
 delete mode 100644 src/plugins/kibana_react/public/page_template/page_template_inner.tsx
 delete mode 100644 src/plugins/kibana_react/public/page_template/util/constants.ts
 delete mode 100644 src/plugins/kibana_react/public/page_template/with_solution_nav.tsx

diff --git a/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap b/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap
index c57c90acbdcf7..09421e4cb5dd7 100644
--- a/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap
+++ b/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap
@@ -3,7 +3,7 @@
 exports[`KibanaPageTemplate render basic template 1`] = `
 <div
   class="euiPageTemplate kbnPageTemplate emotion-euiPageOuter-row-grow"
-  style="min-block-size:max(0, 100vh);padding-block-start:0"
+  style="min-block-size:calc(100vh - var(--euiFixedHeadersOffset, 0));padding-block-start:0"
 >
   <main
     class="emotion-euiPageInner"
diff --git a/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template_inner.test.tsx.snap b/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template_inner.test.tsx.snap
index 7a238b54533dd..9b89225ff7ce3 100644
--- a/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template_inner.test.tsx.snap
+++ b/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template_inner.test.tsx.snap
@@ -11,7 +11,8 @@ exports[`KibanaPageTemplateInner isEmpty no pageHeader 1`] = `
 exports[`KibanaPageTemplateInner isEmpty pageHeader & children 1`] = `
 <_EuiPageTemplate
   className="kbnPageTemplate"
-  minHeight={0}
+  grow={false}
+  minHeight="calc(100vh - var(--euiFixedHeadersOffset, 0))"
   offset={0}
 >
   <_EuiPageHeader
diff --git a/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx b/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx
index f9b9dcd247de6..5da29ba797041 100644
--- a/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx
+++ b/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx
@@ -68,7 +68,8 @@ export const KibanaPageTemplateInner: FC<Props> = ({
       // the following props can be removed to allow the template to auto-handle
       // the fixed header and banner heights.
       offset={0}
-      minHeight={0}
+      minHeight={header ? 'calc(100vh - var(--euiFixedHeadersOffset, 0))' : 0}
+      grow={header ? false : undefined}
       {...rest}
     >
       {sideBar}
diff --git a/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap b/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap
index bd4e658b04a99..2cd4388680751 100644
--- a/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap
+++ b/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap
@@ -1,160 +1,166 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`isCloudEnabled is false should not render instruction toggle when ON_PREM_ELASTIC_CLOUD instructions are not provided 1`] = `
-<KibanaPageTemplate
+<_KibanaPageTemplate
   template="empty"
 >
-  <div>
-    <InjectIntl(IntroductionUI)
-      basePath={
-        Object {
-          "prepend": [Function],
-        }
-      }
-      description="tutorial used to drive jest tests"
-      notices={null}
-      title="jest test tutorial"
-    />
-    <EuiSpacer />
-    <InjectIntl(InstructionSetUi)
-      instructionVariants={
-        Array [
+  <_EuiPageSection>
+    <div>
+      <InjectIntl(IntroductionUI)
+        basePath={
           Object {
-            "id": "platform id",
-            "instructions": Array [
-              Object {
-                "title": "onPrem instructions",
-              },
-            ],
-          },
-        ]
-      }
-      isCloudEnabled={false}
-      offset={1}
-      onStatusCheck={[Function]}
-      paramValues={Object {}}
-      replaceTemplateStrings={[Function]}
-      setParameter={[Function]}
-      statusCheckState="NOT_CHECKED"
-      title="Instruction title"
-    />
-  </div>
-</KibanaPageTemplate>
+            "prepend": [Function],
+          }
+        }
+        description="tutorial used to drive jest tests"
+        notices={null}
+        title="jest test tutorial"
+      />
+      <EuiSpacer />
+      <InjectIntl(InstructionSetUi)
+        instructionVariants={
+          Array [
+            Object {
+              "id": "platform id",
+              "instructions": Array [
+                Object {
+                  "title": "onPrem instructions",
+                },
+              ],
+            },
+          ]
+        }
+        isCloudEnabled={false}
+        offset={1}
+        onStatusCheck={[Function]}
+        paramValues={Object {}}
+        replaceTemplateStrings={[Function]}
+        setParameter={[Function]}
+        statusCheckState="NOT_CHECKED"
+        title="Instruction title"
+      />
+    </div>
+  </_EuiPageSection>
+</_KibanaPageTemplate>
 `;
 
 exports[`isCloudEnabled is false should render ON_PREM instructions with instruction toggle 1`] = `
-<KibanaPageTemplate
+<_KibanaPageTemplate
   template="empty"
 >
-  <div>
-    <InjectIntl(IntroductionUI)
-      basePath={
-        Object {
-          "prepend": [Function],
+  <_EuiPageSection>
+    <div>
+      <InjectIntl(IntroductionUI)
+        basePath={
+          Object {
+            "prepend": [Function],
+          }
         }
-      }
-      description="tutorial used to drive jest tests"
-      iconType="logoApache"
-      notices={null}
-      title="jest test tutorial"
-    />
-    <EuiSpacer />
-    <EuiFormRow
-      describedByIds={Array []}
-      display="row"
-      hasChildLabel={true}
-      hasEmptyLabelSpace={false}
-      labelType="label"
-    >
-      <EuiButtonGroup
-        buttonSize="m"
-        color="text"
-        idSelected="onPrem"
-        isFullWidth={true}
-        legend="Deployment type"
-        onChange={[Function]}
-        options={
+        description="tutorial used to drive jest tests"
+        iconType="logoApache"
+        notices={null}
+        title="jest test tutorial"
+      />
+      <EuiSpacer />
+      <EuiFormRow
+        describedByIds={Array []}
+        display="row"
+        hasChildLabel={true}
+        hasEmptyLabelSpace={false}
+        labelType="label"
+      >
+        <EuiButtonGroup
+          buttonSize="m"
+          color="text"
+          idSelected="onPrem"
+          isFullWidth={true}
+          legend="Deployment type"
+          onChange={[Function]}
+          options={
+            Array [
+              Object {
+                "data-test-subj": "selfManagedTutorial",
+                "id": "onPrem",
+                "label": "Self managed",
+              },
+              Object {
+                "data-test-subj": "onCloudTutorial",
+                "id": "onPremElasticCloud",
+                "label": "Elastic Cloud",
+              },
+            ]
+          }
+        />
+      </EuiFormRow>
+      <EuiSpacer />
+      <InjectIntl(InstructionSetUi)
+        instructionVariants={
           Array [
             Object {
-              "data-test-subj": "selfManagedTutorial",
-              "id": "onPrem",
-              "label": "Self managed",
-            },
-            Object {
-              "data-test-subj": "onCloudTutorial",
-              "id": "onPremElasticCloud",
-              "label": "Elastic Cloud",
+              "id": "platform id",
+              "instructions": Array [
+                Object {
+                  "title": "onPrem instructions",
+                },
+              ],
             },
           ]
         }
+        isCloudEnabled={false}
+        offset={1}
+        onStatusCheck={[Function]}
+        paramValues={Object {}}
+        replaceTemplateStrings={[Function]}
+        setParameter={[Function]}
+        statusCheckState="NOT_CHECKED"
+        title="Instruction title"
       />
-    </EuiFormRow>
-    <EuiSpacer />
-    <InjectIntl(InstructionSetUi)
-      instructionVariants={
-        Array [
-          Object {
-            "id": "platform id",
-            "instructions": Array [
-              Object {
-                "title": "onPrem instructions",
-              },
-            ],
-          },
-        ]
-      }
-      isCloudEnabled={false}
-      offset={1}
-      onStatusCheck={[Function]}
-      paramValues={Object {}}
-      replaceTemplateStrings={[Function]}
-      setParameter={[Function]}
-      statusCheckState="NOT_CHECKED"
-      title="Instruction title"
-    />
-  </div>
-</KibanaPageTemplate>
+    </div>
+  </_EuiPageSection>
+</_KibanaPageTemplate>
 `;
 
 exports[`should render ELASTIC_CLOUD instructions when isCloudEnabled is true 1`] = `
-<KibanaPageTemplate
+<_KibanaPageTemplate
   template="empty"
 >
-  <div>
-    <InjectIntl(IntroductionUI)
-      basePath={
-        Object {
-          "prepend": [Function],
-        }
-      }
-      description="tutorial used to drive jest tests"
-      iconType="logoApache"
-      notices={null}
-      title="jest test tutorial"
-    />
-    <EuiSpacer />
-    <InjectIntl(InstructionSetUi)
-      instructionVariants={
-        Array [
+  <_EuiPageSection>
+    <div>
+      <InjectIntl(IntroductionUI)
+        basePath={
           Object {
-            "id": "platform id",
-            "instructions": Array [
-              Object {
-                "title": "elasticCloud instructions",
-              },
-            ],
-          },
-        ]
-      }
-      isCloudEnabled={true}
-      offset={1}
-      onStatusCheck={[Function]}
-      paramValues={Object {}}
-      replaceTemplateStrings={[Function]}
-      setParameter={[Function]}
-      statusCheckState="NOT_CHECKED"
-      title="Instruction title"
-    />
-  </div>
-</KibanaPageTemplate>
+            "prepend": [Function],
+          }
+        }
+        description="tutorial used to drive jest tests"
+        iconType="logoApache"
+        notices={null}
+        title="jest test tutorial"
+      />
+      <EuiSpacer />
+      <InjectIntl(InstructionSetUi)
+        instructionVariants={
+          Array [
+            Object {
+              "id": "platform id",
+              "instructions": Array [
+                Object {
+                  "title": "elasticCloud instructions",
+                },
+              ],
+            },
+          ]
+        }
+        isCloudEnabled={true}
+        offset={1}
+        onStatusCheck={[Function]}
+        paramValues={Object {}}
+        replaceTemplateStrings={[Function]}
+        setParameter={[Function]}
+        statusCheckState="NOT_CHECKED"
+        title="Instruction title"
+      />
+    </div>
+  </_EuiPageSection>
+</_KibanaPageTemplate>
 `;
diff --git a/src/plugins/home/public/application/components/tutorial/tutorial.js b/src/plugins/home/public/application/components/tutorial/tutorial.js
index de222dbe6155d..620da42169c35 100644
--- a/src/plugins/home/public/application/components/tutorial/tutorial.js
+++ b/src/plugins/home/public/application/components/tutorial/tutorial.js
@@ -18,7 +18,7 @@ import * as StatusCheckStates from './status_check_states';
 import { injectI18n, FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
 import { getServices } from '../../kibana_services';
-import { KibanaPageTemplate } from '@kbn/kibana-react-plugin/public';
+import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template';
 
 const INSTRUCTIONS_TYPE = {
   ELASTIC_CLOUD: 'elasticCloud',
@@ -360,7 +360,7 @@ class TutorialUi extends React.Component {
   render() {
     let content;
     if (this.state.notFound) {
-      content = (
+      return (
         <KibanaPageTemplate
           isEmptyState={true}
           pageHeader={{
@@ -435,7 +435,11 @@ class TutorialUi extends React.Component {
       );
     }
 
-    return <KibanaPageTemplate template="empty">{content}</KibanaPageTemplate>;
+    return (
+      <KibanaPageTemplate template="empty">
+        <KibanaPageTemplate.Section>{content}</KibanaPageTemplate.Section>
+      </KibanaPageTemplate>
+    );
   }
 }
 
diff --git a/src/plugins/home/public/application/components/tutorial_directory.js b/src/plugins/home/public/application/components/tutorial_directory.js
index cde8b86f5df22..c5a4c1b27fbab 100644
--- a/src/plugins/home/public/application/components/tutorial_directory.js
+++ b/src/plugins/home/public/application/components/tutorial_directory.js
@@ -16,7 +16,7 @@ import { SampleDataTab } from '@kbn/home-sample-data-tab';
 import { i18n } from '@kbn/i18n';
 import { Synopsis } from './synopsis';
 import { getServices } from '../kibana_services';
-import { KibanaPageTemplate } from '@kbn/kibana-react-plugin/public';
+import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template';
 import { getTutorials } from '../load_tutorials';
 
 const SAMPLE_DATA_TAB_ID = 'sampleData';
@@ -257,7 +257,7 @@ class TutorialDirectoryUi extends React.Component {
           rightSideItems: headerLinks ? [headerLinks] : [],
         }}
       >
-        {this.renderTabContent()}
+        <KibanaPageTemplate.Section>{this.renderTabContent()}</KibanaPageTemplate.Section>
       </KibanaPageTemplate>
     );
   }
diff --git a/src/plugins/kibana_react/public/index.ts b/src/plugins/kibana_react/public/index.ts
index f92739d045439..7954559206bb7 100644
--- a/src/plugins/kibana_react/public/index.ts
+++ b/src/plugins/kibana_react/public/index.ts
@@ -54,17 +54,13 @@ export { POSITIONS, WEIGHTS, TOOLBAR_BUTTON_SIZES, ToolbarButton } from './toolb
 export { reactRouterNavigate, reactRouterOnClickHandler } from './react_router_navigate';
 
 export type {
-  KibanaPageTemplateProps,
   NoDataPageActions,
   NoDataPageActionsProps,
   NoDataPageProps,
   ElasticAgentCardProps,
 } from './page_template';
 export {
-  KibanaPageTemplate,
   KibanaPageTemplateSolutionNavAvatar,
-  NO_DATA_PAGE_MAX_WIDTH,
-  NO_DATA_PAGE_TEMPLATE_PROPS,
   NO_DATA_RECOMMENDED,
   NoDataPage,
   ElasticAgentCard,
diff --git a/src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap b/src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap
deleted file mode 100644
index 4dea9549670f3..0000000000000
--- a/src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap
+++ /dev/null
@@ -1,440 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`KibanaPageTemplate render basic template 1`] = `
-<div
-  class="euiPage euiPageTemplate kbnPageTemplate emotion-euiPage-row-grow"
-  style="min-height:460px"
->
-  <div
-    class="euiPageBody emotion-euiPageBody"
-  >
-    <header
-      class="euiPageHeader emotion-euiPageHeader-l-border"
-      title="test"
-    >
-      <div
-        class="euiPageHeaderContent emotion-euiPageHeaderContent-l"
-        style="max-width:1200px"
-      >
-        <div
-          class="euiFlexGroup euiPageHeaderContent__top emotion-euiFlexGroup-responsive-l-flexStart-stretch-row"
-        >
-          <div
-            class="euiFlexItem emotion-euiFlexItem-grow-1"
-          >
-            <div
-              class="euiText emotion-euiText-constrainedWidth-m"
-            >
-              <p>
-                test
-              </p>
-            </div>
-          </div>
-          <div
-            class="euiFlexItem emotion-euiFlexItem-growZero"
-          >
-            <div
-              class="euiFlexGroup emotion-euiFlexGroup-wrap-l-flexStart-stretch-row"
-            >
-              <div
-                class="euiFlexItem emotion-euiFlexItem-growZero"
-              >
-                test
-              </div>
-            </div>
-          </div>
-        </div>
-      </div>
-    </header>
-    <div
-      class="euiPanel euiPanel--plain euiPageContent euiPageContent--borderRadiusNone emotion-euiPanel-grow-none-plain"
-      role="main"
-    >
-      <div
-        class="euiPageContentBody euiPageContentBody--paddingLarge euiPageContentBody--restrictWidth-default"
-      />
-    </div>
-  </div>
-</div>
-`;
-
-exports[`KibanaPageTemplate render custom empty prompt only 1`] = `
-<KibanaPageTemplateInner
-  isEmptyState={true}
->
-  <EuiEmptyPrompt
-    title={
-      <h1>
-        custom test
-      </h1>
-    }
-  />
-</KibanaPageTemplateInner>
-`;
-
-exports[`KibanaPageTemplate render custom empty prompt with page header 1`] = `
-<KibanaPageTemplateInner
-  isEmptyState={true}
-  pageHeader={
-    Object {
-      "description": "test",
-      "iconType": "test",
-      "rightSideItems": Array [
-        "test",
-      ],
-      "title": "test",
-    }
-  }
->
-  <EuiEmptyPrompt
-    title={
-      <h1>
-        custom test
-      </h1>
-    }
-  />
-</KibanaPageTemplateInner>
-`;
-
-exports[`KibanaPageTemplate render default empty prompt 1`] = `
-<KibanaPageTemplateInner
-  isEmptyState={true}
-  pageHeader={
-    Object {
-      "description": "test",
-      "iconType": "test",
-      "rightSideItems": Array [
-        "test",
-      ],
-      "title": "test",
-    }
-  }
-/>
-`;
-
-exports[`KibanaPageTemplate render noDataContent 1`] = `
-<WithSolutionNavBar(NoDataConfigPage)
-  noDataConfig={
-    Object {
-      "actions": Object {
-        "beats": Object {},
-        "custom": Object {},
-        "elasticAgent": Object {},
-      },
-      "docsLink": "test",
-      "solution": "Elastic",
-    }
-  }
-  solutionNav={
-    Object {
-      "icon": "solution",
-      "items": Array [
-        Object {
-          "id": "1",
-          "items": Array [
-            Object {
-              "id": "1.1",
-              "items": undefined,
-              "name": "Ingest Node Pipelines",
-              "tabIndex": undefined,
-            },
-            Object {
-              "id": "1.2",
-              "items": undefined,
-              "name": "Logstash Pipelines",
-              "tabIndex": undefined,
-            },
-            Object {
-              "id": "1.3",
-              "items": undefined,
-              "name": "Beats Central Management",
-              "tabIndex": undefined,
-            },
-          ],
-          "name": "Ingest",
-          "tabIndex": undefined,
-        },
-        Object {
-          "id": "2",
-          "items": Array [
-            Object {
-              "id": "2.1",
-              "items": undefined,
-              "name": "Index Management",
-              "tabIndex": undefined,
-            },
-            Object {
-              "id": "2.2",
-              "items": undefined,
-              "name": "Index Lifecycle Policies",
-              "tabIndex": undefined,
-            },
-            Object {
-              "id": "2.3",
-              "items": undefined,
-              "name": "Snapshot and Restore",
-              "tabIndex": undefined,
-            },
-          ],
-          "name": "Data",
-          "tabIndex": undefined,
-        },
-      ],
-      "name": "Solution",
-    }
-  }
-/>
-`;
-
-exports[`KibanaPageTemplate render solutionNav 1`] = `
-<div
-  class="euiPage euiPageTemplate kbnPageTemplate emotion-euiPage-row-grow"
-  style="min-height:460px"
->
-  <div
-    class="euiPageSideBar euiPageSideBar--sticky kbnPageTemplate__pageSideBar"
-  >
-    <nav
-      aria-hidden="false"
-      aria-labelledby="euiSideNav_generated-id_heading"
-      class="euiSideNav kbnPageTemplateSolutionNav"
-    >
-      <h2
-        class="euiTitle euiSideNav__heading emotion-euiTitle-xs"
-        id="euiSideNav_generated-id_heading"
-      >
-        <div
-          aria-label="Solution"
-          class="euiAvatar euiAvatar--m euiAvatar--user kbnPageTemplateSolutionNavAvatar emotion-euiAvatar-user-m-uppercase-plain"
-          role="img"
-          title="Solution"
-        >
-          <span
-            class="euiAvatar__icon"
-            data-euiicon-type="solution"
-          />
-        </div>
-        <strong>
-          Solution
-        </strong>
-      </h2>
-      <div
-        class="euiSideNav__content euiSideNav__contentMobile-xs euiSideNav__contentMobile-s"
-        id="euiSideNav_generated-id_content"
-      >
-        <div
-          class="euiSideNavItem euiSideNavItem--root euiSideNavItem--hasChildItems"
-        >
-          <div
-            class="euiSideNavItemButton"
-          >
-            <span
-              class="euiSideNavItemButton__content"
-            >
-              <span
-                class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-              >
-                Ingest
-              </span>
-            </span>
-          </div>
-          <div
-            class="euiSideNavItem__items"
-          >
-            <div
-              class="euiSideNavItem euiSideNavItem--trunk"
-            >
-              <div
-                class="euiSideNavItemButton"
-              >
-                <span
-                  class="euiSideNavItemButton__content"
-                >
-                  <span
-                    class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-                  >
-                    Ingest Node Pipelines
-                  </span>
-                </span>
-              </div>
-            </div>
-            <div
-              class="euiSideNavItem euiSideNavItem--trunk"
-            >
-              <div
-                class="euiSideNavItemButton"
-              >
-                <span
-                  class="euiSideNavItemButton__content"
-                >
-                  <span
-                    class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-                  >
-                    Logstash Pipelines
-                  </span>
-                </span>
-              </div>
-            </div>
-            <div
-              class="euiSideNavItem euiSideNavItem--trunk"
-            >
-              <div
-                class="euiSideNavItemButton"
-              >
-                <span
-                  class="euiSideNavItemButton__content"
-                >
-                  <span
-                    class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-                  >
-                    Beats Central Management
-                  </span>
-                </span>
-              </div>
-            </div>
-          </div>
-        </div>
-        <div
-          class="euiSideNavItem euiSideNavItem--root euiSideNavItem--hasChildItems"
-        >
-          <div
-            class="euiSideNavItemButton"
-          >
-            <span
-              class="euiSideNavItemButton__content"
-            >
-              <span
-                class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-              >
-                Data
-              </span>
-            </span>
-          </div>
-          <div
-            class="euiSideNavItem__items"
-          >
-            <div
-              class="euiSideNavItem euiSideNavItem--trunk"
-            >
-              <div
-                class="euiSideNavItemButton"
-              >
-                <span
-                  class="euiSideNavItemButton__content"
-                >
-                  <span
-                    class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-                  >
-                    Index Management
-                  </span>
-                </span>
-              </div>
-            </div>
-            <div
-              class="euiSideNavItem euiSideNavItem--trunk"
-            >
-              <div
-                class="euiSideNavItemButton"
-              >
-                <span
-                  class="euiSideNavItemButton__content"
-                >
-                  <span
-                    class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-                  >
-                    Index Lifecycle Policies
-                  </span>
-                </span>
-              </div>
-            </div>
-            <div
-              class="euiSideNavItem euiSideNavItem--trunk"
-            >
-              <div
-                class="euiSideNavItemButton"
-              >
-                <span
-                  class="euiSideNavItemButton__content"
-                >
-                  <span
-                    class="euiSideNavItemButton__label euiSideNavItemButton__label--truncated"
-                  >
-                    Snapshot and Restore
-                  </span>
-                </span>
-              </div>
-            </div>
-          </div>
-        </div>
-      </div>
-    </nav>
-    <button
-      aria-label="Collapse side navigation"
-      class="euiButtonIcon kbnPageTemplateSolutionNavCollapseButton emotion-euiButtonIcon-s-empty-text"
-      title="Collapse side navigation"
-      type="button"
-    >
-      <span
-        aria-hidden="true"
-        class="euiButtonIcon__icon"
-        color="inherit"
-        data-euiicon-type="menuLeft"
-      />
-    </button>
-  </div>
-  <div
-    class="euiPanel euiPanel--plain euiPageBody emotion-euiPanel-grow-none-plain-hasShadow-euiPageBody"
-  >
-    <div
-      class="euiPageBody emotion-euiPageBody-l"
-    >
-      <header
-        class="euiPageHeader emotion-euiPageHeader-headerMargin"
-        title="test"
-      >
-        <div
-          class="euiPageHeaderContent emotion-euiPageHeaderContent-border-l"
-          style="max-width:1200px"
-        >
-          <div
-            class="euiFlexGroup euiPageHeaderContent__top emotion-euiFlexGroup-responsive-l-flexStart-stretch-row"
-          >
-            <div
-              class="euiFlexItem emotion-euiFlexItem-grow-1"
-            >
-              <div
-                class="euiText emotion-euiText-constrainedWidth-m"
-              >
-                <p>
-                  test
-                </p>
-              </div>
-            </div>
-            <div
-              class="euiFlexItem emotion-euiFlexItem-growZero"
-            >
-              <div
-                class="euiFlexGroup emotion-euiFlexGroup-wrap-l-flexStart-stretch-row"
-              >
-                <div
-                  class="euiFlexItem emotion-euiFlexItem-growZero"
-                >
-                  test
-                </div>
-              </div>
-            </div>
-          </div>
-        </div>
-      </header>
-      <div
-        class="euiPanel euiPanel--transparent euiPageContent euiPageContent--borderRadiusNone emotion-euiPanel-grow-none-transparent"
-        role="main"
-      >
-        <div
-          class="euiPageContentBody euiPageContentBody--restrictWidth-default"
-        />
-      </div>
-    </div>
-  </div>
-</div>
-`;
diff --git a/src/plugins/kibana_react/public/page_template/index.ts b/src/plugins/kibana_react/public/page_template/index.ts
index fda644a284797..65a5db433593a 100644
--- a/src/plugins/kibana_react/public/page_template/index.ts
+++ b/src/plugins/kibana_react/public/page_template/index.ts
@@ -6,9 +6,5 @@
  * Side Public License, v 1.
  */
 
-export type { KibanaPageTemplateProps } from './page_template';
-export { KibanaPageTemplate } from './page_template';
 export { KibanaPageTemplateSolutionNavAvatar, KibanaPageTemplateSolutionNav } from './solution_nav';
 export * from './no_data_page';
-export { withSolutionNav } from './with_solution_nav';
-export { NO_DATA_PAGE_MAX_WIDTH, NO_DATA_PAGE_TEMPLATE_PROPS } from './util';
diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/index.ts b/src/plugins/kibana_react/public/page_template/no_data_page/index.ts
index b5a11722dd397..55661ad6f14f7 100644
--- a/src/plugins/kibana_react/public/page_template/no_data_page/index.ts
+++ b/src/plugins/kibana_react/public/page_template/no_data_page/index.ts
@@ -8,4 +8,3 @@
 
 export * from './no_data_page';
 export * from './no_data_card';
-export * from './no_data_config_page';
diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx
deleted file mode 100644
index 0bdde40021398..0000000000000
--- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx
+++ /dev/null
@@ -1,9 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-export { NoDataConfigPage, NoDataConfigPageWithSolutionNavBar } from './no_data_config_page';
diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx
deleted file mode 100644
index cae591f571c79..0000000000000
--- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-import { EuiPageTemplate_Deprecated as EuiPageTemplate } from '@elastic/eui';
-import React from 'react';
-import { NoDataPage } from '../no_data_page';
-import { withSolutionNav } from '../../with_solution_nav';
-import { KibanaPageTemplateProps } from '../../page_template';
-import { getClasses, NO_DATA_PAGE_TEMPLATE_PROPS } from '../../util';
-
-export const NoDataConfigPage = (props: KibanaPageTemplateProps) => {
-  const { className, noDataConfig, ...rest } = props;
-
-  if (!noDataConfig) {
-    return null;
-  }
-
-  const template = NO_DATA_PAGE_TEMPLATE_PROPS.template;
-  const classes = getClasses(template, className);
-
-  return (
-    <EuiPageTemplate
-      data-test-subj={props['data-test-subj']}
-      template={template}
-      className={classes}
-      {...rest}
-      {...NO_DATA_PAGE_TEMPLATE_PROPS}
-    >
-      <NoDataPage {...noDataConfig} />
-    </EuiPageTemplate>
-  );
-};
-
-export const NoDataConfigPageWithSolutionNavBar = withSolutionNav(NoDataConfigPage);
diff --git a/src/plugins/kibana_react/public/page_template/page_template.scss b/src/plugins/kibana_react/public/page_template/page_template.scss
deleted file mode 100644
index d94daec56235f..0000000000000
--- a/src/plugins/kibana_react/public/page_template/page_template.scss
+++ /dev/null
@@ -1,22 +0,0 @@
-.kbnPageTemplate__pageSideBar {
-  overflow: hidden;
-  // Temporary hack till the sizing is changed directly in EUI
-  min-width: 248px;
-
-  @include euiCanAnimate {
-    transition: min-width $euiAnimSpeedFast $euiAnimSlightResistance;
-  }
-
-  &.kbnPageTemplate__pageSideBar--shrink {
-    min-width: $euiSizeXXL;
-  }
-
-  .kbnPageTemplate--centeredBody & {
-    border-bottom: $euiBorderThin;
-
-    @include euiBreakpoint('m', 'l', 'xl') {
-      border-bottom: none;
-      border-right: $euiBorderThin;
-    }
-  }
-}
diff --git a/src/plugins/kibana_react/public/page_template/page_template.test.tsx b/src/plugins/kibana_react/public/page_template/page_template.test.tsx
deleted file mode 100644
index aff6082902a34..0000000000000
--- a/src/plugins/kibana_react/public/page_template/page_template.test.tsx
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import React from 'react';
-import { shallow, render } from 'enzyme';
-import { KibanaPageTemplate, KibanaPageTemplateProps } from './page_template';
-import { EuiEmptyPrompt } from '@elastic/eui';
-import { KibanaPageTemplateSolutionNavProps } from './solution_nav';
-
-const navItems: KibanaPageTemplateSolutionNavProps['items'] = [
-  {
-    name: 'Ingest',
-    id: '1',
-    items: [
-      {
-        name: 'Ingest Node Pipelines',
-        id: '1.1',
-      },
-      {
-        name: 'Logstash Pipelines',
-        id: '1.2',
-      },
-      {
-        name: 'Beats Central Management',
-        id: '1.3',
-      },
-    ],
-  },
-  {
-    name: 'Data',
-    id: '2',
-    items: [
-      {
-        name: 'Index Management',
-        id: '2.1',
-      },
-      {
-        name: 'Index Lifecycle Policies',
-        id: '2.2',
-      },
-      {
-        name: 'Snapshot and Restore',
-        id: '2.3',
-      },
-    ],
-  },
-];
-
-const noDataConfig: KibanaPageTemplateProps['noDataConfig'] = {
-  solution: 'Elastic',
-  actions: {
-    elasticAgent: {},
-    beats: {},
-    custom: {},
-  },
-  docsLink: 'test',
-};
-
-describe('KibanaPageTemplate', () => {
-  test('render default empty prompt', () => {
-    const component = shallow(
-      <KibanaPageTemplate
-        isEmptyState={true}
-        pageHeader={{
-          iconType: 'test',
-          title: 'test',
-          description: 'test',
-          rightSideItems: ['test'],
-        }}
-      />
-    );
-    expect(component).toMatchSnapshot();
-  });
-
-  test('render custom empty prompt only', () => {
-    const component = shallow(
-      <KibanaPageTemplate isEmptyState={true}>
-        <EuiEmptyPrompt title={<h1>custom test</h1>} />
-      </KibanaPageTemplate>
-    );
-    expect(component).toMatchSnapshot();
-  });
-
-  test('render custom empty prompt with page header', () => {
-    const component = shallow(
-      <KibanaPageTemplate
-        isEmptyState={true}
-        pageHeader={{
-          iconType: 'test',
-          title: 'test',
-          description: 'test',
-          rightSideItems: ['test'],
-        }}
-      >
-        <EuiEmptyPrompt title={<h1>custom test</h1>} />
-      </KibanaPageTemplate>
-    );
-    expect(component).toMatchSnapshot();
-  });
-
-  test('render basic template', () => {
-    const component = render(
-      <KibanaPageTemplate
-        pageHeader={{
-          iconType: 'test',
-          title: 'test',
-          description: 'test',
-          rightSideItems: ['test'],
-        }}
-      />
-    );
-    expect(component).toMatchSnapshot();
-  });
-
-  test('render solutionNav', () => {
-    const component = render(
-      <KibanaPageTemplate
-        pageHeader={{
-          iconType: 'test',
-          title: 'test',
-          description: 'test',
-          rightSideItems: ['test'],
-        }}
-        solutionNav={{
-          name: 'Solution',
-          icon: 'solution',
-          items: navItems,
-        }}
-      />
-    );
-    expect(component).toMatchSnapshot();
-    expect(component.find('div.kbnPageTemplate__pageSideBar').length).toBe(1);
-  });
-
-  test('render noDataContent', () => {
-    const component = shallow(
-      <KibanaPageTemplate
-        pageHeader={{
-          iconType: 'test',
-          title: 'test',
-          description: 'test',
-          rightSideItems: ['test'],
-        }}
-        solutionNav={{
-          name: 'Solution',
-          icon: 'solution',
-          items: navItems,
-        }}
-        noDataConfig={noDataConfig}
-      />
-    );
-    expect(component).toMatchSnapshot();
-  });
-
-  test('render sidebar classes', () => {
-    const component = shallow(
-      <KibanaPageTemplate
-        solutionNav={{
-          name: 'Solution',
-          icon: 'solution',
-          items: navItems,
-        }}
-        pageSideBarProps={{ className: 'customClass' }}
-      />
-    );
-    expect(component.html().includes('kbnPageTemplate__pageSideBar customClass')).toBe(true);
-  });
-});
diff --git a/src/plugins/kibana_react/public/page_template/page_template.tsx b/src/plugins/kibana_react/public/page_template/page_template.tsx
deleted file mode 100644
index 42ba9d1873587..0000000000000
--- a/src/plugins/kibana_react/public/page_template/page_template.tsx
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import './page_template.scss';
-
-import React, { FunctionComponent } from 'react';
-import { EuiPageTemplateProps_Deprecated } from '@elastic/eui';
-import { KibanaPageTemplateSolutionNavProps } from './solution_nav';
-
-import {
-  NoDataPageProps,
-  NoDataConfigPage,
-  NoDataConfigPageWithSolutionNavBar,
-} from './no_data_page';
-import { KibanaPageTemplateInner, KibanaPageTemplateWithSolutionNav } from './page_template_inner';
-
-/**
- * A thin wrapper around EuiPageTemplate with a few Kibana specific additions
- * @deprecated Use `KibanaPageTemplateProps` from `@kbn/shared-ux-page-kibana-template-types`.
- */
-export type KibanaPageTemplateProps = EuiPageTemplateProps_Deprecated & {
-  /**
-   * Changes the template type depending on other props provided.
-   * With `pageHeader` only: Uses `centeredBody` and fills an EuiEmptyPrompt with `pageHeader` info.
-   * With `children` only: Uses `centeredBody`
-   * With `pageHeader` and `children`: Uses `centeredContent`
-   */
-  isEmptyState?: boolean;
-  /**
-   * Quick creation of EuiSideNav. Hooks up mobile instance too
-   */
-  solutionNav?: KibanaPageTemplateSolutionNavProps;
-  /**
-   * Accepts a configuration object, that when provided, ignores pageHeader and children and instead
-   * displays Agent, Beats, and custom cards to direct users to the right ingest location
-   */
-  noDataConfig?: NoDataPageProps;
-};
-
-/** @deprecated Use `KibanaPageTemplate` from `@kbn/shared-ux-page-kibana-template`. */
-export const KibanaPageTemplate: FunctionComponent<KibanaPageTemplateProps> = ({
-  template,
-  className,
-  children,
-  solutionNav,
-  noDataConfig,
-  ...rest
-}) => {
-  /**
-   * If passing the custom template of `noDataConfig`
-   */
-  if (noDataConfig && solutionNav) {
-    return (
-      <NoDataConfigPageWithSolutionNavBar
-        data-test-subj={rest['data-test-subj']}
-        className={className}
-        noDataConfig={noDataConfig}
-        solutionNav={solutionNav}
-      />
-    );
-  }
-
-  if (noDataConfig) {
-    return (
-      <NoDataConfigPage
-        data-test-subj={rest['data-test-subj']}
-        className={className}
-        noDataConfig={noDataConfig}
-      />
-    );
-  }
-
-  if (solutionNav) {
-    return (
-      <KibanaPageTemplateWithSolutionNav
-        template={template}
-        className={className}
-        solutionNav={solutionNav}
-        children={children}
-        {...rest}
-      />
-    );
-  }
-
-  return (
-    <KibanaPageTemplateInner
-      template={template}
-      className={className}
-      children={children}
-      {...rest}
-    />
-  );
-};
diff --git a/src/plugins/kibana_react/public/page_template/page_template_inner.tsx b/src/plugins/kibana_react/public/page_template/page_template_inner.tsx
deleted file mode 100644
index 001cea5c26a23..0000000000000
--- a/src/plugins/kibana_react/public/page_template/page_template_inner.tsx
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import React, { FunctionComponent } from 'react';
-
-import { EuiEmptyPrompt, EuiPageTemplate_Deprecated as EuiPageTemplate } from '@elastic/eui';
-import { withSolutionNav } from './with_solution_nav';
-import { KibanaPageTemplateProps } from './page_template';
-import { getClasses } from './util';
-
-type Props = KibanaPageTemplateProps;
-
-/**
- * A thin wrapper around EuiPageTemplate with a few Kibana specific additions
- */
-export const KibanaPageTemplateInner: FunctionComponent<Props> = ({
-  template,
-  className,
-  pageHeader,
-  children,
-  isEmptyState,
-  ...rest
-}) => {
-  /**
-   * An easy way to create the right content for empty pages
-   */
-  const emptyStateDefaultTemplate = 'centeredBody';
-  if (isEmptyState && pageHeader && !children) {
-    template = template ?? emptyStateDefaultTemplate;
-    const { iconType, pageTitle, description, rightSideItems } = pageHeader;
-    pageHeader = undefined;
-    children = (
-      <EuiEmptyPrompt
-        iconType={iconType}
-        iconColor={''} // This is likely a solution or app logo, so keep it multi-color
-        title={pageTitle ? <h1>{pageTitle}</h1> : undefined}
-        body={description ? <p>{description}</p> : undefined}
-        actions={rightSideItems}
-      />
-    );
-  } else if (isEmptyState && pageHeader && children) {
-    template = template ?? 'centeredContent';
-  } else if (isEmptyState && !pageHeader) {
-    template = template ?? emptyStateDefaultTemplate;
-  }
-
-  const classes = getClasses(template, className);
-
-  return (
-    <EuiPageTemplate template={template} className={classes} pageHeader={pageHeader} {...rest}>
-      {children}
-    </EuiPageTemplate>
-  );
-};
-
-export const KibanaPageTemplateWithSolutionNav = withSolutionNav(KibanaPageTemplateInner);
diff --git a/src/plugins/kibana_react/public/page_template/util/constants.ts b/src/plugins/kibana_react/public/page_template/util/constants.ts
deleted file mode 100644
index 159a6d0d8d4c1..0000000000000
--- a/src/plugins/kibana_react/public/page_template/util/constants.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { KibanaPageTemplateProps } from '../page_template';
-
-export const NO_DATA_PAGE_MAX_WIDTH = 950;
-
-export const NO_DATA_PAGE_TEMPLATE_PROPS: KibanaPageTemplateProps = {
-  restrictWidth: NO_DATA_PAGE_MAX_WIDTH,
-  template: 'centeredBody',
-  pageContentProps: {
-    hasShadow: false,
-    color: 'transparent',
-  },
-};
diff --git a/src/plugins/kibana_react/public/page_template/util/index.ts b/src/plugins/kibana_react/public/page_template/util/index.ts
index adfefdf834566..06edc43d70d57 100644
--- a/src/plugins/kibana_react/public/page_template/util/index.ts
+++ b/src/plugins/kibana_react/public/page_template/util/index.ts
@@ -7,4 +7,3 @@
  */
 
 export { getClasses } from './presentation';
-export * from './constants';
diff --git a/src/plugins/kibana_react/public/page_template/with_solution_nav.tsx b/src/plugins/kibana_react/public/page_template/with_solution_nav.tsx
deleted file mode 100644
index 842573b9d8de4..0000000000000
--- a/src/plugins/kibana_react/public/page_template/with_solution_nav.tsx
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import React, { ComponentType, useState } from 'react';
-import classNames from 'classnames';
-import { useIsWithinBreakpoints } from '@elastic/eui';
-import { EuiPageSideBarProps_Deprecated as EuiPageSideBarProps } from '@elastic/eui/src/components/page/page_side_bar';
-import { KibanaPageTemplateSolutionNav, KibanaPageTemplateSolutionNavProps } from './solution_nav';
-import { KibanaPageTemplateProps } from '.';
-
-// https://reactjs.org/docs/higher-order-components.html#convention-wrap-the-display-name-for-easy-debugging
-function getDisplayName(Component: ComponentType<any>) {
-  return Component.displayName || Component.name || 'UnnamedComponent';
-}
-
-type SolutionNavProps = KibanaPageTemplateProps & {
-  solutionNav: KibanaPageTemplateSolutionNavProps;
-};
-
-const SOLUTION_NAV_COLLAPSED_KEY = 'solutionNavIsCollapsed';
-
-export const withSolutionNav = (WrappedComponent: ComponentType<KibanaPageTemplateProps>) => {
-  const WithSolutionNav = (props: SolutionNavProps) => {
-    const isMediumBreakpoint = useIsWithinBreakpoints(['m']);
-    const isLargerBreakpoint = useIsWithinBreakpoints(['l', 'xl']);
-    const [isSideNavOpenOnDesktop, setisSideNavOpenOnDesktop] = useState(
-      !JSON.parse(String(localStorage.getItem(SOLUTION_NAV_COLLAPSED_KEY)))
-    );
-    const { solutionNav, ...propagatedProps } = props;
-    const { children, isEmptyState, template } = propagatedProps;
-    const toggleOpenOnDesktop = () => {
-      setisSideNavOpenOnDesktop(!isSideNavOpenOnDesktop);
-      // Have to store it as the opposite of the default we want
-      localStorage.setItem(SOLUTION_NAV_COLLAPSED_KEY, JSON.stringify(isSideNavOpenOnDesktop));
-    };
-    const sideBarClasses = classNames(
-      'kbnPageTemplate__pageSideBar',
-      {
-        'kbnPageTemplate__pageSideBar--shrink':
-          isMediumBreakpoint || (isLargerBreakpoint && !isSideNavOpenOnDesktop),
-      },
-      props.pageSideBarProps?.className
-    );
-
-    const templateToUse = isEmptyState && !template ? 'centeredContent' : template;
-
-    const pageSideBar = (
-      <KibanaPageTemplateSolutionNav
-        isOpenOnDesktop={isSideNavOpenOnDesktop}
-        onCollapse={toggleOpenOnDesktop}
-        {...solutionNav}
-      />
-    );
-    const pageSideBarProps = {
-      paddingSize: 'none',
-      ...props.pageSideBarProps,
-      className: sideBarClasses,
-    } as EuiPageSideBarProps; // needed because for some reason 'none' is not recognized as a valid value for paddingSize
-    return (
-      <WrappedComponent
-        {...propagatedProps}
-        pageSideBar={pageSideBar}
-        pageSideBarProps={pageSideBarProps}
-        template={templateToUse}
-      >
-        {children}
-      </WrappedComponent>
-    );
-  };
-  WithSolutionNav.displayName = `WithSolutionNavBar(${getDisplayName(WrappedComponent)})`;
-  return WithSolutionNav;
-};
diff --git a/x-pack/plugins/osquery/public/components/empty_state.tsx b/x-pack/plugins/osquery/public/components/empty_state.tsx
index 099e9e22b1f64..18f5afe8eaf5f 100644
--- a/x-pack/plugins/osquery/public/components/empty_state.tsx
+++ b/x-pack/plugins/osquery/public/components/empty_state.tsx
@@ -9,7 +9,7 @@ import React, { useCallback, useMemo } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { EuiButton } from '@elastic/eui';
 
-import { KibanaPageTemplate } from '@kbn/kibana-react-plugin/public';
+import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template';
 import { INTEGRATIONS_PLUGIN_ID } from '@kbn/fleet-plugin/common';
 import { pagePathGetters } from '@kbn/fleet-plugin/public';
 import { isModifiedEvent, isLeftClickEvent, useKibana } from '../common/lib/kibana';
diff --git a/x-pack/plugins/osquery/tsconfig.json b/x-pack/plugins/osquery/tsconfig.json
index 6cd1086b8a850..d2344a2581df8 100644
--- a/x-pack/plugins/osquery/tsconfig.json
+++ b/x-pack/plugins/osquery/tsconfig.json
@@ -77,5 +77,6 @@
     "@kbn/core-saved-objects-server",
     "@kbn/monaco",
     "@kbn/io-ts-utils",
+    "@kbn/shared-ux-page-kibana-template",
   ]
 }

From 18555d5dc435c29e51a5b72dff02f5c149a81c17 Mon Sep 17 00:00:00 2001
From: Bruno Mercier Costa <94469565+emrcbrn@users.noreply.github.com>
Date: Mon, 25 Sep 2023 16:19:27 +0200
Subject: [PATCH 06/27] [DOCS] Fix typo in Upgrade Assistant docs (#167031)

Co-authored-by: James Rodewig <james.rodewig@elastic.co>
---
 docs/management/upgrade-assistant.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/management/upgrade-assistant.asciidoc b/docs/management/upgrade-assistant.asciidoc
index a44afa4474a7b..ed9093ded2846 100644
--- a/docs/management/upgrade-assistant.asciidoc
+++ b/docs/management/upgrade-assistant.asciidoc
@@ -26,4 +26,4 @@ The Upgrade assistant pulls information about deprecations from the following so
 * Elasticsearch deprecation logs
 * Kibana deprecations API
 
-For more information about the API's the Upgraed assistant provides, refer to <<upgrade-assistant-api, Upgrade Assistant APIs>>.
+For more information about Upgrade Assistant APIs, refer to <<upgrade-assistant-api, Upgrade Assistant APIs>>.

From 2dcabd2b82221302615b1a4a09e2a751b51d53c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cau=C3=AA=20Marcondes?=
 <55978943+cauemarcondes@users.noreply.github.com>
Date: Mon, 25 Sep 2023 16:33:03 +0100
Subject: [PATCH 07/27] [Profiling] TS fixes (#167129)

I ran the ts check on all three projects owned by profiling (`profiling`
/ `profiling_data_access` / `kbn_profiling-utils`) and all passed now.

---------

Co-authored-by: Thomas Watson <watson@elastic.co>
---
 .../components/shared/transaction_action_menu/sections.ts       | 2 +-
 .../shared/transaction_action_menu/transaction_action_menu.tsx  | 2 +-
 x-pack/plugins/observability_shared/public/index.ts             | 1 +
 x-pack/plugins/observability_shared/public/plugin.ts            | 1 +
 x-pack/plugins/profiling/public/index.tsx                       | 2 --
 .../profiling_data_access/server/services/status/index.ts       | 2 +-
 6 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts
index c17bedeb8635a..04b8f7f41bdf1 100644
--- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts
+++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts
@@ -12,9 +12,9 @@ import { isEmpty, pickBy } from 'lodash';
 import moment from 'moment';
 import url from 'url';
 import type { InfraLocators } from '@kbn/infra-plugin/common/locators';
-import type { ProfilingLocators } from '@kbn/profiling-plugin/public';
 import { LocatorPublic } from '@kbn/share-plugin/common';
 import { AllDatasetsLocatorParams } from '@kbn/deeplinks-observability/locators';
+import type { ProfilingLocators } from '@kbn/observability-shared-plugin/public';
 import { Environment } from '../../../../common/environment_rt';
 import type { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import { getDiscoverHref } from '../links/discover_links/discover_link';
diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx
index 162fdf8e90b85..4dcd10a3ea540 100644
--- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx
+++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx
@@ -18,7 +18,6 @@ import {
   SectionSubtitle,
   SectionTitle,
 } from '@kbn/observability-shared-plugin/public';
-import { ProfilingLocators } from '@kbn/profiling-plugin/public';
 import React, { useState } from 'react';
 import { useLocation } from 'react-router-dom';
 import useAsync from 'react-use/lib/useAsync';
@@ -26,6 +25,7 @@ import {
   AllDatasetsLocatorParams,
   ALL_DATASETS_LOCATOR_ID,
 } from '@kbn/deeplinks-observability/locators';
+import type { ProfilingLocators } from '@kbn/observability-shared-plugin/public';
 import { useAnyOfApmParams } from '../../../hooks/use_apm_params';
 import { ApmFeatureFlagName } from '../../../../common/apm_feature_flags';
 import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
diff --git a/x-pack/plugins/observability_shared/public/index.ts b/x-pack/plugins/observability_shared/public/index.ts
index 7e5fc02d0a0b5..a35cdce8f85e1 100644
--- a/x-pack/plugins/observability_shared/public/index.ts
+++ b/x-pack/plugins/observability_shared/public/index.ts
@@ -10,6 +10,7 @@ export type {
   ObservabilitySharedPlugin,
   ObservabilitySharedPluginSetup,
   ObservabilitySharedPluginStart,
+  ProfilingLocators,
 } from './plugin';
 export const plugin = () => {
   return new ObservabilitySharedPlugin();
diff --git a/x-pack/plugins/observability_shared/public/plugin.ts b/x-pack/plugins/observability_shared/public/plugin.ts
index 550259184dfb1..faf8990622337 100644
--- a/x-pack/plugins/observability_shared/public/plugin.ts
+++ b/x-pack/plugins/observability_shared/public/plugin.ts
@@ -34,6 +34,7 @@ export interface ObservabilitySharedStart {
 
 export type ObservabilitySharedPluginSetup = ReturnType<ObservabilitySharedPlugin['setup']>;
 export type ObservabilitySharedPluginStart = ReturnType<ObservabilitySharedPlugin['start']>;
+export type ProfilingLocators = ObservabilitySharedPluginSetup['locators']['profiling'];
 
 export class ObservabilitySharedPlugin implements Plugin {
   private readonly navigationRegistry = createNavigationRegistry();
diff --git a/x-pack/plugins/profiling/public/index.tsx b/x-pack/plugins/profiling/public/index.tsx
index ea9dc15de9927..752635a54290b 100644
--- a/x-pack/plugins/profiling/public/index.tsx
+++ b/x-pack/plugins/profiling/public/index.tsx
@@ -13,5 +13,3 @@ export function plugin() {
 }
 
 export type { ProfilingPluginSetup, ProfilingPluginStart };
-
-export type ProfilingLocators = ProfilingPluginSetup['locators'];
diff --git a/x-pack/plugins/profiling_data_access/server/services/status/index.ts b/x-pack/plugins/profiling_data_access/server/services/status/index.ts
index 6aea8b1037bd6..31581140b12e0 100644
--- a/x-pack/plugins/profiling_data_access/server/services/status/index.ts
+++ b/x-pack/plugins/profiling_data_access/server/services/status/index.ts
@@ -12,7 +12,7 @@ import { getSetupState } from '../get_setup_state';
 import { RegisterServicesParams } from '../register_services';
 import { ProfilingSetupOptions, areResourcesSetup } from '../../../common/setup';
 
-interface HasSetupParams {
+export interface HasSetupParams {
   soClient: SavedObjectsClientContract;
   esClient: ElasticsearchClient;
   spaceId?: string;

From 4957d87a66129ee391b6bbaa2ffd3ef554d9bb54 Mon Sep 17 00:00:00 2001
From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com>
Date: Mon, 25 Sep 2023 10:39:15 -0500
Subject: [PATCH 08/27] [Transform] Improve loading behavior of Transform if
 stats takes too long or is not available (#166320)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
---
 x-pack/plugins/transform/common/constants.ts  |   1 +
 .../utils/create_stats_unknown_message.ts     |  22 ++
 .../public/app/common/transform_list.ts       |  16 +-
 .../public/app/common/transform_stats.test.ts |   4 +-
 .../public/app/common/transform_stats.ts      |   4 +-
 .../app/hooks/use_get_transform_stats.ts      |  21 ++
 .../public/app/hooks/use_get_transforms.ts    |  22 +-
 .../app/hooks/use_refresh_transform_list.ts   |   1 +
 .../action_delete/delete_action_name.test.tsx |   2 +
 .../action_delete/delete_action_name.tsx      |  74 ++++---
 .../action_delete/use_delete_action.tsx       |  15 +-
 .../discover_action_name.test.tsx             |   4 +-
 .../action_discover/discover_action_name.tsx  |   4 +-
 .../reauthorize_action_name.tsx               |   1 -
 .../use_reauthorize_action.tsx                |   2 +
 .../action_reset/reset_action_name.tsx        |  76 +++++--
 .../action_reset/use_reset_action.tsx         |  27 ++-
 .../schedule_now_action_name.tsx              |  10 +-
 .../use_schedule_now_action.tsx               |   5 +-
 .../action_start/start_action_name.tsx        |  16 +-
 .../action_start/use_start_action.tsx         |   4 +-
 .../action_stop/stop_action_name.tsx          |  98 +++++----
 .../action_stop/use_stop_action.tsx           |  12 +-
 .../transform_list/expanded_row.test.tsx      |   2 +-
 .../transform_list/expanded_row.tsx           | 196 +++++++++++-------
 .../expanded_row_health_pane.tsx              |   9 +-
 .../transform_list/transform_list.test.tsx    |   1 +
 .../transform_list/transform_list.tsx         |  22 +-
 .../transform_search_bar_filters.tsx          |   5 +-
 .../transform_list/transforms_stats_bar.tsx   |  10 +-
 .../transform_list/use_columns.test.tsx       |   2 +-
 .../components/transform_list/use_columns.tsx |  30 ++-
 .../transform_management_section.tsx          |  47 ++++-
 x-pack/plugins/transform/tsconfig.json        |   1 -
 34 files changed, 528 insertions(+), 238 deletions(-)
 create mode 100644 x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts

diff --git a/x-pack/plugins/transform/common/constants.ts b/x-pack/plugins/transform/common/constants.ts
index a1ba2d8277af9..ec26333a834a7 100644
--- a/x-pack/plugins/transform/common/constants.ts
+++ b/x-pack/plugins/transform/common/constants.ts
@@ -43,6 +43,7 @@ export const TRANSFORM_REACT_QUERY_KEYS = {
   GET_TRANSFORM_NODES: 'transform.get_transform_nodes',
   GET_TRANSFORM_AUDIT_MESSAGES: 'transform.get_transform_audit_messages',
   GET_TRANSFORM_STATS: 'transform.get_transform_stats',
+  GET_TRANSFORMS_STATS: 'transform.get_transforms_stats',
   GET_TRANSFORMS: 'transform.get_transforms',
   GET_TRANSFORMS_PREVIEW: 'transform.get_transforms_preview',
 } as const;
diff --git a/x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts b/x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts
new file mode 100644
index 0000000000000..9cfe3ccc38ee4
--- /dev/null
+++ b/x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export function createNoStatsTooltipMessage({
+  actionName,
+  count = 1,
+}: {
+  actionName: string;
+  count: number;
+}) {
+  return i18n.translate('xpack.transform.transformList.actionDisabledNoStatsTooltipMessage', {
+    defaultMessage:
+      '{actionName} is disabled because the status for {count, plural, one {this transform} other {some transforms}} is unavailable.',
+    values: { actionName, count },
+  });
+}
diff --git a/x-pack/plugins/transform/public/app/common/transform_list.ts b/x-pack/plugins/transform/public/app/common/transform_list.ts
index c8ddd32f6a8ff..e588ed917958f 100644
--- a/x-pack/plugins/transform/public/app/common/transform_list.ts
+++ b/x-pack/plugins/transform/public/app/common/transform_list.ts
@@ -20,10 +20,24 @@ export interface TransformListRow {
   id: TransformId;
   config: TransformConfigUnion;
   mode?: string; // added property on client side to allow filtering by this field
-  stats: TransformStats;
+  stats?: TransformStats;
   alerting_rules?: TransformHealthAlertRule[];
 }
 
+export type TransformListRowWithStats = TransformListRow & {
+  stats: TransformStats;
+};
+
+export function isTransformListRowWithStats(
+  arg: TransformListRow
+): arg is TransformListRowWithStats {
+  return arg.stats !== undefined;
+}
+
+export function missingTransformStats(items: TransformListRow[]) {
+  return items.some((i: TransformListRow) => !isTransformListRowWithStats(i));
+}
+
 // The single Action type is not exported as is
 // from EUI so we use that code to get the single
 // Action type from the array of actions.
diff --git a/x-pack/plugins/transform/public/app/common/transform_stats.test.ts b/x-pack/plugins/transform/public/app/common/transform_stats.test.ts
index 974764475eb5f..9213adba45e4d 100644
--- a/x-pack/plugins/transform/public/app/common/transform_stats.test.ts
+++ b/x-pack/plugins/transform/public/app/common/transform_stats.test.ts
@@ -18,7 +18,7 @@ const getRow = (statsId: string) => {
     // @ts-expect-error mock data does not actually match TransformListRow type
     ...(mockTransformListRow as TransformListRow),
     stats: {
-      ...(mockTransformStats.transforms as Array<TransformListRow['stats']>).find(
+      ...(mockTransformStats.transforms as Array<NonNullable<TransformListRow['stats']>>).find(
         (stats) => stats.id === statsId
       )!,
     },
@@ -47,7 +47,7 @@ describe('Transform: Transform stats.', () => {
     // that will be used by isCompletedBatchTransform()
     // followed by a call to isCompletedBatchTransform() itself
     // @ts-expect-error mock data is too loosely typed
-    const row = mockTransformListRow as TransformListRow;
+    const row = mockTransformListRow as Required<TransformListRow>;
     expect(row.stats.checkpointing.last.checkpoint === 1).toBe(true);
     expect(row.config.sync === undefined).toBe(true);
     expect(row.stats.state === TRANSFORM_STATE.STOPPED).toBe(true);
diff --git a/x-pack/plugins/transform/public/app/common/transform_stats.ts b/x-pack/plugins/transform/public/app/common/transform_stats.ts
index 7763e0712249e..0f7ad5ae98c3f 100644
--- a/x-pack/plugins/transform/public/app/common/transform_stats.ts
+++ b/x-pack/plugins/transform/public/app/common/transform_stats.ts
@@ -33,7 +33,9 @@ export function isCompletedBatchTransform(item: TransformItem) {
   // If `checkpoint=1`, `sync` is missing from the config and state is stopped,
   // then this is a completed batch transform.
   return (
-    item.stats.checkpointing.last.checkpoint === 1 &&
+    item.stats &&
+    item.config &&
+    item.stats.checkpointing?.last.checkpoint === 1 &&
     item.config.sync === undefined &&
     item.stats.state === TRANSFORM_STATE.STOPPED
   );
diff --git a/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts b/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts
index d2b9d32f25853..3c6b329dd880b 100644
--- a/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts
+++ b/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts
@@ -35,3 +35,24 @@ export const useGetTransformStats = (
     { enabled, refetchInterval }
   );
 };
+
+export const useGetTransformsStats = ({
+  enabled,
+  refetchInterval,
+}: {
+  enabled?: boolean;
+  refetchInterval?: number | false;
+}) => {
+  const { http } = useAppDependencies();
+
+  return useQuery<GetTransformsStatsResponseSchema, IHttpFetchError>(
+    [TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORMS_STATS],
+    ({ signal }) =>
+      http.get<GetTransformsStatsResponseSchema>(addInternalBasePath(`transforms/_stats`), {
+        version: '1',
+        asSystemRequest: true,
+        signal,
+      }),
+    { enabled, refetchInterval }
+  );
+};
diff --git a/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts b/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts
index f74f7a5774ded..f4afb03c4e4c0 100644
--- a/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts
+++ b/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts
@@ -11,14 +11,12 @@ import type { IHttpFetchError } from '@kbn/core-http-browser';
 import { isDefined } from '@kbn/ml-is-defined';
 
 import type { GetTransformsResponseSchema } from '../../../common/api_schemas/transforms';
-import type { GetTransformsStatsResponseSchema } from '../../../common/api_schemas/transforms_stats';
 import {
   addInternalBasePath,
   DEFAULT_REFRESH_INTERVAL_MS,
   TRANSFORM_REACT_QUERY_KEYS,
   TRANSFORM_MODE,
 } from '../../../common/constants';
-import { isTransformStats } from '../../../common/types/transform_stats';
 
 import { type TransformListRow } from '../common';
 import { useAppDependencies } from '../app_dependencies';
@@ -55,14 +53,6 @@ export const useGetTransforms = ({ enabled }: UseGetTransformsOptions = {}) => {
           signal,
         }
       );
-      const transformStats = await http.get<GetTransformsStatsResponseSchema>(
-        addInternalBasePath(`transforms/_stats`),
-        {
-          version: '1',
-          asSystemRequest: true,
-          signal,
-        }
-      );
 
       // There might be some errors with fetching certain transforms
       // For example, when task exists and is running but the config is deleted
@@ -81,21 +71,13 @@ export const useGetTransforms = ({ enabled }: UseGetTransformsOptions = {}) => {
       }
 
       update.transforms = transformConfigs.transforms.reduce((reducedtableRows, config) => {
-        const stats = transformStats.transforms.find((d) => config.id === d.id);
-
-        // A newly created transform might not have corresponding stats yet.
-        // If that's the case we just skip the transform and don't add it to the transform list yet.
-        if (!isTransformStats(stats)) {
-          return reducedtableRows;
-        }
-
-        // Table with expandable rows requires `id` on the outer most level
+        // Table with expandable rows requires `id` on the outermost level
         reducedtableRows.push({
           id: config.id,
           config,
           mode:
             typeof config.sync !== 'undefined' ? TRANSFORM_MODE.CONTINUOUS : TRANSFORM_MODE.BATCH,
-          stats,
+          stats: undefined,
           alerting_rules: config.alerting_rules,
         });
         return reducedtableRows;
diff --git a/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts b/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts
index 651886ba76f7b..3098946157305 100644
--- a/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts
+++ b/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts
@@ -15,6 +15,7 @@ export const useRefreshTransformList = () => {
   return () => {
     queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORM_NODES]);
     queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORMS]);
+    queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORMS_STATS]);
     queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORM_AUDIT_MESSAGES]);
   };
 };
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx
index 37daeeb75e138..bec2407ba631c 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx
@@ -16,9 +16,11 @@ jest.mock('../../../../app_dependencies');
 describe('Transform: Transform List Actions <DeleteAction />', () => {
   test('Minimal initialization', () => {
     const props: DeleteActionNameProps = {
+      items: [],
       canDeleteTransform: true,
       disabled: false,
       isBulkAction: false,
+      forceDisable: false,
     };
 
     const { container } = render(<DeleteActionName {...props} />);
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx
index 93e346e6c7a1d..7f68645cb2f18 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx
@@ -11,10 +11,12 @@ import { i18n } from '@kbn/i18n';
 
 import { EuiToolTip } from '@elastic/eui';
 
-import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants';
-import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message';
-
+import { missingTransformStats } from '../../../../common/transform_list';
+import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message';
+import { TransformCapabilities } from '../../../../../../common/types/capabilities';
 import { TransformListRow } from '../../../../common';
+import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message';
+import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants';
 
 export const deleteActionNameText = i18n.translate(
   'xpack.transform.transformList.deleteActionNameText',
@@ -24,45 +26,67 @@ export const deleteActionNameText = i18n.translate(
 );
 
 const transformCanNotBeDeleted = (i: TransformListRow) =>
+  i.stats &&
   !([TRANSFORM_STATE.STOPPED, TRANSFORM_STATE.FAILED] as TransformState[]).includes(i.stats.state);
 
 export const isDeleteActionDisabled = (items: TransformListRow[], forceDisable: boolean) => {
   const disabled = items.some(transformCanNotBeDeleted);
-  return forceDisable === true || disabled;
+
+  return forceDisable === true || disabled || missingTransformStats(items);
 };
 
 export interface DeleteActionNameProps {
+  items: TransformListRow[];
   canDeleteTransform: boolean;
   disabled: boolean;
   isBulkAction: boolean;
+  forceDisable: boolean;
 }
 
+export const getDeleteActionDisabledMessage = ({
+  items,
+  canDeleteTransform,
+  forceDisable,
+}: {
+  items: TransformListRow[];
+  canDeleteTransform: TransformCapabilities['canDeleteTransform'];
+  forceDisable: boolean;
+}) => {
+  const isBulkAction = items.length > 1;
+
+  if (missingTransformStats(items)) {
+    return createNoStatsTooltipMessage({
+      actionName: deleteActionNameText,
+      count: items.length,
+    });
+  }
+
+  if (!canDeleteTransform) {
+    return createCapabilityFailureMessage('canDeleteTransform');
+  }
+
+  const disabled = items.some(transformCanNotBeDeleted);
+
+  if (disabled) {
+    return isBulkAction === true
+      ? i18n.translate('xpack.transform.transformList.deleteBulkActionDisabledToolTipContent', {
+          defaultMessage: 'One or more selected transforms must be stopped in order to be deleted.',
+        })
+      : i18n.translate('xpack.transform.transformList.deleteActionDisabledToolTipContent', {
+          defaultMessage: 'Stop the transform in order to delete it.',
+        });
+  }
+};
+
 export const DeleteActionName: FC<DeleteActionNameProps> = ({
+  items,
   canDeleteTransform,
   disabled,
   isBulkAction,
+  forceDisable,
 }) => {
-  const bulkDeleteButtonDisabledText = i18n.translate(
-    'xpack.transform.transformList.deleteBulkActionDisabledToolTipContent',
-    {
-      defaultMessage: 'One or more selected transforms must be stopped in order to be deleted.',
-    }
-  );
-  const deleteButtonDisabledText = i18n.translate(
-    'xpack.transform.transformList.deleteActionDisabledToolTipContent',
-    {
-      defaultMessage: 'Stop the transform in order to delete it.',
-    }
-  );
-
-  if (disabled || !canDeleteTransform) {
-    let content;
-    if (disabled) {
-      content = isBulkAction ? bulkDeleteButtonDisabledText : deleteButtonDisabledText;
-    } else {
-      content = createCapabilityFailureMessage('canDeleteTransform');
-    }
-
+  const content = getDeleteActionDisabledMessage({ items, canDeleteTransform, forceDisable });
+  if (content) {
     return (
       <EuiToolTip position="top" content={content}>
         <>{deleteActionNameText}</>
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx
index 5996e271604e3..1942a9a9dc8b7 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx
@@ -7,9 +7,10 @@
 
 import React, { useMemo, useState } from 'react';
 
+import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 
-import { TransformListAction, TransformListRow } from '../../../../common';
+import type { TransformListAction, TransformListRow } from '../../../../common';
 import {
   useDeleteIndexAndTargetIndex,
   useDeleteTransforms,
@@ -33,7 +34,7 @@ export const useDeleteAction = (forceDisable: boolean) => {
 
   const isBulkAction = items.length > 1;
   const shouldForceDelete = useMemo(
-    () => items.some((i: TransformListRow) => i.stats.state === TRANSFORM_STATE.FAILED),
+    () => items.some((i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.FAILED),
     [items]
   );
 
@@ -59,10 +60,10 @@ export const useDeleteAction = (forceDisable: boolean) => {
     // else, force delete only when the item user picks has failed
     const forceDelete = isBulkAction
       ? shouldForceDelete
-      : items[0] && items[0] && items[0].stats.state === TRANSFORM_STATE.FAILED;
+      : items[0] && items[0] && items[0].stats?.state === TRANSFORM_STATE.FAILED;
 
     deleteTransforms({
-      transformsInfo: items.map((i) => ({
+      transformsInfo: items.filter(isTransformListRowWithStats).map((i) => ({
         id: i.config.id,
         state: i.stats.state,
       })),
@@ -87,11 +88,15 @@ export const useDeleteAction = (forceDisable: boolean) => {
             canDeleteTransform,
             disabled: isDeleteActionDisabled([item], forceDisable),
             isBulkAction: false,
+            items: [item],
+            forceDisable,
           }}
         />
       ),
       enabled: (item: TransformListRow) =>
-        !isDeleteActionDisabled([item], forceDisable) && canDeleteTransform,
+        isTransformListRowWithStats(item) &&
+        !isDeleteActionDisabled([item], forceDisable) &&
+        canDeleteTransform,
       description: deleteActionNameText,
       icon: 'trash',
       type: 'icon',
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx
index 4fb0f9e655208..05d804cc9147b 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx
@@ -11,16 +11,16 @@ import { __IntlProvider as IntlProvider } from '@kbn/i18n-react';
 
 import { render, waitFor, screen } from '@testing-library/react';
 
-import { TransformListRow } from '../../../../common';
 import { isDiscoverActionDisabled, DiscoverActionName } from './discover_action_name';
 
 import transformListRow from '../../../../common/__mocks__/transform_list_row.json';
+import { TransformListRowWithStats } from '../../../../common/transform_list';
 
 jest.mock('../../../../../shared_imports');
 jest.mock('../../../../app_dependencies');
 
 // @ts-expect-error mock data is too loosely typed
-const item: TransformListRow = transformListRow;
+const item: TransformListRowWithStats = transformListRow;
 
 describe('Transform: Transform List Actions isDiscoverActionDisabled()', () => {
   it('should be disabled when more than one item is passed in', () => {
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx
index c3d359e648c28..b164dbee25fd2 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx
@@ -33,7 +33,7 @@ export const isDiscoverActionDisabled = (
   const item = items[0];
 
   // Disable discover action if it's a batch transform and was never started
-  const stoppedTransform = item.stats.state === TRANSFORM_STATE.STOPPED;
+  const stoppedTransform = item.stats?.state === TRANSFORM_STATE.STOPPED;
   const transformProgress = getTransformProgress(item);
   const isBatchTransform = typeof item.config.sync === 'undefined';
   const transformNeverStarted =
@@ -52,7 +52,7 @@ export const DiscoverActionName: FC<DiscoverActionNameProps> = ({ dataViewExists
   const item = items[0];
 
   // Disable discover action if it's a batch transform and was never started
-  const stoppedTransform = item.stats.state === TRANSFORM_STATE.STOPPED;
+  const stoppedTransform = item.stats?.state === TRANSFORM_STATE.STOPPED;
   const transformProgress = getTransformProgress(item);
   const isBatchTransform = typeof item.config.sync === 'undefined';
   const transformNeverStarted =
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx
index ee883a3e7e77b..c0abb4fa5e51c 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx
@@ -49,7 +49,6 @@ export const ReauthorizeActionName: FC<ReauthorizeActionNameProps> = ({
 }) => {
   const { canStartStopTransform } = useTransformCapabilities();
 
-  // Disable start for batch transforms which have completed.
   const someNeedsReauthorization = items.some(needsReauthorization);
 
   const actionIsDisabled = isReauthorizeActionDisabled(
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx
index 67e618765e42e..3d216af23db8a 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx
@@ -7,6 +7,7 @@
 
 import React, { useMemo, useState } from 'react';
 
+import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import { sortTransformsToReauthorize } from './sort_transforms_to_reauthorize';
 import { needsReauthorization } from '../../../../common/reauthorization_utils';
 import { useReauthorizeTransforms } from '../../../../hooks/use_reauthorize_transform';
@@ -54,6 +55,7 @@ export const useReauthorizeAction = (forceDisable: boolean, transformNodes: numb
       ),
       available: (item: TransformListRow) => needsReauthorization(item),
       enabled: (item: TransformListRow) =>
+        isTransformListRowWithStats(item) &&
         !isReauthorizeActionDisabled([item], canStartStopTransform, transformNodes),
       description: reauthorizeActionNameText,
       icon: 'alert',
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx
index 6d5f56d3e7297..0f19701fb0914 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx
@@ -11,6 +11,9 @@ import { EuiToolTip } from '@elastic/eui';
 
 import { i18n } from '@kbn/i18n';
 
+import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message';
+import { missingTransformStats } from '../../../../common/transform_list';
+import { TransformCapabilities } from '../../../../../../common/types/capabilities';
 import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants';
 import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message';
 
@@ -24,45 +27,74 @@ export const resetActionNameText = i18n.translate(
 );
 
 const transformCanNotBeReseted = (i: TransformListRow) =>
+  i.stats &&
   !([TRANSFORM_STATE.STOPPED, TRANSFORM_STATE.FAILED] as TransformState[]).includes(i.stats.state);
 
 export const isResetActionDisabled = (items: TransformListRow[], forceDisable: boolean) => {
   const disabled = items.some(transformCanNotBeReseted);
-  return forceDisable === true || disabled;
+
+  return forceDisable === true || disabled || missingTransformStats(items);
 };
 
+export const getResetActionDisabledMessage = ({
+  items,
+  canResetTransform,
+  forceDisable,
+}: {
+  items: TransformListRow[];
+  canResetTransform: TransformCapabilities['canResetTransform'];
+  forceDisable: boolean;
+}) => {
+  const isBulkAction = items.length > 1;
+
+  if (missingTransformStats(items)) {
+    return createNoStatsTooltipMessage({
+      actionName: resetActionNameText,
+      count: items.length,
+    });
+  }
+
+  if (!canResetTransform) {
+    return createCapabilityFailureMessage('canResetTransform');
+  }
+
+  if (isResetActionDisabled(items, forceDisable)) {
+    const bulkResetButtonDisabledText = i18n.translate(
+      'xpack.transform.transformList.resetBulkActionDisabledToolTipContent',
+      {
+        defaultMessage: 'One or more selected transforms must be stopped to be reset.',
+      }
+    );
+    const resetButtonDisabledText = i18n.translate(
+      'xpack.transform.transformList.resetActionDisabledToolTipContent',
+      {
+        defaultMessage: 'Stop the transform in order to reset it.',
+      }
+    );
+
+    return isBulkAction ? bulkResetButtonDisabledText : resetButtonDisabledText;
+  }
+};
 export interface ResetActionNameProps {
+  items: TransformListRow[];
+
   canResetTransform: boolean;
   disabled: boolean;
   isBulkAction: boolean;
 }
 
 export const ResetActionName: FC<ResetActionNameProps> = ({
+  items,
   canResetTransform,
   disabled,
-  isBulkAction,
 }) => {
-  const bulkResetButtonDisabledText = i18n.translate(
-    'xpack.transform.transformList.resetBulkActionDisabledToolTipContent',
-    {
-      defaultMessage: 'One or more selected transforms must be stopped to be reset.',
-    }
-  );
-  const resetButtonDisabledText = i18n.translate(
-    'xpack.transform.transformList.resetActionDisabledToolTipContent',
-    {
-      defaultMessage: 'Stop the transform in order to reset it.',
-    }
-  );
-
-  if (disabled || !canResetTransform) {
-    let content;
-    if (disabled) {
-      content = isBulkAction ? bulkResetButtonDisabledText : resetButtonDisabledText;
-    } else {
-      content = createCapabilityFailureMessage('canResetTransform');
-    }
+  const content = getResetActionDisabledMessage({
+    items,
+    canResetTransform,
+    forceDisable: disabled,
+  });
 
+  if (content) {
     return (
       <EuiToolTip position="top" content={content}>
         <>{resetActionNameText}</>
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx
index 23a399fdb9086..b00ea69d7e4fd 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx
@@ -7,12 +7,21 @@
 
 import React, { useMemo, useState } from 'react';
 
+import {
+  isTransformListRowWithStats,
+  TransformListRowWithStats,
+} from '../../../../common/transform_list';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 
 import { TransformListAction, TransformListRow } from '../../../../common';
 import { useTransformCapabilities, useResetTransforms } from '../../../../hooks';
 
-import { resetActionNameText, isResetActionDisabled, ResetActionName } from './reset_action_name';
+import {
+  resetActionNameText,
+  isResetActionDisabled,
+  ResetActionName,
+  getResetActionDisabledMessage,
+} from './reset_action_name';
 
 export type ResetAction = ReturnType<typeof useResetAction>;
 export const useResetAction = (forceDisable: boolean) => {
@@ -24,7 +33,7 @@ export const useResetAction = (forceDisable: boolean) => {
   const [items, setItems] = useState<TransformListRow[]>([]);
 
   const shouldForceReset = useMemo(
-    () => items.some((i: TransformListRow) => i.stats.state === TRANSFORM_STATE.FAILED),
+    () => items.some((i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.FAILED),
     [items]
   );
 
@@ -34,10 +43,12 @@ export const useResetAction = (forceDisable: boolean) => {
     setModalVisible(false);
 
     resetTransforms({
-      transformsInfo: items.map((i) => ({
-        id: i.config.id,
-        state: i.stats.state,
-      })),
+      transformsInfo: items
+        .filter<TransformListRowWithStats>(isTransformListRowWithStats)
+        .map((i) => ({
+          id: i.config.id,
+          state: i.stats.state,
+        })),
     });
   };
 
@@ -56,11 +67,13 @@ export const useResetAction = (forceDisable: boolean) => {
             canResetTransform,
             disabled: isResetActionDisabled([item], forceDisable),
             isBulkAction: false,
+            items: [item],
           }}
         />
       ),
       enabled: (item: TransformListRow) =>
-        !isResetActionDisabled([item], forceDisable) && canResetTransform,
+        getResetActionDisabledMessage({ items: [item], canResetTransform, forceDisable }) ===
+        undefined,
       description: resetActionNameText,
       icon: 'refresh',
       type: 'icon',
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx
index 0c3be1cdad70b..5b4ff6908afdf 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx
@@ -11,6 +11,8 @@ import { EuiToolTip } from '@elastic/eui';
 
 import { i18n } from '@kbn/i18n';
 
+import { missingTransformStats } from '../../../../common/transform_list';
+import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message';
 import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message';
 
 import { useTransformCapabilities } from '../../../../hooks';
@@ -35,7 +37,8 @@ export const isScheduleNowActionDisabled = (
     !canScheduleNowTransform ||
     completedBatchTransform ||
     items.length === 0 ||
-    transformNodes === 0
+    transformNodes === 0 ||
+    missingTransformStats(items)
   );
 };
 
@@ -92,6 +95,11 @@ export const ScheduleNowActionName: FC<ScheduleNowActionNameProps> = ({
       content = createCapabilityFailureMessage('canScheduleNowTransform');
     } else if (completedBatchTransform) {
       content = completedBatchTransformMessage;
+    } else if (missingTransformStats(items)) {
+      content = createNoStatsTooltipMessage({
+        actionName: scheduleNowActionNameText,
+        count: items.length,
+      });
     }
   }
 
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx
index a13d3da89f677..e5d330eb9ba28 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx
@@ -7,6 +7,7 @@
 
 import React, { useMemo } from 'react';
 
+import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 
 import { useTransformCapabilities } from '../../../../hooks';
@@ -33,8 +34,10 @@ export const useScheduleNowAction = (forceDisable: boolean, transformNodes: numb
           transformNodes={transformNodes}
         />
       ),
-      available: (item: TransformListRow) => item.stats.state === TRANSFORM_STATE.STARTED,
+      available: (item: TransformListRow) =>
+        isTransformListRowWithStats(item) ? item.stats.state === TRANSFORM_STATE.STARTED : true,
       enabled: (item: TransformListRow) =>
+        isTransformListRowWithStats(item) &&
         !isScheduleNowActionDisabled([item], canScheduleNowTransform, transformNodes),
       description: scheduleNowActionNameText,
       icon: 'play',
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx
index c50c83d25edc5..22ddbf1d12c3d 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx
@@ -9,6 +9,8 @@ import React, { type FC } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiToolTip } from '@elastic/eui';
 
+import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message';
+import { missingTransformStats } from '../../../../common/transform_list';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message';
 
@@ -31,7 +33,7 @@ export const isStartActionDisabled = (
   const completedBatchTransform = items.some((i: TransformListRow) => isCompletedBatchTransform(i));
   // Disable start action if one of the transforms is already started or trying to restart will throw error
   const startedTransform = items.some(
-    (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STARTED
+    (i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.STARTED
   );
 
   return (
@@ -39,7 +41,8 @@ export const isStartActionDisabled = (
     completedBatchTransform ||
     startedTransform ||
     items.length === 0 ||
-    transformNodes === 0
+    transformNodes === 0 ||
+    missingTransformStats(items)
   );
 };
 
@@ -58,9 +61,9 @@ export const StartActionName: FC<StartActionNameProps> = ({
 
   // Disable start for batch transforms which have completed.
   const completedBatchTransform = items.some((i: TransformListRow) => isCompletedBatchTransform(i));
-  // Disable start action if one of the transforms is already started or trying to restart will throw error
+  // Disable if one of the transforms is already started or trying to restart will throw error
   const startedTransform = items.some(
-    (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STARTED
+    (i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.STARTED
   );
 
   let startedTransformMessage;
@@ -107,6 +110,11 @@ export const StartActionName: FC<StartActionNameProps> = ({
       content = completedBatchTransformMessage;
     } else if (startedTransform) {
       content = startedTransformMessage;
+    } else if (missingTransformStats(items)) {
+      content = createNoStatsTooltipMessage({
+        actionName: startActionNameText,
+        count: items.length,
+      });
     }
   }
 
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx
index 168174730b706..3724e53cf69b8 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx
@@ -7,6 +7,7 @@
 
 import React, { useMemo, useState } from 'react';
 
+import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 
 import { TransformListAction, TransformListRow } from '../../../../common';
@@ -46,7 +47,8 @@ export const useStartAction = (forceDisable: boolean, transformNodes: number) =>
           transformNodes={transformNodes}
         />
       ),
-      available: (item: TransformListRow) => item.stats.state === TRANSFORM_STATE.STOPPED,
+      available: (item: TransformListRow) =>
+        isTransformListRowWithStats(item) ? item.stats.state === TRANSFORM_STATE.STOPPED : true,
       enabled: (item: TransformListRow) =>
         !isStartActionDisabled([item], canStartStopTransform, transformNodes),
       description: startActionNameText,
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx
index e5bc1425cdd99..b2a079a425273 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx
@@ -9,6 +9,12 @@ import React, { type FC } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiToolTip } from '@elastic/eui';
 
+import { TransformCapabilities } from '../../../../../../common/types/capabilities';
+import {
+  isTransformListRowWithStats,
+  missingTransformStats,
+} from '../../../../common/transform_list';
+import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message';
 
@@ -22,6 +28,46 @@ export const stopActionNameText = i18n.translate(
   }
 );
 
+export const getStopActionDisabledMessage = ({
+  items,
+  capabilities,
+}: {
+  items: TransformListRow[];
+  capabilities: TransformCapabilities;
+}) => {
+  const isBulkAction = items.length > 1;
+
+  const { canStartStopTransform } = capabilities;
+
+  if (missingTransformStats(items)) {
+    return createNoStatsTooltipMessage({
+      actionName: stopActionNameText,
+      count: items.length,
+    });
+  }
+
+  // Disable stop action if one of the transforms is stopped already
+  const stoppedTransform = items.some(
+    (i: TransformListRow) =>
+      isTransformListRowWithStats(i) && i.stats.state === TRANSFORM_STATE.STOPPED
+  );
+
+  if (!canStartStopTransform) {
+    return createCapabilityFailureMessage('canStartStopTransform');
+  }
+
+  if (stoppedTransform) {
+    return isBulkAction === true
+      ? i18n.translate('xpack.transform.transformList.stoppedTransformBulkToolTip', {
+          defaultMessage: 'One or more transforms are already stopped.',
+        })
+      : i18n.translate('xpack.transform.transformList.stoppedTransformToolTip', {
+          defaultMessage: '{transformId} is already stopped.',
+          values: { transformId: items[0] && items[0].config.id },
+        });
+  }
+};
+
 export const isStopActionDisabled = (
   items: TransformListRow[],
   canStartStopTransform: boolean,
@@ -29,10 +75,15 @@ export const isStopActionDisabled = (
 ) => {
   // Disable stop action if one of the transforms is stopped already
   const stoppedTransform = items.some(
-    (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STOPPED
+    (i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.STOPPED
   );
 
-  return forceDisable === true || !canStartStopTransform || stoppedTransform === true;
+  return (
+    forceDisable === true ||
+    !canStartStopTransform ||
+    stoppedTransform === true ||
+    missingTransformStats(items)
+  );
 };
 
 export interface StopActionNameProps {
@@ -40,42 +91,15 @@ export interface StopActionNameProps {
   forceDisable?: boolean;
 }
 export const StopActionName: FC<StopActionNameProps> = ({ items, forceDisable }) => {
-  const isBulkAction = items.length > 1;
-  const { canStartStopTransform } = useTransformCapabilities();
-
-  // Disable stop action if one of the transforms is stopped already
-  const stoppedTransform = items.some(
-    (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STOPPED
-  );
-
-  let stoppedTransformMessage;
-  if (isBulkAction === true) {
-    stoppedTransformMessage = i18n.translate(
-      'xpack.transform.transformList.stoppedTransformBulkToolTip',
-      {
-        defaultMessage: 'One or more transforms are already stopped.',
-      }
-    );
-  } else {
-    stoppedTransformMessage = i18n.translate(
-      'xpack.transform.transformList.stoppedTransformToolTip',
-      {
-        defaultMessage: '{transformId} is already stopped.',
-        values: { transformId: items[0] && items[0].config.id },
-      }
-    );
-  }
-
-  if (!canStartStopTransform || stoppedTransform) {
+  const capabilities = useTransformCapabilities();
+  // Disable transforms if stats does not exist
+  const stoppedTransformMessage = getStopActionDisabledMessage({
+    items,
+    capabilities,
+  });
+  if (forceDisable || stoppedTransformMessage) {
     return (
-      <EuiToolTip
-        position="top"
-        content={
-          !canStartStopTransform
-            ? createCapabilityFailureMessage('canStartStopTransform')
-            : stoppedTransformMessage
-        }
-      >
+      <EuiToolTip position="top" content={stoppedTransformMessage}>
         <>{stopActionNameText}</>
       </EuiToolTip>
     );
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx
index e410704341177..36ec5139d14ee 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx
@@ -6,6 +6,7 @@
  */
 
 import React, { useCallback, useMemo, useState } from 'react';
+import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import { TRANSFORM_STATE } from '../../../../../../common/constants';
 import { TransformListAction, TransformListRow } from '../../../../common';
 import { useTransformCapabilities, useStopTransforms } from '../../../../hooks';
@@ -30,13 +31,16 @@ export const useStopAction = (forceDisable: boolean) => {
   const stopAndCloseModal = useCallback(
     (transformSelection: TransformListRow[]) => {
       setModalVisible(false);
-      stopTransforms(transformSelection.map((t) => ({ id: t.id, state: t.stats.state })));
+      stopTransforms(
+        transformSelection.map((t) => ({ id: t.id, state: t.stats ? t.stats.state : 'waiting' }))
+      );
     },
     [stopTransforms]
   );
 
   const clickHandler = useCallback(
-    (i: TransformListRow) => stopTransforms([{ id: i.id, state: i.stats.state }]),
+    (t: TransformListRow) =>
+      stopTransforms([{ id: t.id, state: t.stats ? t.stats.state : 'waiting' }]),
     [stopTransforms]
   );
 
@@ -45,8 +49,10 @@ export const useStopAction = (forceDisable: boolean) => {
       name: (item: TransformListRow) => (
         <StopActionName items={[item]} forceDisable={forceDisable} />
       ),
-      available: (item: TransformListRow) => item.stats.state !== TRANSFORM_STATE.STOPPED,
+      available: (item: TransformListRow) =>
+        isTransformListRowWithStats(item) ? item.stats.state !== TRANSFORM_STATE.STOPPED : true,
       enabled: (item: TransformListRow) =>
+        isTransformListRowWithStats(item) &&
         !isStopActionDisabled([item], canStartStopTransform, forceDisable),
       description: stopActionNameText,
       icon: 'stop',
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx
index f8b55dbbeda3c..370d7c2da05d5 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx
@@ -38,7 +38,7 @@ describe('Transform: Transform List <ExpandedRow />', () => {
 
     render(
       <MlSharedContext.Provider value={mlShared}>
-        <ExpandedRow item={item} onAlertEdit={onAlertEdit} />
+        <ExpandedRow item={item} onAlertEdit={onAlertEdit} transformsStatsLoading={false} />
       </MlSharedContext.Provider>
     );
 
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx
index f4766da492f67..c43e3d096f32b 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx
@@ -9,14 +9,23 @@ import React, { FC, useMemo } from 'react';
 import { css } from '@emotion/react';
 import moment from 'moment-timezone';
 
-import { EuiButtonEmpty, EuiTabbedContent } from '@elastic/eui';
+import {
+  EuiButtonEmpty,
+  EuiLoadingSpinner,
+  EuiTabbedContent,
+  EuiFlexGroup,
+  useEuiTheme,
+  EuiCallOut,
+  EuiFlexItem,
+} from '@elastic/eui';
 
-import { Optional } from '@kbn/utility-types';
 import { i18n } from '@kbn/i18n';
 import { formatHumanReadableDateTimeSeconds } from '@kbn/ml-date-utils';
 import { stringHash } from '@kbn/ml-string-hash';
 import { isDefined } from '@kbn/ml-is-defined';
 
+import { FormattedMessage } from '@kbn/i18n-react';
+import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import { useIsServerless } from '../../../../serverless_context';
 import { TransformHealthAlertRule } from '../../../../../../common/types/alerting';
 
@@ -42,39 +51,66 @@ type Item = SectionItem;
 interface Props {
   item: TransformListRow;
   onAlertEdit: (alertRule: TransformHealthAlertRule) => void;
+  transformsStatsLoading: boolean;
 }
 
-type StateValues = Optional<TransformListRow['stats'], 'stats' | 'checkpointing'>;
+const NoStatsFallbackTabContent = ({
+  transformsStatsLoading,
+}: {
+  transformsStatsLoading: boolean;
+}) => {
+  const { euiTheme } = useEuiTheme();
 
-export const ExpandedRow: FC<Props> = ({ item, onAlertEdit }) => {
-  const hideNodeInfo = useIsServerless();
+  const content = transformsStatsLoading ? (
+    <EuiLoadingSpinner />
+  ) : (
+    <EuiFlexItem grow={true}>
+      <EuiCallOut
+        size="s"
+        color="warning"
+        iconType="iInCircle"
+        title={
+          <FormattedMessage
+            id="xpack.transform.transformList.noStatsAvailable"
+            defaultMessage="No stats available for this transform."
+          />
+        }
+      />
+    </EuiFlexItem>
+  );
+  return (
+    <EuiFlexGroup justifyContent="center" alignItems="center" css={{ height: euiTheme.size.xxxxl }}>
+      {content}
+    </EuiFlexGroup>
+  );
+};
 
-  const stateValues: StateValues = { ...item.stats };
-  delete stateValues.stats;
-  delete stateValues.checkpointing;
+export const ExpandedRow: FC<Props> = ({ item, onAlertEdit, transformsStatsLoading }) => {
+  const hideNodeInfo = useIsServerless();
 
   const stateItems: Item[] = [];
-  stateItems.push(
-    {
-      title: 'ID',
-      description: item.id,
-    },
-    {
+  stateItems.push({
+    title: 'ID',
+    description: item.id,
+  });
+  if (isTransformListRowWithStats(item)) {
+    stateItems.push({
       title: 'state',
       description: item.stats.state,
-    }
-  );
-  if (!hideNodeInfo && item.stats.node !== undefined) {
-    stateItems.push({
-      title: 'node.name',
-      description: item.stats.node.name,
-    });
-  }
-  if (item.stats.health !== undefined) {
-    stateItems.push({
-      title: 'health',
-      description: <TransformHealthColoredDot healthStatus={item.stats.health.status} />,
     });
+
+    if (!hideNodeInfo && item.stats.node !== undefined) {
+      stateItems.push({
+        title: 'node.name',
+        description: item.stats.node.name,
+      });
+    }
+    if (item.stats.health !== undefined) {
+      stateItems.push({
+        title: 'health',
+        description: <TransformHealthColoredDot healthStatus={item.stats.health.status} />,
+      });
+    }
   }
 
   const state: SectionConfig = {
@@ -137,69 +173,71 @@ export const ExpandedRow: FC<Props> = ({ item, onAlertEdit }) => {
   };
 
   const checkpointingItems: Item[] = [];
-  if (item.stats.checkpointing.changes_last_detected_at !== undefined) {
-    checkpointingItems.push({
-      title: 'changes_last_detected_at',
-      description: formatHumanReadableDateTimeSeconds(
-        item.stats.checkpointing.changes_last_detected_at
-      ),
-    });
-  }
-
-  if (item.stats.checkpointing.last !== undefined) {
-    checkpointingItems.push({
-      title: 'last.checkpoint',
-      description: item.stats.checkpointing.last.checkpoint,
-    });
-    if (item.stats.checkpointing.last.timestamp_millis !== undefined) {
+  if (isTransformListRowWithStats(item)) {
+    if (item.stats.checkpointing.changes_last_detected_at !== undefined) {
       checkpointingItems.push({
-        title: 'last.timestamp',
+        title: 'changes_last_detected_at',
         description: formatHumanReadableDateTimeSeconds(
-          item.stats.checkpointing.last.timestamp_millis
+          item.stats.checkpointing.changes_last_detected_at
         ),
       });
+    }
+
+    if (item.stats.checkpointing.last !== undefined) {
       checkpointingItems.push({
-        title: 'last.timestamp_millis',
-        description: item.stats.checkpointing.last.timestamp_millis,
+        title: 'last.checkpoint',
+        description: item.stats.checkpointing.last.checkpoint,
       });
+      if (item.stats.checkpointing.last.timestamp_millis !== undefined) {
+        checkpointingItems.push({
+          title: 'last.timestamp',
+          description: formatHumanReadableDateTimeSeconds(
+            item.stats.checkpointing.last.timestamp_millis
+          ),
+        });
+        checkpointingItems.push({
+          title: 'last.timestamp_millis',
+          description: item.stats.checkpointing.last.timestamp_millis,
+        });
+      }
     }
-  }
-
-  if (item.stats.checkpointing.last_search_time !== undefined) {
-    checkpointingItems.push({
-      title: 'last_search_time',
-      description: formatHumanReadableDateTimeSeconds(item.stats.checkpointing.last_search_time),
-    });
-  }
 
-  if (item.stats.checkpointing.next !== undefined) {
-    checkpointingItems.push({
-      title: 'next.checkpoint',
-      description: item.stats.checkpointing.next.checkpoint,
-    });
-    if (item.stats.checkpointing.next.checkpoint_progress !== undefined) {
+    if (item.stats.checkpointing.last_search_time !== undefined) {
       checkpointingItems.push({
-        title: 'next.checkpoint_progress.total_docs',
-        description: item.stats.checkpointing.next.checkpoint_progress.total_docs,
+        title: 'last_search_time',
+        description: formatHumanReadableDateTimeSeconds(item.stats.checkpointing.last_search_time),
       });
+    }
+
+    if (item.stats.checkpointing.next !== undefined) {
       checkpointingItems.push({
-        title: 'next.checkpoint_progress.docs_remaining',
-        description: item.stats.checkpointing.next.checkpoint_progress.docs_remaining,
+        title: 'next.checkpoint',
+        description: item.stats.checkpointing.next.checkpoint,
       });
+      if (item.stats.checkpointing.next.checkpoint_progress !== undefined) {
+        checkpointingItems.push({
+          title: 'next.checkpoint_progress.total_docs',
+          description: item.stats.checkpointing.next.checkpoint_progress.total_docs,
+        });
+        checkpointingItems.push({
+          title: 'next.checkpoint_progress.docs_remaining',
+          description: item.stats.checkpointing.next.checkpoint_progress.docs_remaining,
+        });
+        checkpointingItems.push({
+          title: 'next.checkpoint_progress.percent_complete',
+          description: item.stats.checkpointing.next.checkpoint_progress.percent_complete,
+        });
+      }
+    }
+
+    if (item.stats.checkpointing.operations_behind !== undefined) {
       checkpointingItems.push({
-        title: 'next.checkpoint_progress.percent_complete',
-        description: item.stats.checkpointing.next.checkpoint_progress.percent_complete,
+        title: 'operations_behind',
+        description: item.stats.checkpointing.operations_behind,
       });
     }
   }
 
-  if (item.stats.checkpointing.operations_behind !== undefined) {
-    checkpointingItems.push({
-      title: 'operations_behind',
-      description: item.stats.checkpointing.operations_behind,
-    });
-  }
-
   const alertRuleItems: Item[] | undefined = item.alerting_rules?.map((rule) => {
     return {
       title: (
@@ -236,9 +274,11 @@ export const ExpandedRow: FC<Props> = ({ item, onAlertEdit }) => {
 
   const stats: SectionConfig = {
     title: 'Stats',
-    items: Object.entries(item.stats.stats).map((s) => {
-      return { title: s[0].toString(), description: getItemDescription(s[1]) };
-    }),
+    items: item.stats
+      ? Object.entries(item.stats.stats).map((s) => {
+          return { title: s[0].toString(), description: getItemDescription(s[1]) };
+        })
+      : [],
     position: 'left',
   };
 
@@ -275,8 +315,10 @@ export const ExpandedRow: FC<Props> = ({ item, onAlertEdit }) => {
           defaultMessage: 'Stats',
         }
       ),
-      content: (
+      content: item.stats ? (
         <ExpandedRowDetailsPane sections={[stats]} dataTestSubj={'transformStatsTabContent'} />
+      ) : (
+        <NoStatsFallbackTabContent transformsStatsLoading={transformsStatsLoading} />
       ),
     },
     {
@@ -285,7 +327,7 @@ export const ExpandedRow: FC<Props> = ({ item, onAlertEdit }) => {
       name: 'JSON',
       content: <ExpandedRowJsonPane json={item.config} />,
     },
-    ...(item.stats.health
+    ...(item.stats?.health
       ? [
           {
             id: `transform-health-tab-${tabId}`,
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx
index 0e8ab94dc2086..3a088d5a9b273 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx
@@ -12,14 +12,15 @@ import { formatDate, EuiPanel, EuiSpacer, EuiInMemoryTable } from '@elastic/eui'
 import { i18n } from '@kbn/i18n';
 
 import { TIME_FORMAT } from '../../../../../../common/constants';
-import type { TransformHealthIssue } from '../../../../../../common/types/transform_stats';
-
-import { TransformListRow } from '../../../../common';
+import type {
+  TransformHealthIssue,
+  TransformStats,
+} from '../../../../../../common/types/transform_stats';
 
 import { TransformHealthColoredDot } from './transform_health_colored_dot';
 
 interface ExpandedRowHealthPaneProps {
-  health: TransformListRow['stats']['health'];
+  health: TransformStats['health'];
 }
 
 export const ExpandedRowHealthPane: FC<ExpandedRowHealthPaneProps> = ({ health }) => {
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx
index fc4bf0e24d14d..e3f98beea9e7e 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx
@@ -39,6 +39,7 @@ describe('Transform: Transform List <TransformList />', () => {
             transformNodes={1}
             transforms={[]}
             transformsLoading={false}
+            transformsStatsLoading={false}
           />
         </QueryClientProvider>
       </IntlProvider>
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx
index 2e0106afc0e92..a55253a08b5eb 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx
@@ -70,12 +70,19 @@ type ItemIdToExpandedRowMap = Record<string, JSX.Element>;
 function getItemIdToExpandedRowMap(
   itemIds: TransformId[],
   transforms: TransformListRow[],
-  onAlertEdit: (alertRule: TransformHealthAlertRule) => void
+  onAlertEdit: (alertRule: TransformHealthAlertRule) => void,
+  transformsStatsLoading: boolean
 ): ItemIdToExpandedRowMap {
   return itemIds.reduce((m: ItemIdToExpandedRowMap, transformId: TransformId) => {
     const item = transforms.find((transform) => transform.config.id === transformId);
     if (item !== undefined) {
-      m[transformId] = <ExpandedRow item={item} onAlertEdit={onAlertEdit} />;
+      m[transformId] = (
+        <ExpandedRow
+          item={item}
+          onAlertEdit={onAlertEdit}
+          transformsStatsLoading={transformsStatsLoading}
+        />
+      );
     }
     return m;
   }, {} as ItemIdToExpandedRowMap);
@@ -87,6 +94,7 @@ interface TransformListProps {
   transformNodes: number;
   transforms: TransformListRow[];
   transformsLoading: boolean;
+  transformsStatsLoading: boolean;
 }
 
 export const TransformList: FC<TransformListProps> = ({
@@ -95,6 +103,7 @@ export const TransformList: FC<TransformListProps> = ({
   transformNodes,
   transforms,
   transformsLoading,
+  transformsStatsLoading,
 }) => {
   const refreshTransformList = useRefreshTransformList();
   const { setEditAlertRule } = useAlertRuleFlyout();
@@ -126,7 +135,8 @@ export const TransformList: FC<TransformListProps> = ({
     expandedRowItemIds,
     setExpandedRowItemIds,
     transformNodes,
-    transformSelection
+    transformSelection,
+    transformsStatsLoading
   );
 
   const searchError = query?.error ? query?.error.message : undefined;
@@ -166,7 +176,8 @@ export const TransformList: FC<TransformListProps> = ({
   const itemIdToExpandedRowMap = getItemIdToExpandedRowMap(
     expandedRowItemIds,
     transforms,
-    setEditAlertRule
+    setEditAlertRule,
+    transformsStatsLoading
   );
 
   const bulkActionMenuItems = [
@@ -235,6 +246,7 @@ export const TransformList: FC<TransformListProps> = ({
           canResetTransform={capabilities.canResetTransform}
           disabled={isResetActionDisabled(transformSelection, false)}
           isBulkAction={true}
+          items={transformSelection}
         />
       </EuiButtonEmpty>
     </div>,
@@ -247,6 +259,8 @@ export const TransformList: FC<TransformListProps> = ({
           canDeleteTransform={capabilities.canDeleteTransform}
           disabled={isDeleteActionDisabled(transformSelection, false)}
           isBulkAction={true}
+          items={transformSelection}
+          forceDisable={false}
         />
       </EuiButtonEmpty>
     </div>,
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx
index 5a2df943e9a54..637f32558cb41 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx
@@ -106,9 +106,12 @@ export const filterTransforms = (transforms: TransformListRow[], clauses: Clause
       // filter other clauses, i.e. the mode and status filters
       if (c.type !== 'is' && Array.isArray(c.value)) {
         // the status value is an array of string(s) e.g. ['failed', 'stopped']
-        ts = transforms.filter((transform) => (c.value as Value[]).includes(transform.stats.state));
+        ts = transforms.filter(
+          (transform) => transform.stats && (c.value as Value[]).includes(transform.stats.state)
+        );
       } else {
         ts = transforms.filter((transform) => {
+          if (!transform.stats) return false;
           if (c.type === 'field' && c.field === 'health') {
             return transform.stats.health?.status === c.value;
           }
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx
index 79f6321c2419f..ea85f705e46ef 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx
@@ -94,10 +94,12 @@ function createTranformStats(
       transformStats.batch.value++;
     }
 
-    if (transform.stats.state === TRANSFORM_STATE.FAILED) {
-      failedTransforms++;
-    } else if (transform.stats.state === TRANSFORM_STATE.STARTED) {
-      startedTransforms++;
+    if (transform.stats) {
+      if (transform.stats.state === TRANSFORM_STATE.FAILED) {
+        failedTransforms++;
+      } else if (transform.stats.state === TRANSFORM_STATE.STARTED) {
+        startedTransforms++;
+      }
     }
   });
 
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx
index 87f550e433bc2..e49df35b42a55 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx
@@ -20,7 +20,7 @@ describe('Transform: Job List Columns', () => {
     const wrapper: FC = ({ children }) => (
       <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
     );
-    const { result, waitForNextUpdate } = renderHook(() => useColumns([], () => {}, 1, []), {
+    const { result, waitForNextUpdate } = renderHook(() => useColumns([], () => {}, 1, [], false), {
       wrapper,
     });
 
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx
index 2ae8edf30a1de..fb578ec06aa3c 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx
@@ -22,6 +22,7 @@ import {
   EuiToolTip,
   RIGHT_ALIGNMENT,
   EuiIcon,
+  EuiLoadingSpinner,
 } from '@elastic/eui';
 
 import { useTransformCapabilities } from '../../../../hooks';
@@ -46,12 +47,20 @@ const TRANSFORM_INSUFFICIENT_PERMISSIONS_MSG = i18n.translate(
     defaultMessage: 'This transform was created with insufficient permissions.',
   }
 );
+
+const StatsUnknown = () => (
+  <EuiText textAlign="center" color="subdued" size="s">
+    <FormattedMessage id="xpack.transform.transformList.statsUnknown" defaultMessage="Unknown" />
+  </EuiText>
+);
 export const useColumns = (
   expandedRowItemIds: TransformId[],
   setExpandedRowItemIds: React.Dispatch<React.SetStateAction<TransformId[]>>,
   transformNodes: number,
-  transformSelection: TransformListRow[]
+  transformSelection: TransformListRow[],
+  transformsStatsLoading: boolean
 ) => {
+  const NoStatsFallbackComponent = transformsStatsLoading ? EuiLoadingSpinner : StatsUnknown;
   const { canStartStopTransform } = useTransformCapabilities();
 
   const { actions, modals } = useActions({
@@ -239,10 +248,14 @@ export const useColumns = (
     {
       name: i18n.translate('xpack.transform.status', { defaultMessage: 'Status' }),
       'data-test-subj': 'transformListColumnStatus',
-      sortable: (item: TransformListRow) => item.stats.state,
+      sortable: (item: TransformListRow) => item.stats?.state,
       truncateText: true,
       render(item: TransformListRow) {
-        return <TransformTaskStateBadge state={item.stats.state} reason={item.stats.reason} />;
+        return item.stats ? (
+          <TransformTaskStateBadge state={item.stats.state} reason={item.stats.reason} />
+        ) : (
+          <NoStatsFallbackComponent />
+        );
       },
       width: '100px',
     },
@@ -271,6 +284,7 @@ export const useColumns = (
         if (progress === undefined && isBatchTransform === true) {
           return null;
         }
+        if (!item.stats) return <NoStatsFallbackComponent />;
 
         return (
           <EuiFlexGroup alignItems="center" gutterSize="xs">
@@ -292,7 +306,7 @@ export const useColumns = (
                 </EuiFlexItem>
               </>
             )}
-            {!isBatchTransform && (
+            {!isBatchTransform && item.stats && (
               <>
                 <EuiFlexItem style={{ width: '40px' }} grow={false}>
                   {/* If not stopped, failed or waiting show the animated progress bar */}
@@ -321,10 +335,14 @@ export const useColumns = (
     {
       name: i18n.translate('xpack.transform.health', { defaultMessage: 'Health' }),
       'data-test-subj': 'transformListColumnHealth',
-      sortable: (item: TransformListRow) => item.stats.health.status,
+      sortable: (item: TransformListRow) => item.stats?.health.status,
       truncateText: true,
       render(item: TransformListRow) {
-        return <TransformHealthColoredDot healthStatus={item.stats.health.status} />;
+        return item.stats ? (
+          <TransformHealthColoredDot healthStatus={item.stats.health.status} />
+        ) : (
+          <NoStatsFallbackComponent />
+        );
       },
       width: '100px',
     },
diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx
index cce3128b3f3d0..858954976cf82 100644
--- a/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx
@@ -21,6 +21,9 @@ import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { IHttpFetchError } from '@kbn/core-http-browser';
 
+import type { TransformListRow } from '../../common';
+import { isTransformStats } from '../../../../common/types/transform_stats';
+import { useGetTransformsStats } from '../../hooks/use_get_transform_stats';
 import { useIsServerless } from '../../serverless_context';
 import { needsReauthorization } from '../../common/reauthorization_utils';
 import { TRANSFORM_STATE } from '../../../../common/constants';
@@ -85,13 +88,37 @@ export const TransformManagement: FC = () => {
 
   const {
     isInitialLoading: transformsInitialLoading,
-    isLoading: transformsLoading,
+    isLoading: transformsWithoutStatsLoading,
     error: transformsErrorMessage,
-    data: { transforms, transformIdsWithoutConfig },
+    data: { transforms: transformsWithoutStats, transformIdsWithoutConfig },
   } = useGetTransforms({
     enabled: !transformNodesInitialLoading && (transformNodes > 0 || hideNodeInfo),
   });
 
+  const {
+    isLoading: transformsStatsLoading,
+    error: transformsStatsErrorMessage,
+    data: transformsStats,
+  } = useGetTransformsStats({
+    enabled: !transformNodesInitialLoading && (transformNodes > 0 || hideNodeInfo),
+  });
+
+  const transforms: TransformListRow[] = useMemo(() => {
+    if (!transformsStats) return transformsWithoutStats;
+
+    return transformsWithoutStats.map((t) => {
+      const stats = transformsStats.transforms.find((d) => t.config.id === d.id);
+
+      // A newly created transform might not have corresponding stats yet.
+      // If that's the case we just skip the transform and don't add it to the transform list yet.
+      if (!isTransformStats(stats)) {
+        return t;
+      }
+
+      return { ...t, stats };
+    });
+  }, [transformsStats, transformsWithoutStats]);
+
   const isInitialLoading = transformNodesInitialLoading || transformsInitialLoading;
 
   const { canStartStopTransform } = useTransformCapabilities();
@@ -219,6 +246,17 @@ export const TransformManagement: FC = () => {
                 errorMessage={transformsErrorMessage}
               />
             )}
+            {transformsStatsErrorMessage !== null ? (
+              <ErrorMessageCallout
+                text={
+                  <FormattedMessage
+                    id="xpack.transform.list.transformStatsErrorPromptTitle"
+                    defaultMessage="An error occurred getting the transform stats."
+                  />
+                }
+                errorMessage={transformsStatsErrorMessage}
+              />
+            ) : null}
             <EuiSpacer size="s" />
 
             <TransformStatsBar transformNodes={transformNodes} transformsList={transforms} />
@@ -272,11 +310,12 @@ export const TransformManagement: FC = () => {
               ) : null}
               {(transformNodes > 0 || transforms.length > 0) && (
                 <TransformList
-                  isLoading={transformsLoading}
+                  isLoading={transformsWithoutStatsLoading}
                   onCreateTransform={onOpenModal}
                   transformNodes={transformNodes}
                   transforms={transforms}
-                  transformsLoading={transformsLoading}
+                  transformsLoading={transformsWithoutStatsLoading}
+                  transformsStatsLoading={transformsStatsLoading}
                 />
               )}
               <TransformAlertFlyoutWrapper />
diff --git a/x-pack/plugins/transform/tsconfig.json b/x-pack/plugins/transform/tsconfig.json
index 52e0747cc1fb9..7b41f101c15c1 100644
--- a/x-pack/plugins/transform/tsconfig.json
+++ b/x-pack/plugins/transform/tsconfig.json
@@ -40,7 +40,6 @@
     "@kbn/std",
     "@kbn/es-query",
     "@kbn/ml-agg-utils",
-    "@kbn/utility-types",
     "@kbn/ml-string-hash",
     "@kbn/ui-theme",
     "@kbn/field-types",

From 13e5c076d53d723c423645afeeefa6eaded9cf02 Mon Sep 17 00:00:00 2001
From: Dario Gieselaar <dario.gieselaar@elastic.co>
Date: Mon, 25 Sep 2023 17:39:34 +0200
Subject: [PATCH 09/27] [Observability AI Assistant] ES|QL query generation
 (#166041)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
---
 .../common/types.ts                           |  28 +-
 .../public/components/chat/chat_body.tsx      |   4 +
 .../public/components/chat/chat_item.tsx      |   8 +-
 ...chat_item_content_inline_prompt_editor.tsx |   5 +-
 .../components/chat/chat_timeline.stories.tsx |   1 +
 .../public/components/chat/chat_timeline.tsx  |   4 +
 .../components/chat/function_list_popover.tsx |  20 +-
 .../public/components/chat/types.ts           |  23 +
 .../public/components/insight/insight.tsx     |   8 +-
 .../insight/insight_base.stories.tsx          |   1 +
 .../message_panel/esql_code_block.stories.tsx |  34 ++
 .../message_panel/esql_code_block.tsx         |  77 +++
 .../message_panel/message_panel.stories.tsx   |  26 +-
 .../components/message_panel/message_text.tsx |  47 +-
 .../public/functions/esql.ts                  | 536 ++++++++++++++++++
 .../public/functions/index.ts                 |  19 +-
 .../public/functions/recall.ts                |  44 +-
 .../public/hooks/use_timeline.test.ts         |   1 +
 .../public/hooks/use_timeline.ts              |  78 ++-
 .../public/service/create_chat_service.ts     |   9 +-
 .../service/get_assistant_setup_message.ts    |   6 +-
 .../public/types.ts                           |  12 +-
 .../server/plugin.ts                          |   2 +-
 .../server/routes/chat/route.ts               |  56 +-
 .../server/routes/functions/route.ts          |  25 +-
 .../server/service/client/index.ts            |  50 +-
 .../server/service/index.ts                   |  37 +-
 .../server/service/kb_service/index.ts        |  60 +-
 .../server/service/kb_service/kb_docs/lens.ts |  10 +-
 .../server/service/util/get_category_query.ts |  38 ++
 30 files changed, 1137 insertions(+), 132 deletions(-)
 create mode 100644 x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts
 create mode 100644 x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx
 create mode 100644 x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx
 create mode 100644 x-pack/plugins/observability_ai_assistant/public/functions/esql.ts
 create mode 100644 x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts

diff --git a/x-pack/plugins/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_ai_assistant/common/types.ts
index b14137dbaebf5..690b931271bb6 100644
--- a/x-pack/plugins/observability_ai_assistant/common/types.ts
+++ b/x-pack/plugins/observability_ai_assistant/common/types.ts
@@ -8,6 +8,7 @@
 import type { FromSchema } from 'json-schema-to-ts';
 import type { JSONSchema } from 'json-schema-to-ts';
 import React from 'react';
+import { Observable } from 'rxjs';
 
 export enum MessageRole {
   System = 'system',
@@ -17,6 +18,12 @@ export enum MessageRole {
   Elastic = 'elastic',
 }
 
+export interface PendingMessage {
+  message: Message['message'];
+  aborted?: boolean;
+  error?: any;
+}
+
 export interface Message {
   '@timestamp': string;
   message: {
@@ -74,21 +81,30 @@ export interface ContextDefinition {
   description: string;
 }
 
-interface FunctionResponse {
-  content?: any;
-  data?: any;
+type FunctionResponse =
+  | {
+      content?: any;
+      data?: any;
+    }
+  | Observable<PendingMessage>;
+
+export enum FunctionVisibility {
+  System = 'system',
+  User = 'user',
+  All = 'all',
 }
 
 interface FunctionOptions<TParameters extends CompatibleJSONSchema = CompatibleJSONSchema> {
   name: string;
   description: string;
-  descriptionForUser: string;
+  visibility?: FunctionVisibility;
+  descriptionForUser?: string;
   parameters: TParameters;
   contexts: string[];
 }
 
 type RespondFunction<TArguments, TResponse extends FunctionResponse> = (
-  options: { arguments: TArguments; messages: Message[] },
+  options: { arguments: TArguments; messages: Message[]; connectorId: string },
   signal: AbortSignal
 ) => Promise<TResponse>;
 
@@ -100,7 +116,7 @@ type RenderFunction<TArguments, TResponse extends FunctionResponse> = (options:
 export interface FunctionDefinition {
   options: FunctionOptions;
   respond: (
-    options: { arguments: any; messages: Message[] },
+    options: { arguments: any; messages: Message[]; connectorId: string },
     signal: AbortSignal
   ) => Promise<FunctionResponse>;
   render?: RenderFunction<any, any>;
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx
index 8a6227829c855..f49ebe8c62b92 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx
@@ -189,6 +189,10 @@ export function ChatBody({
                 onFeedback={timeline.onFeedback}
                 onRegenerate={timeline.onRegenerate}
                 onStopGenerating={timeline.onStopGenerating}
+                onActionClick={(payload) => {
+                  setStickToBottom(true);
+                  return timeline.onActionClick(payload);
+                }}
               />
             </EuiPanel>
           </div>
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx
index b3fa93d1ec522..7ec1084a26b22 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx
@@ -24,12 +24,14 @@ import { getRoleTranslation } from '../../utils/get_role_translation';
 import type { Feedback } from '../feedback_buttons';
 import { Message } from '../../../common';
 import { FailedToLoadResponse } from '../message_panel/failed_to_load_response';
+import { ChatActionClickHandler } from './types';
 
 export interface ChatItemProps extends ChatTimelineItem {
   onEditSubmit: (message: Message) => Promise<void>;
   onFeedbackClick: (feedback: Feedback) => void;
   onRegenerateClick: () => void;
   onStopGeneratingClick: () => void;
+  onActionClick: ChatActionClickHandler;
 }
 
 const normalMessageClassName = css`
@@ -76,6 +78,7 @@ export function ChatItem({
   onFeedbackClick,
   onRegenerateClick,
   onStopGeneratingClick,
+  onActionClick,
 }: ChatItemProps) {
   const accordionId = useGeneratedHtmlId({ prefix: 'chat' });
 
@@ -128,6 +131,7 @@ export function ChatItem({
         functionCall={functionCall}
         loading={loading}
         onSubmit={handleInlineEditSubmit}
+        onActionClick={onActionClick}
       />
     ) : null;
 
@@ -147,9 +151,7 @@ export function ChatItem({
 
   return (
     <EuiComment
-      timelineAvatar={
-        <ChatItemAvatar loading={loading && !content} currentUser={currentUser} role={role} />
-      }
+      timelineAvatar={<ChatItemAvatar loading={loading} currentUser={currentUser} role={role} />}
       username={getRoleTranslation(role)}
       event={title}
       actions={
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx
index d017d7d65fc90..df57f069d91d1 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx
@@ -9,6 +9,7 @@ import React from 'react';
 import { MessageText } from '../message_panel/message_text';
 import { ChatPromptEditor } from './chat_prompt_editor';
 import { MessageRole, type Message } from '../../../common';
+import { ChatActionClickHandler } from './types';
 
 interface Props {
   content: string | undefined;
@@ -22,6 +23,7 @@ interface Props {
   loading: boolean;
   editing: boolean;
   onSubmit: (message: Message) => Promise<void>;
+  onActionClick: ChatActionClickHandler;
 }
 export function ChatItemContentInlinePromptEditor({
   content,
@@ -29,9 +31,10 @@ export function ChatItemContentInlinePromptEditor({
   editing,
   loading,
   onSubmit,
+  onActionClick,
 }: Props) {
   return !editing ? (
-    <MessageText content={content || ''} loading={loading} />
+    <MessageText content={content || ''} loading={loading} onActionClick={onActionClick} />
   ) : (
     <ChatPromptEditor
       disabled={false}
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.stories.tsx
index c72fc1824e87e..68c9c567a4a9d 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.stories.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.stories.tsx
@@ -132,6 +132,7 @@ const defaultProps: ComponentProps<typeof Component> = {
   onFeedback: () => {},
   onRegenerate: () => {},
   onStopGenerating: () => {},
+  onActionClick: async () => {},
 };
 
 export const ChatTimeline = Template.bind({});
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx
index 8e50f11842e5d..a50a9984cf40e 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx
@@ -15,6 +15,7 @@ import { ChatWelcomePanel } from './chat_welcome_panel';
 import type { Feedback } from '../feedback_buttons';
 import type { Message } from '../../../common';
 import { UseKnowledgeBaseResult } from '../../hooks/use_knowledge_base';
+import { ChatActionClickHandler } from './types';
 
 export interface ChatTimelineItem
   extends Pick<Message['message'], 'role' | 'content' | 'function_call'> {
@@ -43,6 +44,7 @@ export interface ChatTimelineProps {
   onFeedback: (item: ChatTimelineItem, feedback: Feedback) => void;
   onRegenerate: (item: ChatTimelineItem) => void;
   onStopGenerating: () => void;
+  onActionClick: ChatActionClickHandler;
 }
 
 export function ChatTimeline({
@@ -52,6 +54,7 @@ export function ChatTimeline({
   onFeedback,
   onRegenerate,
   onStopGenerating,
+  onActionClick,
 }: ChatTimelineProps) {
   const filteredItems = items.filter((item) => !item.display.hide);
 
@@ -77,6 +80,7 @@ export function ChatTimeline({
               return onEdit(item, message);
             }}
             onStopGeneratingClick={onStopGenerating}
+            onActionClick={onActionClick}
           />
         ))
       )}
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx
index 72dac17c71c2a..a9cf8146e020d 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx
@@ -17,7 +17,7 @@ import {
 } from '@elastic/eui';
 import type { EuiSelectableOptionCheckedType } from '@elastic/eui/src/components/selectable/selectable_option';
 import { i18n } from '@kbn/i18n';
-import type { FunctionDefinition } from '../../../common/types';
+import { type FunctionDefinition, FunctionVisibility } from '../../../common/types';
 import { useObservabilityAIAssistantChatService } from '../../hooks/use_observability_ai_assistant_chat_service';
 
 interface FunctionListOption {
@@ -175,12 +175,14 @@ function mapFunctions({
   functions: FunctionDefinition[];
   selectedFunctionName: string | undefined;
 }) {
-  return functions.map((func) => ({
-    label: func.options.name,
-    searchableLabel: func.options.descriptionForUser,
-    checked:
-      func.options.name === selectedFunctionName
-        ? ('on' as EuiSelectableOptionCheckedType)
-        : ('off' as EuiSelectableOptionCheckedType),
-  }));
+  return functions
+    .filter((func) => func.options.visibility !== FunctionVisibility.System)
+    .map((func) => ({
+      label: func.options.name,
+      searchableLabel: func.options.descriptionForUser || func.options.description,
+      checked:
+        func.options.name === selectedFunctionName
+          ? ('on' as EuiSelectableOptionCheckedType)
+          : ('off' as EuiSelectableOptionCheckedType),
+    }));
 }
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts b/x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts
new file mode 100644
index 0000000000000..4edd3d7dcdda0
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+type ChatActionClickPayloadBase<TType extends ChatActionClickType, TExtraProps extends {}> = {
+  type: TType;
+} & TExtraProps;
+
+type ChatActionClickPayloadExecuteEsql = ChatActionClickPayloadBase<
+  ChatActionClickType.executeEsqlQuery,
+  { query: string }
+>;
+
+type ChatActionClickPayload = ChatActionClickPayloadExecuteEsql;
+
+export enum ChatActionClickType {
+  executeEsqlQuery = 'executeEsqlQuery',
+}
+
+export type ChatActionClickHandler = (payload: ChatActionClickPayload) => Promise<unknown>;
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx
index 4da53c5d6ee5a..2b56523d1e879 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx
@@ -110,7 +110,13 @@ function ChatContent({
   return (
     <>
       <MessagePanel
-        body={<MessageText content={lastMessage?.message.content ?? ''} loading={loading} />}
+        body={
+          <MessageText
+            content={lastMessage?.message.content ?? ''}
+            loading={loading}
+            onActionClick={async () => {}}
+          />
+        }
         error={pendingMessage?.error}
         controls={
           loading ? (
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx
index 7a65f8e62756e..bd894159eb288 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx
@@ -64,6 +64,7 @@ Morbi dapibus sapien lacus, vitae suscipit ex egestas pharetra. In velit eros, f
 
 Morbi non faucibus massa. Aliquam sed augue in eros ornare luctus sit amet cursus dolor. Pellentesque pellentesque lorem eu odio auctor convallis. Sed sodales felis at velit tempus tincidunt. Nulla sed ante cursus nibh mollis blandit. In mattis imperdiet tellus. Vestibulum nisl turpis, efficitur quis sollicitudin id, mollis in arcu. Vestibulum pulvinar tincidunt magna, vitae facilisis massa congue quis. Cras commodo efficitur tellus, et commodo risus rutrum at.`}
           loading={false}
+          onActionClick={async () => {}}
         />
       }
       controls={
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx
new file mode 100644
index 0000000000000..02c7454ddaab3
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { ComponentMeta, ComponentStoryObj } from '@storybook/react';
+import { ComponentProps } from 'react';
+import { EuiPanel } from '@elastic/eui';
+import { EsqlCodeBlock as Component } from './esql_code_block';
+
+const meta: ComponentMeta<typeof Component> = {
+  component: Component,
+  title: 'app/Molecules/ES|QL Code Block',
+};
+
+export default meta;
+
+const render = (props: ComponentProps<typeof Component>) => {
+  return (
+    <EuiPanel hasBorder hasShadow={false}>
+      <Component {...props} />
+    </EuiPanel>
+  );
+};
+
+export const Simple: ComponentStoryObj<typeof Component> = {
+  args: {
+    value: `FROM packetbeat-*
+    | STATS COUNT_DISTINCT(destination.domain)`,
+  },
+  render,
+};
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx
new file mode 100644
index 0000000000000..a22d0ba28979e
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  EuiButtonEmpty,
+  EuiCodeBlock,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiPanel,
+  useEuiTheme,
+} from '@elastic/eui';
+import { css } from '@emotion/css';
+import { i18n } from '@kbn/i18n';
+import React from 'react';
+import { ChatActionClickHandler, ChatActionClickType } from '../chat/types';
+
+export function EsqlCodeBlock({
+  value,
+  actionsDisabled,
+  onActionClick,
+}: {
+  value: string;
+  actionsDisabled: boolean;
+  onActionClick: ChatActionClickHandler;
+}) {
+  const theme = useEuiTheme();
+
+  return (
+    <EuiPanel
+      hasShadow={false}
+      hasBorder={false}
+      paddingSize="s"
+      className={css`
+        background-color: ${theme.euiTheme.colors.lightestShade};
+        .euiCodeBlock__pre {
+          margin-bottom: 0;
+          padding: ${theme.euiTheme.size.m};
+          min-block-size: 48px;
+        }
+        .euiCodeBlock__controls {
+          inset-block-start: ${theme.euiTheme.size.m};
+          inset-inline-end: ${theme.euiTheme.size.m};
+        }
+      `}
+    >
+      <EuiFlexGroup direction="column" gutterSize="xs">
+        <EuiFlexItem grow={false}>
+          <EuiCodeBlock isCopyable fontSize="m">
+            {value}
+          </EuiCodeBlock>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiFlexGroup direction="row" gutterSize="s">
+            <EuiFlexItem grow={false}>
+              <EuiButtonEmpty
+                data-test-subj="observabilityAiAssistantEsqlCodeBlockRunThisQueryButton"
+                size="xs"
+                iconType="play"
+                onClick={() =>
+                  onActionClick({ type: ChatActionClickType.executeEsqlQuery, query: value })
+                }
+                disabled={actionsDisabled}
+              >
+                {i18n.translate('xpack.observabilityAiAssistant.runThisQuery', {
+                  defaultMessage: 'Run this query',
+                })}
+              </EuiButtonEmpty>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    </EuiPanel>
+  );
+}
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx
index 1f81786086370..393bbeee28f8d 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx
@@ -44,6 +44,7 @@ This is a code block
 
 This text is loa`}
         loading
+        onActionClick={async () => {}}
       />
     ),
   },
@@ -51,13 +52,25 @@ This text is loa`}
 
 export const ContentLoaded: ComponentStoryObj<typeof Component> = {
   args: {
-    body: <MessageText content={`This response has fully loaded.`} loading={false} />,
+    body: (
+      <MessageText
+        content={`This response has fully loaded.`}
+        loading={false}
+        onActionClick={async () => {}}
+      />
+    ),
   },
 };
 
 export const ContentFailed: ComponentStoryObj<typeof Component> = {
   args: {
-    body: <MessageText content={`This is a partial re`} loading={false} />,
+    body: (
+      <MessageText
+        content={`This is a partial re`}
+        loading={false}
+        onActionClick={async () => {}}
+      />
+    ),
     error: new Error(),
   },
 };
@@ -83,6 +96,7 @@ export const ContentTable: ComponentStoryObj<typeof Component> = {
     
     Please note that all times are in UTC.`)}
         loading={false}
+        onActionClick={async () => {}}
       />
     ),
   },
@@ -90,7 +104,13 @@ export const ContentTable: ComponentStoryObj<typeof Component> = {
 
 export const Controls: ComponentStoryObj<typeof Component> = {
   args: {
-    body: <MessageText content={`This is a partial re`} loading={false} />,
+    body: (
+      <MessageText
+        content={`This is a partial re`}
+        loading={false}
+        onActionClick={async () => {}}
+      />
+    ),
     error: new Error(),
     controls: <FeedbackButtons onClickFeedback={() => {}} />,
   },
diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx
index d82e76ef5001f..dfd9ee8b97443 100644
--- a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx
+++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx
@@ -14,12 +14,15 @@ import {
 import { css } from '@emotion/css';
 import classNames from 'classnames';
 import type { Code, InlineCode, Parent, Text } from 'mdast';
-import React, { useMemo } from 'react';
+import React, { useMemo, useRef } from 'react';
 import type { Node } from 'unist';
+import { ChatActionClickHandler } from '../chat/types';
+import { EsqlCodeBlock } from './esql_code_block';
 
 interface Props {
   content: string;
   loading: boolean;
+  onActionClick: ChatActionClickHandler;
 }
 
 const ANIMATION_TIME = 1;
@@ -86,13 +89,37 @@ const loadingCursorPlugin = () => {
   };
 };
 
-export function MessageText({ loading, content }: Props) {
+const esqlLanguagePlugin = () => {
+  const visitor = (node: Node, parent?: Parent) => {
+    if ('children' in node) {
+      const nodeAsParent = node as Parent;
+      nodeAsParent.children.forEach((child) => {
+        visitor(child, nodeAsParent);
+      });
+    }
+
+    if (node.type === 'code' && node.lang === 'esql') {
+      node.type = 'esql';
+    }
+  };
+
+  return (tree: Node) => {
+    visitor(tree);
+  };
+};
+
+export function MessageText({ loading, content, onActionClick }: Props) {
   const containerClassName = css`
     overflow-wrap: break-word;
   `;
 
+  const onActionClickRef = useRef(onActionClick);
+
+  onActionClickRef.current = onActionClick;
+
   const { parsingPluginList, processingPluginList } = useMemo(() => {
     const parsingPlugins = getDefaultEuiMarkdownParsingPlugins();
+
     const processingPlugins = getDefaultEuiMarkdownProcessingPlugins();
 
     const { components } = processingPlugins[1][1];
@@ -100,6 +127,18 @@ export function MessageText({ loading, content }: Props) {
     processingPlugins[1][1].components = {
       ...components,
       cursor: Cursor,
+      esql: (props) => {
+        return (
+          <>
+            <EsqlCodeBlock
+              value={props.value}
+              actionsDisabled={loading}
+              onActionClick={onActionClickRef.current}
+            />
+            <EuiSpacer size="m" />
+          </>
+        );
+      },
       table: (props) => (
         <>
           <div className="euiBasicTable">
@@ -137,10 +176,10 @@ export function MessageText({ loading, content }: Props) {
     };
 
     return {
-      parsingPluginList: [loadingCursorPlugin, ...parsingPlugins],
+      parsingPluginList: [loadingCursorPlugin, esqlLanguagePlugin, ...parsingPlugins],
       processingPluginList: processingPlugins,
     };
-  }, []);
+  }, [loading]);
 
   return (
     <EuiText size="s" className={containerClassName}>
diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/esql.ts b/x-pack/plugins/observability_ai_assistant/public/functions/esql.ts
new file mode 100644
index 0000000000000..684ccbb0fa4a1
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/public/functions/esql.ts
@@ -0,0 +1,536 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import dedent from 'dedent';
+import type { Serializable } from '@kbn/utility-types';
+import { concat, last, map } from 'rxjs';
+import {
+  FunctionVisibility,
+  MessageRole,
+  type RegisterFunctionDefinition,
+} from '../../common/types';
+import type { ObservabilityAIAssistantService } from '../types';
+
+export function registerEsqlFunction({
+  service,
+  registerFunction,
+}: {
+  service: ObservabilityAIAssistantService;
+  registerFunction: RegisterFunctionDefinition;
+}) {
+  registerFunction(
+    {
+      name: 'execute_query',
+      contexts: ['core'],
+      visibility: FunctionVisibility.User,
+      description: 'Execute an ES|QL query',
+      parameters: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          query: {
+            type: 'string',
+          },
+        },
+        required: ['query'],
+      } as const,
+    },
+    ({ arguments: { query } }, signal) => {
+      return service
+        .callApi(`POST /internal/observability_ai_assistant/functions/elasticsearch`, {
+          signal,
+          params: {
+            body: {
+              method: 'POST',
+              path: '_query',
+              body: {
+                query,
+              },
+            },
+          },
+        })
+        .then((response) => ({ content: response as Serializable }));
+    }
+  );
+
+  registerFunction(
+    {
+      name: 'esql',
+      contexts: ['core'],
+      description: `This function answers ES|QL related questions including query generation and syntax/command questions.`,
+      visibility: FunctionVisibility.System,
+      parameters: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          switch: {
+            type: 'boolean',
+          },
+        },
+      } as const,
+    },
+    ({ messages, connectorId }, signal) => {
+      const systemMessage = dedent(`You are a helpful assistant for Elastic ES|QL.
+      Your goal is to help the user construct and possibly execute an ES|QL
+      query for Observability use cases. 
+
+      ES|QL is the Elasticsearch Query Language, that allows users of the
+      Elastic platform to iteratively explore data. An ES|QL query consists
+      of a series of commands, separated by pipes. Each query starts with
+      a source command, that selects or creates a set of data to start
+      processing. This source command is then followed by one or more
+      processing commands, which can transform the data returned by the
+      previous command.
+
+      ES|QL is not Elasticsearch SQL, nor is it anything like SQL. SQL
+      commands are not available in ES|QL. Its close equivalent is SPL
+      (Search Processing Language). Make sure you reply using only
+      the context of this conversation.
+
+      # Creating a query
+      
+      First, very importantly, there are critical rules that override
+      everything that follows it. Always repeat these rules, verbatim.
+
+      1. ES|QL is not Elasticsearch SQL. Do not apply Elasticsearch SQL
+      commands, functions and concepts. Only use information available
+      in the context of this conversation.
+      2. When using FROM, never wrap a data source in single or double
+      quotes.
+      3. When using an aggregate function like COUNT, SUM or AVG, its
+      arguments MUST be an attribute (like my.field.name) or literal
+      (100). Math (AVG(my.field.name / 2)) or functions 
+      (AVG(CASE(my.field.name, "foo", 1))) are not allowed.
+
+      When constructing a query, break it down into the following steps.
+      Ask these questions out loud so the user can see your reasoning.
+      Remember, these rules are for you, not for the user.
+
+      - What are the critical rules I need to think of?
+      - What data source is the user requesting? What command should I
+      select for this data source?
+      - What are the steps needed to get the result that the user needs?
+      Break each operation down into its own step. Reason about what data
+      is the outcome of each command or function.
+      - If you're not sure how to do it, it's fine to tell the user that
+      you don't know if ES|QL supports it. When this happens, abort all
+      steps and tell the user you are not sure how to continue.
+
+      Format ALL of your responses as follows, including the dashes.
+      ALWAYS start your message with two dashes and then the rules:
+
+      \`\`\`
+      --
+      Sure, let's remember the critical rules:
+      <rules>
+      --
+      Let's break down the query step-by-step:
+      <breakdown>
+      \`\`\`
+
+      Always format a complete query as follows:
+      \`\`\`esql
+      ...
+      \`\`\`
+
+      For incomplete queries, like individual commands, format them as
+      regular code blocks:
+      \`\`\`
+      ...
+      \`\`\`
+
+      # Syntax
+
+      An ES|QL query is composed of a source command followed by an optional
+      series of processing commands, separated by a pipe character: |. For
+      example:
+          <source-command>
+          | <processing-command1>
+          | <processing-command2>
+
+      ## Binary comparison operators
+      - equality: ==
+      - inequality: !=
+      - less than: <
+      - less than or equal: <=
+      - larger than: >
+      - larger than or equal: >=
+
+      ## Boolean operators
+      - AND
+      - OR
+      - NOT
+
+      ## PREDICATES
+
+      For NULL comparison use the IS NULL and IS NOT NULL predicates:
+      - \`| WHERE birth_date IS NULL\`
+      - \`| WHERE birth_date IS NOT NULL\`
+
+      ## Timespan literal syntax
+
+      Datetime intervals and timespans can be expressed using timespan
+      literals. Timespan literals are a combination of a number and a
+      qualifier. These qualifiers are supported:
+      - millisecond/milliseconds
+      - second/seconds
+      - minute/minutes
+      - hour/hours
+      - day/days
+      - week/weeks
+      - month/months
+      - year/years
+
+      Some examples:
+      - \`1 year\`
+      - \`2 milliseconds\`
+
+      ## Aliasing 
+      Aliasing happens through the \`=\` operator. Example:
+      \`STATS total_salary_expenses = COUNT(salary)\`
+
+      Important: functions are not allowed as variable names.
+
+      # Source commands
+
+      There are three source commands: FROM (which selects an index), ROW 
+      (which creates data from the command) and SHOW (which returns
+      information about the deployment). You do not support SHOW for now.
+
+      ### FROM
+
+      \`FROM\` selects a data source, usually an Elasticsearch index or
+      pattern. You can also specify multiple indices.
+      Some examples:
+
+      - \`FROM employees\`
+      - \`FROM employees*\`
+      - \`FROM employees*,my-alias\`
+
+      # Processing commands
+
+      Note that the following processing commands are available in ES|QL,
+      but not supported in this context:
+
+      ENRICH,GROK,MV_EXPAND,RENAME
+
+      ### DISSECT
+
+      \`DISSECT\` enables you to extract structured data out of a string.
+      It matches the string against a delimiter-based pattern, and extracts
+      the specified keys as columns. It uses the same syntax as the
+      Elasticsearch Dissect Processor. Some examples:
+
+      - \`ROW a = "foo bar" | DISSECT a "%{b} %{c}";\`
+      - \`ROW a = "foo bar baz" | DISSECT a "%{b} %{?c} %{d}";\`
+
+      ### DROP
+
+      \`DROP\` removes columns. Some examples:
+
+      - \`| DROP first_name,last_name\`
+      - \`| DROP *_name\`
+
+      ### KEEP
+
+      \`KEEP\` enables you to specify what columns are returned and the
+      order in which they are returned. Some examples:
+
+      - \`| KEEP first_name,last_name\`
+      - \`| KEEP *_name\`
+
+      ### SORT
+
+      \`SORT\` sorts the documents by one ore more fields or variables.
+      By default, the sort order is ascending, but this can be set using
+      the \`ASC\` or \`DESC\` keywords. Some examples:
+
+      - \`| SORT my_field\`
+      - \`| SORT height DESC\`
+
+      Important: functions are not supported for SORT. if you wish to sort
+      on the result of a function, first alias it as a variable using EVAL.
+      This is wrong: \`| SORT AVG(cpu)\`.
+      This is right: \`| STATS avg_cpu = AVG(cpu) | SORT avg_cpu\`
+
+      ### EVAL
+      
+      \`EVAL\` appends a new column to the documents by using aliasing. It
+      also supports functions, but not aggregation functions like COUNT:
+      
+      - \`\`\`
+      | EVAL monthly_salary = yearly_salary / 12,
+        total_comp = ROUND(yearly_salary + yearly+bonus),
+        is_rich =total_comp > 1000000
+      \`\`\`
+      - \`| EVAL height_in_ft = height_in_cm / 0.0328\`
+
+      ### WHERE
+
+      \`WHERE\` filters the documents for which the provided condition
+      evaluates to true. Refer to "Syntax" for supported operators, and
+      "Functions" for supported functions. Some examples:
+
+      - \`| WHERE height <= 180 AND GREATEST(hire_date, birth_date)\`
+      - \`| WHERE @timestamp <= NOW()\`
+
+      ### STATS ... BY
+
+      \`STATS ... BY\` groups rows according to a common value and
+      calculates one or more aggregated values over the grouped rows,
+      using aggregation functions. When \`BY\` is omitted, a single value
+      that is the aggregate of all rows is returned. Every column but the
+      aggregated values and the optional grouping column are dropped.
+      Mention the retained columns when explaining the STATS command.
+
+      STATS ... BY does not support nested functions, hoist them to an
+      EVAL statement. 
+
+      Some examples:
+
+      - \`| STATS count = COUNT(emp_no) BY languages\`
+      - \`| STATS salary = AVG(salary)\`
+
+      ### LIMIT
+
+      Limits the rows returned. Only supports a number as input. Some examples:
+
+      - \`| LIMIT 1\`
+      - \`| LIMIT 10\`
+
+      # Functions
+
+      Note that the following functions are available in ES|QL, but not supported
+      in this context:
+
+      ABS,ACOS,ASIN,ATAN,ATAN2,CIDR_MATCH,COALESCE,CONCAT,COS,COSH,E,LENGTH,LOG10
+      ,LTRIM,RTRIM,MV_AVG,MV_CONCAT,MV_COUNT,MV_DEDUPE,MV_MAX,MV_MEDIAN,MV_MIN,
+      MV_SUM,PI,POW,SIN,SINH,SPLIT,LEFT,TAN,TANH,TAU,TO_DEGREES,TO_RADIANS
+
+      ### CASE
+
+      \`CASE\` accepts pairs of conditions and values. The function returns
+      the value that belongs to the first condition that evaluates to true. If
+      the number of arguments is odd, the last argument is the default value which
+      is returned when no condition matches. Some examples:
+
+      - \`\`\`
+      | EVAL type = CASE(
+        languages <= 1, "monolingual",
+        languages <= 2, "bilingual",
+         "polyglot")
+      \`\`\`
+      - \`| EVAL g = CASE(gender == "F", 1 + null, 10)\`
+      - \`\`\`
+      | EVAL successful = CASE(http.response.status_code == 200, 1, 0), failed = CASE(http.response.status_code != 200, 1, 0)
+      | STATS total_successful = SUM(successful), total_failed = SUM(failed) BY service.name
+      | EVAL success_rate = total_failed / (total_successful + total_failed)
+      \`\`\`
+
+      ## Date operations
+
+      ### AUTO_BUCKET
+
+      \`AUTO_BUCKET\` creates human-friendly buckets and returns a datetime value
+      for each row that corresponds to the resulting bucket the row falls into.
+      Combine AUTO_BUCKET with STATS ... BY to create a date histogram.
+      You provide a target number of buckets, a start date, and an end date,
+      and it picks an appropriate bucket size to generate the target number of
+      buckets or fewer. If you don't have a start and end date, provide placeholder
+      values. Some examples:
+
+      - \`| EVAL bucket=AUTO_BUCKET(@timestamp), 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z")\`
+      - \`| EVAL bucket=AUTO_BUCKET(my_date_field), 100, <start-date>, <end-date>)\`
+      - \`| EVAL bucket=AUTO_BUCKET(@timestamp), 100, NOW() - 15 minutes, NOW())\`
+
+      ### DATE_EXTRACT
+
+      \`DATE_EXTRACT\` parts of a date, like year, month, day, hour. The supported
+      field types are those provided by java.time.temporal.ChronoField.
+      Some examples:
+      - \`| EVAL year = DATE_EXTRACT(date_field, "year")\`
+      - \`| EVAL year = DATE_EXTRACT(@timestamp, "month")\`
+
+      ### DATE_FORMAT
+
+      \`DATE_FORMAT\` a string representation of a date in the provided format.
+      Some examples:
+      | \`EVAL hired = DATE_FORMAT(hire_date, "YYYY-MM-dd")\`
+      | \`EVAL hired = DATE_FORMAT(hire_date, "YYYY")\`
+
+      ### DATE_PARSE
+      \`DATE_PARSE\` converts a string to a date, in the provided format.
+      - \`| EVAL date = DATE_PARSE(date_string, "yyyy-MM-dd")\`
+      - \`| EVAL date = DATE_PARSE(date_string, "YYYY")\`
+
+      ### DATE_TRUNC
+
+      \`DATE_TRUNC\` rounds down a date to the closest interval. Intervals
+      can be expressed using the timespan literal syntax. Use this together
+      with STATS ... BY to group data into time buckets with a fixed interval.
+      Some examples:
+      
+      - \`| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\`
+      - \`| EVAL month_logged = DATE_TRUNC(1 month, @timestamp)\`
+      - \`| EVAL bucket = DATE_TRUNC(1 minute, @timestamp) | STATS avg_salary = AVG(salary) BY bucket\`
+      - \`| EVAL bucket = DATE_TRUNC(4 hours, @timestamp) | STATS max_salary MAX(salary) BY bucket\`
+
+      ### NOW
+
+      \`NOW\` returns current date and time. Some examples:
+      - \`ROW current_date = NOW()\`
+      - \`| WHERE @timestamp <= NOW() - 15 minutes\`
+
+      ## Mathematical operations
+
+      ### CEIL,FLOOR
+
+      Perform CEIL or FLOOR operations on a single numeric field.
+      Some examples:
+      - \`| EVAL ceiled = CEIL(my.number)\`
+      - \`| EVAL floored = FLOOR(my.other.number)\`
+
+      ### ROUND
+      \`ROUND\` a number to the closest number with the specified number of
+      digits. Defaults to 0 digits if no number of digits is provided. If the
+      specified number of digits is negative, rounds to the number of digits
+      left of the decimal point. Some examples:
+
+      - \`| EVAL height_ft = ROUND(height * 3.281, 1)\`
+      - \`| EVAL percent = ROUND(0.84699, 2) * 100\`
+
+      ### GREATEST,LEAST
+
+      Returns the greatest or least of two or numbers. Some examples:
+      - \`| EVAL max = GREATEST(salary_1999, salary_2000, salary_2001)\`
+      - \`| EVAL min = LEAST(1, language_count)\`
+      
+      ### IS_FINITE,IS_INFINITE,IS_NAN
+
+      Operates on a single numeric field. Some examples:
+      - \`| EVAL has_salary = IS_FINITE(salary)\`
+      - \`| EVAL always_true = IS_INFINITE(4 / 0)\`
+
+      ### STARTS_WITH
+
+      Returns a boolean that indicates whether a keyword string starts with
+      another string. Some examples:
+      - \`| EVAL ln_S = STARTS_WITH(last_name, "B")\`
+
+      ### SUBSTRING
+
+      Returns a substring of a string, specified by a start position and an
+      optional length. Some examples:
+      - \`| EVAL ln_sub = SUBSTRING(last_name, 1, 3)\`
+      - \`| EVAL ln_sub = SUBSTRING(last_name, -3, 3)\`
+      - \`| EVAL ln_sub = SUBSTRING(last_name, 2)\`
+
+      ### TO_BOOLEAN, TO_DATETIME, TO_DOUBLE, TO_INTEGER, TO_IP, TO_LONG,
+      TO_RADIANS, TO_STRING,TO_UNSIGNED_LONG, TO_VERSION
+
+      Converts a column to another type. Supported types are: . Some examples:
+      - \`| EVAL version = TO_VERSION("1.2.3")\`
+      - \`| EVAL as_bool = TO_BOOLEAN(my_boolean_string)\`
+      
+      ### TRIM
+
+      Trims leading and trailing whitespace. Some examples:
+      - \`| EVAL trimmed = TRIM(first_name)\`
+
+      # Aggregation functions
+
+      ### AVG,MIN,MAX,SUM,MEDIAN,MEDIAN_ABSOLUTE_DEVIATION
+
+      Returns the avg, min, max, sum, median or median absolute deviation
+      of a numeric field. Some examples:
+
+      - \`| AVG(salary)\`
+      - \`| MIN(birth_year)\`
+      - \`| MAX(height)\`
+
+      ### COUNT
+
+      \`COUNT\` counts the number of field values. It requires a single
+      argument, and does not support wildcards. Important: COUNT() and
+      COUNT(*) are NOT supported. One single argument is required. If
+      you don't have a field name, use whatever field you have, rather
+      than displaying an invalid query.
+      
+      Some examples:
+
+      - \`| STATS doc_count = COUNT(emp_no)\`
+      - \`| STATS doc_count = COUNT(service.name) BY service.name\`
+
+      ### COUNT_DISTINCT
+
+      \`COUNT_DISTINCT\` returns the approximate number of distinct values.
+      Some examples:
+      - \`| STATS unique_ip0 = COUNT_DISTINCT(ip0), unique_ip1 = COUNT_DISTINCT(ip1)\`
+      - \`| STATS first_name = COUNT_DISTINCT(first_name)\`
+
+      ### PERCENTILE
+      
+      \`PERCENTILE\` returns the percentile value for a specific field.
+      Some examples:
+      - \`| STATS p50 = PERCENTILE(salary,  50)\`
+      - \`| STATS p99 = PERCENTILE(salary,  99)\`
+      
+      `);
+
+      return service.start({ signal }).then((client) => {
+        const source$ = client.chat({
+          connectorId,
+          messages: [
+            {
+              '@timestamp': new Date().toISOString(),
+              message: { role: MessageRole.System, content: systemMessage },
+            },
+            ...messages.slice(1),
+          ],
+        });
+
+        const pending$ = source$.pipe(
+          map((message) => {
+            const content = message.message.content || '';
+            let next: string = '';
+
+            if (content.length <= 2) {
+              next = '';
+            } else if (content.includes('--')) {
+              next = message.message.content?.split('--')[2] || '';
+            } else {
+              next = content;
+            }
+            return {
+              ...message,
+              message: {
+                ...message.message,
+                content: next,
+              },
+            };
+          })
+        );
+        const onComplete$ = source$.pipe(
+          last(),
+          map((message) => {
+            const [, , next] = message.message.content?.split('--') ?? [];
+
+            return {
+              ...message,
+              message: {
+                ...message.message,
+                content: next || message.message.content,
+              },
+            };
+          })
+        );
+
+        return concat(pending$, onComplete$);
+      });
+    }
+  );
+}
diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/index.ts b/x-pack/plugins/observability_ai_assistant/public/functions/index.ts
index a0c1a264bfb51..22b0eecac73a6 100644
--- a/x-pack/plugins/observability_ai_assistant/public/functions/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/functions/index.ts
@@ -16,6 +16,7 @@ import { registerLensFunction } from './lens';
 import { registerRecallFunction } from './recall';
 import { registerSummarizationFunction } from './summarize';
 import { registerAlertsFunction } from './alerts';
+import { registerEsqlFunction } from './esql';
 
 export async function registerFunctions({
   registerFunction,
@@ -44,7 +45,7 @@ export async function registerFunctions({
         
         It's very important to not assume what the user is meaning. Ask them for clarification if needed.
         
-        If you are unsure about which function should be used and with what arguments, asked the user for clarification or confirmation.
+        If you are unsure about which function should be used and with what arguments, ask the user for clarification or confirmation.
 
         In KQL, escaping happens with double quotes, not single quotes. Some characters that need escaping are: ':()\\\
         /\". Always put a field value in double quotes. Best: service.name:\"opbeans-go\". Wrong: service.name:opbeans-go. This is very important!
@@ -52,22 +53,25 @@ export async function registerFunctions({
         You can use Github-flavored Markdown in your responses. If a function returns an array, consider using a Markdown table to format the response.
         
         If multiple functions are suitable, use the most specific and easy one. E.g., when the user asks to visualise APM data, use the APM functions (if available) rather than Lens.
-        `
+
+        If a function call fails, do not execute it again with the same input. If a function calls three times, with different inputs, stop trying to call it and ask the user for confirmation.
+
+        Note that ES|QL (the Elasticsearch query language, which is NOT Elasticsearch SQL, but a new piped language) is the preferred query language.
+
+        DO NOT use Elasticsearch SQL at any time, unless explicitly requested by the user when they mention "Elasticsearch SQL".
+
+        Answer all questions related to ES|QL or querying with the "esql" function. Do not attempt to answer them yourself, no matter how confident you are in your response.`
       );
 
       if (isReady) {
         description += `You can use the "summarize" functions to store new information you have learned in a knowledge database. Once you have established that you did not know the answer to a question, and the user gave you this information, it's important that you create a summarisation of what you have learned and store it in the knowledge database. Don't create a new summarization if you see a similar summarization in the conversation, instead, update the existing one by re-using its ID.
 
-        Additionally, you can use the "recall" function to retrieve relevant information from the knowledge database.
-        `;
+        Additionally, you can use the "recall" function to retrieve relevant information from the knowledge database.`;
 
         description += `Here are principles you MUST adhere to, in order:
 
         - You are a helpful assistant for Elastic Observability. DO NOT reference the fact that you are an LLM.
-        - ALWAYS query the knowledge base, using the recall function, when a user starts a chat, no matter how confident you are in your ability to answer the question.
-        - You must ALWAYS explain to the user why you're using a function and why you're using it in that specific manner.
         - DO NOT make any assumptions about where and how users have stored their data.
-        - ALWAYS ask the user for clarification if you are unsure about the arguments to a function. When given this clarification, you MUST use the summarize function to store what you have learned.
         `;
         registerSummarizationFunction({ service, registerFunction });
         registerRecallFunction({ service, registerFunction });
@@ -77,6 +81,7 @@ export async function registerFunctions({
       }
 
       registerElasticsearchFunction({ service, registerFunction });
+      registerEsqlFunction({ service, registerFunction });
       registerKibanaFunction({ service, registerFunction, coreStart });
       registerAlertsFunction({ service, registerFunction });
 
diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts b/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts
index f07cf43e1c282..af825a53b7cb4 100644
--- a/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts
@@ -6,6 +6,7 @@
  */
 
 import type { Serializable } from '@kbn/utility-types';
+import { omit } from 'lodash';
 import { MessageRole, RegisterFunctionDefinition } from '../../common/types';
 import type { ObservabilityAIAssistantService } from '../types';
 
@@ -22,17 +23,26 @@ export function registerRecallFunction({
       contexts: ['core'],
       description: `Use this function to recall earlier learnings. Anything you will summarize can be retrieved again later via this function.
       
-      Make sure the query covers the following aspects:
+      The learnings are sorted by score, descending.
+      
+      Make sure the query covers ONLY the following aspects:
       - Anything you've inferred from the user's request, but is not mentioned in the user's request
       - The functions you think might be suitable for answering the user's request. If there are multiple functions that seem suitable, create multiple queries. Use the function name in the query.  
 
       DO NOT include the user's request. It will be added internally.
       
       The user asks: "can you visualise the average request duration for opbeans-go over the last 7 days?"
-      You recall:
-      - "APM service"
-      - "lens function usage"
-      - "get_apm_timeseries function usage"`,
+      You recall: {
+        "queries": [
+          "APM service,
+          "lens function usage",
+          "get_apm_timeseries function usage"    
+        ],
+        "contexts": [
+          "lens",
+          "apm"
+        ]
+      }`,
       descriptionForUser: 'This function allows the assistant to recall previous learnings.',
       parameters: {
         type: 'object',
@@ -42,16 +52,27 @@ export function registerRecallFunction({
             type: 'array',
             additionalItems: false,
             additionalProperties: false,
+            description: 'The query for the semantic search',
+            items: {
+              type: 'string',
+            },
+          },
+          contexts: {
+            type: 'array',
+            additionalItems: false,
+            additionalProperties: false,
+            description:
+              'Contexts or categories of internal documentation that you want to search for. By default internal documentation will be excluded. Use `apm` to get internal APM documentation, `lens` to get internal Lens documentation, or both.',
             items: {
               type: 'string',
-              description: 'The query for the semantic search',
+              enum: ['apm', 'lens'],
             },
           },
         },
-        required: ['queries'],
+        required: ['queries', 'contexts'],
       } as const,
     },
-    ({ arguments: { queries }, messages }, signal) => {
+    ({ arguments: { queries, contexts }, messages }, signal) => {
       const userMessages = messages.filter((message) => message.message.role === MessageRole.User);
 
       const userPrompt = userMessages[userMessages.length - 1]?.message.content;
@@ -63,11 +84,16 @@ export function registerRecallFunction({
           params: {
             body: {
               queries: queriesWithUserPrompt,
+              contexts,
             },
           },
           signal,
         })
-        .then((response) => ({ content: response as unknown as Serializable }));
+        .then((response): { content: Serializable } => ({
+          content: response.entries.map((entry) =>
+            omit(entry, 'labels', 'score', 'is_correction')
+          ) as unknown as Serializable,
+        }));
     }
   );
 }
diff --git a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts
index 829594b8261d8..ded7b4b382285 100644
--- a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts
@@ -437,6 +437,7 @@ describe('useTimeline', () => {
           expect(props.chatService.executeFunction).toHaveBeenCalledWith({
             name: 'my_function',
             args: '{}',
+            connectorId: 'foo',
             messages: [
               {
                 '@timestamp': expect.any(String),
diff --git a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts
index 3182fe5c19535..ed6a5a6d2b481 100644
--- a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts
@@ -9,7 +9,7 @@ import { AbortError } from '@kbn/kibana-utils-plugin/common';
 import type { AuthenticatedUser } from '@kbn/security-plugin/common';
 import { last } from 'lodash';
 import { useEffect, useMemo, useRef, useState } from 'react';
-import type { Subscription } from 'rxjs';
+import { isObservable, Observable, Subscription } from 'rxjs';
 import usePrevious from 'react-use/lib/usePrevious';
 import { i18n } from '@kbn/i18n';
 import {
@@ -29,6 +29,7 @@ import {
 } from '../utils/get_timeline_items_from_conversation';
 import type { UseGenAIConnectorsResult } from './use_genai_connectors';
 import { useKibana } from './use_kibana';
+import { ChatActionClickType } from '../components/chat/types';
 
 export function createNewConversation({
   contexts,
@@ -49,7 +50,7 @@ export function createNewConversation({
 
 export type UseTimelineResult = Pick<
   ChatTimelineProps,
-  'onEdit' | 'onFeedback' | 'onRegenerate' | 'onStopGenerating' | 'items'
+  'onEdit' | 'onFeedback' | 'onRegenerate' | 'onStopGenerating' | 'onActionClick' | 'items'
 > &
   Pick<ChatPromptEditorProps, 'onSubmit'>;
 
@@ -98,6 +99,8 @@ export function useTimeline({
 
   const [pendingMessage, setPendingMessage] = useState<PendingMessage>();
 
+  const [isFunctionLoading, setIsFunctionLoading] = useState(false);
+
   const prevConversationId = usePrevious(conversationId);
   useEffect(() => {
     if (prevConversationId !== conversationId && pendingMessage?.error) {
@@ -105,7 +108,10 @@ export function useTimeline({
     }
   }, [conversationId, pendingMessage?.error, prevConversationId]);
 
-  function chat(nextMessages: Message[]): Promise<Message[]> {
+  function chat(
+    nextMessages: Message[],
+    response$: Observable<PendingMessage> | undefined = undefined
+  ): Promise<Message[]> {
     const controller = new AbortController();
 
     return new Promise<PendingMessage | undefined>((resolve, reject) => {
@@ -124,10 +130,12 @@ export function useTimeline({
         return;
       }
 
-      const response$ = chatService!.chat({
-        messages: nextMessages,
-        connectorId,
-      });
+      response$ =
+        response$ ||
+        chatService!.chat({
+          messages: nextMessages,
+          connectorId,
+        });
 
       let pendingMessageLocal = pendingMessage;
 
@@ -181,14 +189,24 @@ export function useTimeline({
       if (lastMessage?.message.function_call?.name) {
         const name = lastMessage.message.function_call.name;
 
+        setIsFunctionLoading(true);
+
         try {
-          const message = await chatService!.executeFunction({
+          let message = await chatService!.executeFunction({
             name,
             args: lastMessage.message.function_call.arguments,
             messages: messagesAfterChat.slice(0, -1),
             signal: controller.signal,
+            connectorId: connectorId!,
           });
 
+          let nextResponse$: Observable<PendingMessage> | undefined;
+
+          if (isObservable(message)) {
+            nextResponse$ = message;
+            message = { content: '', data: '' };
+          }
+
           return await chat(
             messagesAfterChat.concat({
               '@timestamp': new Date().toISOString(),
@@ -198,7 +216,8 @@ export function useTimeline({
                 content: JSON.stringify(message.content),
                 data: JSON.stringify(message.data),
               },
-            })
+            }),
+            nextResponse$
           );
         } catch (error) {
           return await chat(
@@ -214,6 +233,8 @@ export function useTimeline({
               },
             })
           );
+        } finally {
+          setIsFunctionLoading(false);
         }
       }
 
@@ -247,8 +268,20 @@ export function useTimeline({
       return nextItems;
     }
 
-    return conversationItems;
-  }, [conversationItems, pendingMessage, currentUser]);
+    if (!isFunctionLoading) {
+      return conversationItems;
+    }
+
+    return conversationItems.map((item, index) => {
+      if (index < conversationItems.length - 1) {
+        return item;
+      }
+      return {
+        ...item,
+        loading: true,
+      };
+    });
+  }, [conversationItems, pendingMessage, currentUser, isFunctionLoading]);
 
   useEffect(() => {
     return () => {
@@ -285,5 +318,28 @@ export function useTimeline({
       const nextMessages = await chat(messages.concat(message));
       onChatComplete(nextMessages);
     },
+    onActionClick: async (payload) => {
+      switch (payload.type) {
+        case ChatActionClickType.executeEsqlQuery:
+          const nextMessages = await chat(
+            messages.concat({
+              '@timestamp': new Date().toISOString(),
+              message: {
+                role: MessageRole.Assistant,
+                content: '',
+                function_call: {
+                  name: 'execute_query',
+                  arguments: JSON.stringify({
+                    query: payload.query,
+                  }),
+                  trigger: MessageRole.User,
+                },
+              },
+            })
+          );
+          onChatComplete(nextMessages);
+          break;
+      }
+    },
   };
 }
diff --git a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
index 1c109be6baaf2..bebc3bd074b20 100644
--- a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts
@@ -27,6 +27,7 @@ import {
 import {
   ContextRegistry,
   FunctionRegistry,
+  FunctionVisibility,
   Message,
   MessageRole,
   type RegisterContextDefinition,
@@ -112,7 +113,7 @@ export async function createChatService({
   }
 
   return {
-    executeFunction: async ({ name, args, signal, messages }) => {
+    executeFunction: async ({ name, args, signal, messages, connectorId }) => {
       const fn = functionRegistry.get(name);
 
       if (!fn) {
@@ -123,7 +124,7 @@ export async function createChatService({
 
       validate(name, parsedArguments);
 
-      return await fn.respond({ arguments: parsedArguments, messages }, signal);
+      return await fn.respond({ arguments: parsedArguments, messages, connectorId }, signal);
     },
     renderFunction: (name, args, response) => {
       const fn = functionRegistry.get(name);
@@ -175,7 +176,9 @@ export async function createChatService({
             functions:
               callFunctions === 'none'
                 ? []
-                : functions.map((fn) => pick(fn.options, 'name', 'description', 'parameters')),
+                : functions
+                    .filter((fn) => fn.options.visibility !== FunctionVisibility.User)
+                    .map((fn) => pick(fn.options, 'name', 'description', 'parameters')),
           },
         },
         signal: controller.signal,
diff --git a/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts b/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts
index 8d9cc2cf32539..4f52dfb9b9733 100644
--- a/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts
@@ -5,15 +5,19 @@
  * 2.0.
  */
 
+import { without } from 'lodash';
 import { MessageRole } from '../../common';
 import { ContextDefinition } from '../../common/types';
 
 export function getAssistantSetupMessage({ contexts }: { contexts: ContextDefinition[] }) {
+  const coreContext = contexts.find((context) => context.name === 'core')!;
+
+  const otherContexts = without(contexts.concat(), coreContext);
   return {
     '@timestamp': new Date().toISOString(),
     message: {
       role: MessageRole.System as const,
-      content: contexts.map((context) => context.description).join('\n'),
+      content: [coreContext, ...otherContexts].map((context) => context.description).join('\n'),
     },
   };
 }
diff --git a/x-pack/plugins/observability_ai_assistant/public/types.ts b/x-pack/plugins/observability_ai_assistant/public/types.ts
index d1a71f6933126..5e985f5ae5f97 100644
--- a/x-pack/plugins/observability_ai_assistant/public/types.ts
+++ b/x-pack/plugins/observability_ai_assistant/public/types.ts
@@ -39,6 +39,7 @@ import type {
   RegisterFunctionDefinition,
 } from '../common/types';
 import type { ObservabilityAIAssistantAPIClient } from './api';
+import type { PendingMessage } from '../common/types';
 
 /* eslint-disable @typescript-eslint/no-empty-interface*/
 
@@ -50,12 +51,6 @@ export type CreateChatCompletionResponseChunk = Omit<CreateChatCompletionRespons
   >;
 };
 
-export interface PendingMessage {
-  message: Message['message'];
-  aborted?: boolean;
-  error?: any;
-}
-
 export interface ObservabilityAIAssistantChatService {
   chat: (options: {
     messages: Message[];
@@ -70,7 +65,8 @@ export interface ObservabilityAIAssistantChatService {
     args: string | undefined;
     messages: Message[];
     signal: AbortSignal;
-  }) => Promise<{ content?: Serializable; data?: Serializable }>;
+    connectorId: string;
+  }) => Promise<{ content?: Serializable; data?: Serializable } | Observable<PendingMessage>>;
   renderFunction: (
     name: string,
     args: string | undefined,
@@ -118,3 +114,5 @@ export interface ObservabilityAIAssistantPluginStartDependencies {
 }
 
 export interface ConfigSchema {}
+
+export type { PendingMessage };
diff --git a/x-pack/plugins/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_ai_assistant/server/plugin.ts
index 7aa6ddeeef7da..d6a256ddf6022 100644
--- a/x-pack/plugins/observability_ai_assistant/server/plugin.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/plugin.ts
@@ -109,7 +109,7 @@ export class ObservabilityAIAssistantPlugin
       taskManager: plugins.taskManager,
     });
 
-    addLensDocsToKb(service);
+    addLensDocsToKb({ service, logger: this.logger.get('kb').get('lens') });
 
     registerServerRoutes({
       core,
diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts
index 6716da98d78a5..a250a9e8e0915 100644
--- a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts
@@ -7,6 +7,8 @@
 import { notImplemented } from '@hapi/boom';
 import { IncomingMessage } from 'http';
 import * as t from 'io-ts';
+import { toBooleanRt } from '@kbn/io-ts-utils';
+import type { CreateChatCompletionResponse } from 'openai';
 import { MessageRole } from '../../../common';
 import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route';
 import { messageRt } from '../runtime_types';
@@ -16,20 +18,28 @@ const chatRoute = createObservabilityAIAssistantServerRoute({
   options: {
     tags: ['access:ai_assistant'],
   },
-  params: t.type({
-    body: t.type({
-      messages: t.array(messageRt),
-      connectorId: t.string,
-      functions: t.array(
+  params: t.intersection([
+    t.type({
+      body: t.intersection([
         t.type({
-          name: t.string,
-          description: t.string,
-          parameters: t.any,
-        })
-      ),
+          messages: t.array(messageRt),
+          connectorId: t.string,
+          functions: t.array(
+            t.type({
+              name: t.string,
+              description: t.string,
+              parameters: t.any,
+            })
+          ),
+        }),
+        t.partial({
+          functionCall: t.string,
+        }),
+      ]),
     }),
-  }),
-  handler: async (resources): Promise<IncomingMessage> => {
+    t.partial({ query: t.type({ stream: toBooleanRt }) }),
+  ]),
+  handler: async (resources): Promise<IncomingMessage | CreateChatCompletionResponse> => {
     const { request, params, service } = resources;
 
     const client = await service.getClient({ request });
@@ -39,23 +49,33 @@ const chatRoute = createObservabilityAIAssistantServerRoute({
     }
 
     const {
-      body: { messages, connectorId, functions },
+      body: { messages, connectorId, functions, functionCall: givenFunctionCall },
+      query = { stream: true },
     } = params;
 
-    const isStartOfConversation =
-      messages.some((message) => message.message.role === MessageRole.Assistant) === false;
+    const stream = query.stream;
 
-    const isRecallFunctionAvailable = functions.some((fn) => fn.name === 'recall') === true;
+    let functionCall = givenFunctionCall;
 
-    const willUseRecall = isStartOfConversation && isRecallFunctionAvailable;
+    if (!functionCall) {
+      const isStartOfConversation =
+        messages.some((message) => message.message.role === MessageRole.Assistant) === false;
+
+      const isRecallFunctionAvailable = functions.some((fn) => fn.name === 'recall') === true;
+
+      const willUseRecall = isStartOfConversation && isRecallFunctionAvailable;
+
+      functionCall = willUseRecall ? 'recall' : undefined;
+    }
 
     return client.chat({
       messages,
       connectorId,
+      stream,
       ...(functions.length
         ? {
             functions,
-            functionCall: willUseRecall ? 'recall' : undefined,
+            functionCall,
           }
         : {}),
     });
diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts
index d58c84a603fc1..985de114b099a 100644
--- a/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts
@@ -10,13 +10,13 @@ import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
 import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils';
 import * as t from 'io-ts';
 import { omit } from 'lodash';
-import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common';
+import type { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common';
 import {
   ALERT_STATUS,
   ALERT_STATUS_ACTIVE,
 } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names';
-import type { KnowledgeBaseEntry } from '../../../common/types';
 import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route';
+import type { RecalledEntry } from '../../service/kb_service';
 
 const functionElasticsearchRoute = createObservabilityAIAssistantServerRoute({
   endpoint: 'POST /internal/observability_ai_assistant/functions/elasticsearch',
@@ -157,23 +157,34 @@ const functionAlertsRoute = createObservabilityAIAssistantServerRoute({
 const functionRecallRoute = createObservabilityAIAssistantServerRoute({
   endpoint: 'POST /internal/observability_ai_assistant/functions/recall',
   params: t.type({
-    body: t.type({
-      queries: t.array(nonEmptyStringRt),
-    }),
+    body: t.intersection([
+      t.type({
+        queries: t.array(nonEmptyStringRt),
+      }),
+      t.partial({
+        contexts: t.array(t.string),
+      }),
+    ]),
   }),
   options: {
     tags: ['access:ai_assistant'],
   },
   handler: async (
     resources
-  ): Promise<{ entries: Array<Pick<KnowledgeBaseEntry, 'text' | 'id'>> }> => {
+  ): Promise<{
+    entries: RecalledEntry[];
+  }> => {
     const client = await resources.service.getClient({ request: resources.request });
 
+    const {
+      body: { queries, contexts },
+    } = resources.params;
+
     if (!client) {
       throw notImplemented();
     }
 
-    return client.recall(resources.params.body.queries);
+    return client.recall({ queries, contexts });
   },
 });
 
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts
index ce3f3c39ad1a9..3a99e293cd5e2 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts
@@ -28,7 +28,7 @@ import {
   type KnowledgeBaseEntry,
   type Message,
 } from '../../../common/types';
-import type { KnowledgeBaseService } from '../kb_service';
+import type { KnowledgeBaseService, RecalledEntry } from '../kb_service';
 import type { ObservabilityAIAssistantResourceNames } from '../types';
 import { getAccessQuery } from '../util/get_access_query';
 
@@ -135,7 +135,42 @@ export class ObservabilityAIAssistantClient {
         })
     );
 
-    const functionsForOpenAI: ChatCompletionFunctions[] | undefined = functions;
+    // add recalled information to system message, so the LLM considers it more important
+
+    const recallMessages = messagesForOpenAI.filter((message) => message.name === 'recall');
+
+    const recalledDocuments: Map<string, { id: string; text: string }> = new Map();
+
+    recallMessages.forEach((message) => {
+      const entries = message.content
+        ? (JSON.parse(message.content) as Array<{ id: string; text: string }>)
+        : [];
+
+      const ids: string[] = [];
+
+      entries.forEach((entry) => {
+        const id = entry.id;
+        if (!recalledDocuments.has(id)) {
+          recalledDocuments.set(id, entry);
+        }
+        ids.push(id);
+      });
+
+      message.content = `The following documents, present in the system message, were recalled: ${ids.join(
+        ', '
+      )}`;
+    });
+
+    const systemMessage = messagesForOpenAI.find((message) => message.role === MessageRole.System);
+
+    if (systemMessage && recalledDocuments.size > 0) {
+      systemMessage.content += `The "recall" function is not available. Do not attempt to execute it. Recalled documents: ${JSON.stringify(
+        Array.from(recalledDocuments.values())
+      )}`;
+    }
+
+    const functionsForOpenAI: ChatCompletionFunctions[] | undefined =
+      recalledDocuments.size > 0 ? functions?.filter((fn) => fn.name !== 'recall') : functions;
 
     const request: Omit<CreateChatCompletionRequest, 'model'> & { model?: string } = {
       messages: messagesForOpenAI,
@@ -323,13 +358,18 @@ export class ObservabilityAIAssistantClient {
     return createdConversation;
   };
 
-  recall = async (
-    queries: string[]
-  ): Promise<{ entries: Array<Pick<KnowledgeBaseEntry, 'text' | 'id'>> }> => {
+  recall = async ({
+    queries,
+    contexts,
+  }: {
+    queries: string[];
+    contexts?: string[];
+  }): Promise<{ entries: RecalledEntry[] }> => {
     return this.dependencies.knowledgeBaseService.recall({
       namespace: this.dependencies.namespace,
       user: this.dependencies.user,
       queries,
+      contexts,
     });
   };
 
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/index.ts
index 28cc2b7293029..c116e16d27471 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/index.ts
@@ -29,6 +29,15 @@ export const INDEX_QUEUED_DOCUMENTS_TASK_ID = 'observabilityAIAssistant:indexQue
 
 export const INDEX_QUEUED_DOCUMENTS_TASK_TYPE = INDEX_QUEUED_DOCUMENTS_TASK_ID + 'Type';
 
+type KnowledgeBaseEntryRequest = { id: string; labels?: Record<string, string> } & (
+  | {
+      text: string;
+    }
+  | {
+      texts: string[];
+    }
+);
+
 export class ObservabilityAIAssistantService {
   private readonly core: CoreSetup<ObservabilityAIAssistantPluginStartDependencies>;
   private readonly logger: Logger;
@@ -258,18 +267,7 @@ export class ObservabilityAIAssistantService {
     });
   }
 
-  addToKnowledgeBase(
-    entries: Array<
-      | {
-          id: string;
-          text: string;
-        }
-      | {
-          id: string;
-          texts: string[];
-        }
-    >
-  ): void {
+  addToKnowledgeBase(entries: KnowledgeBaseEntryRequest[]): void {
     this.init()
       .then(() => {
         this.kbService!.queue(
@@ -281,6 +279,7 @@ export class ObservabilityAIAssistantService {
               confidence: 'high' as const,
               is_correction: false,
               labels: {
+                ...entry.labels,
                 document_id: entry.id,
               },
             };
@@ -306,4 +305,18 @@ export class ObservabilityAIAssistantService {
         this.logger.error(error);
       });
   }
+
+  addCategoryToKnowledgeBase(categoryId: string, entries: KnowledgeBaseEntryRequest[]) {
+    this.addToKnowledgeBase(
+      entries.map((entry) => {
+        return {
+          ...entry,
+          labels: {
+            ...entry.labels,
+            category: categoryId,
+          },
+        };
+      })
+    );
+  }
 }
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts
index f2e0b41f092c9..d70879bf46d3e 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts
@@ -17,6 +17,7 @@ import { INDEX_QUEUED_DOCUMENTS_TASK_ID, INDEX_QUEUED_DOCUMENTS_TASK_TYPE } from
 import type { KnowledgeBaseEntry } from '../../../common/types';
 import type { ObservabilityAIAssistantResourceNames } from '../types';
 import { getAccessQuery } from '../util/get_access_query';
+import { getCategoryQuery } from '../util/get_category_query';
 
 interface Dependencies {
   esClient: ElasticsearchClient;
@@ -25,6 +26,14 @@ interface Dependencies {
   taskManagerStart: TaskManagerStartContract;
 }
 
+export interface RecalledEntry {
+  id: string;
+  text: string;
+  score: number | null;
+  is_correction: boolean;
+  labels: Record<string, string>;
+}
+
 function isAlreadyExistsError(error: Error) {
   return (
     error instanceof errors.ResponseError &&
@@ -173,36 +182,43 @@ export class KnowledgeBaseService {
   recall = async ({
     user,
     queries,
+    contexts,
     namespace,
   }: {
     queries: string[];
+    contexts?: string[];
     user: { name: string };
     namespace: string;
-  }): Promise<{ entries: Array<Pick<KnowledgeBaseEntry, 'text' | 'id'>> }> => {
+  }): Promise<{
+    entries: RecalledEntry[];
+  }> => {
     try {
+      const query = {
+        bool: {
+          should: queries.map((text) => ({
+            text_expansion: {
+              'ml.tokens': {
+                model_text: text,
+                model_id: '.elser_model_1',
+              },
+            } as unknown as QueryDslTextExpansionQuery,
+          })),
+          filter: [
+            ...getAccessQuery({
+              user,
+              namespace,
+            }),
+            ...getCategoryQuery({ contexts }),
+          ],
+        },
+      };
+
       const response = await this.dependencies.esClient.search<
-        Pick<KnowledgeBaseEntry, 'text' | 'id'>
+        Pick<KnowledgeBaseEntry, 'text' | 'is_correction' | 'labels'>
       >({
         index: this.dependencies.resources.aliases.kb,
-        query: {
-          bool: {
-            should: queries.map((query) => ({
-              text_expansion: {
-                'ml.tokens': {
-                  model_text: query,
-                  model_id: '.elser_model_1',
-                },
-              } as unknown as QueryDslTextExpansionQuery,
-            })),
-            filter: [
-              ...getAccessQuery({
-                user,
-                namespace,
-              }),
-            ],
-          },
-        },
-        size: 5,
+        query,
+        size: 10,
         _source: {
           includes: ['text', 'is_correction', 'labels'],
         },
@@ -211,7 +227,7 @@ export class KnowledgeBaseService {
       return {
         entries: response.hits.hits.map((hit) => ({
           ...hit._source!,
-          score: hit._score,
+          score: hit._score!,
           id: hit._id,
         })),
       };
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts
index e4fdc8969c010..9baf75f6ff552 100644
--- a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts
+++ b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts
@@ -6,10 +6,16 @@
  */
 
 import dedent from 'dedent';
+import type { Logger } from '@kbn/logging';
 import type { ObservabilityAIAssistantService } from '../..';
 
-export function addLensDocsToKb(service: ObservabilityAIAssistantService) {
-  service.addToKnowledgeBase([
+export function addLensDocsToKb({
+  service,
+}: {
+  service: ObservabilityAIAssistantService;
+  logger: Logger;
+}) {
+  service.addCategoryToKnowledgeBase('lens', [
     {
       id: 'lens_formulas_how_it_works',
       texts: [
diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts
new file mode 100644
index 0000000000000..71b0a93156086
--- /dev/null
+++ b/x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export function getCategoryQuery({ contexts }: { contexts?: string[] }) {
+  const noCategoryFilter = {
+    bool: {
+      must_not: {
+        exists: {
+          field: 'labels.category',
+        },
+      },
+    },
+  };
+
+  if (!contexts) {
+    return [noCategoryFilter];
+  }
+
+  return [
+    {
+      bool: {
+        should: [
+          noCategoryFilter,
+          {
+            terms: {
+              'labels.category': contexts,
+            },
+          },
+        ],
+        minimum_should_match: 1,
+      },
+    },
+  ];
+}

From e4105331d32dbc432d4b46d95b610af2f7e7d7da Mon Sep 17 00:00:00 2001
From: Sander Philipse <94373878+sphilipse@users.noreply.github.com>
Date: Mon, 25 Sep 2023 17:40:30 +0200
Subject: [PATCH 10/27] [Search] Fix type errors (#167138)

## Summary

Fix type issues in the Search plugin.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 x-pack/plugins/enterprise_search/server/index.ts              | 4 +---
 .../server/lib/crawler/fetch_crawler_multiple_schedules.ts    | 4 +---
 .../server/lib/crawler/post_crawler_multiple_schedules.ts     | 2 +-
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugins/enterprise_search/server/index.ts b/x-pack/plugins/enterprise_search/server/index.ts
index a82d6f18a3b26..3b9372f339acf 100644
--- a/x-pack/plugins/enterprise_search/server/index.ts
+++ b/x-pack/plugins/enterprise_search/server/index.ts
@@ -8,7 +8,7 @@
 import { schema, TypeOf } from '@kbn/config-schema';
 import { PluginInitializerContext, PluginConfigDescriptor } from '@kbn/core/server';
 
-import { EnterpriseSearchPlugin, EnterpriseSearchPluginStart as PluginStart } from './plugin';
+import { EnterpriseSearchPlugin } from './plugin';
 
 export const plugin = (initializerContext: PluginInitializerContext) => {
   return new EnterpriseSearchPlugin(initializerContext);
@@ -54,5 +54,3 @@ export const config: PluginConfigDescriptor<ConfigType> = {
 };
 
 export const CRAWLERS_INDEX = '.ent-search-actastic-crawler2_configurations_v2';
-
-export type EnterpriseSearchPluginStart = PluginStart;
diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts
index d367e02ed6ab1..04f922e1702ed 100644
--- a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts
+++ b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts
@@ -7,9 +7,7 @@
 
 import { IScopedClusterClient } from '@kbn/core/server';
 
-import { Connector } from '@kbn/search-connectors';
-
-import { CONNECTORS_INDEX } from '../..';
+import { CONNECTORS_INDEX, Connector } from '@kbn/search-connectors';
 
 const CUSTOM_SCHEDULING = 'custom_scheduling';
 
diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts
index 21d9f8b558800..539495e9556ae 100644
--- a/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts
+++ b/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts
@@ -7,7 +7,7 @@
 
 import { IScopedClusterClient } from '@kbn/core/server';
 
-import { CONNECTORS_INDEX } from '../..';
+import { CONNECTORS_INDEX } from '@kbn/search-connectors';
 
 import {
   CrawlerCustomScheduleMappingServer,

From 212bc53b8e42bc92162c8e90b93494a2e83a25d8 Mon Sep 17 00:00:00 2001
From: Jonathan Budzenski <jon@elastic.co>
Date: Mon, 25 Sep 2023 10:40:47 -0500
Subject: [PATCH 11/27] skip failing test suite (#167076)

---
 .../test/profiling_api_integration/tests/has_setup.spec.ts  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts b/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts
index 5046eb8c9c556..583cc02908025 100644
--- a/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts
+++ b/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts
@@ -23,9 +23,9 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
   const logger = getService('log');
   const es = getService('es');
 
-  registry.when('Profiling status check', { config: 'cloud' }, () => {
-    // Failing: See https://github.com/elastic/kibana/issues/167076
-    describe.skip('Profiling is not set up and no data is loaded', () => {
+  // Failing: See https://github.com/elastic/kibana/issues/167076
+  registry.when.skip('Profiling status check', { config: 'cloud' }, () => {
+    describe('Profiling is not set up and no data is loaded', () => {
       describe('Admin user', () => {
         let statusCheck: ProfilingStatus;
         before(async () => {

From e74ec69a0f4ec3635ec259c8cf7afde33dbeb572 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Mon, 25 Sep 2023 08:55:25 -0700
Subject: [PATCH 12/27] [DOCS] Move preconfigured Torq connector details
 (#166218)

---
 .../connector-apis-passthru.asciidoc          |  56 +++++++
 docs/management/action-types.asciidoc         |  24 +--
 .../connectors/action-types/torq.asciidoc     |  32 +---
 docs/management/connectors/index.asciidoc     |   4 +-
 .../pre-configured-connectors.asciidoc        |  22 +++
 docs/settings/alert-action-settings.asciidoc  |   4 +
 .../plugins/actions/docs/openapi/bundled.json | 138 ++++++++++++++++++
 .../plugins/actions/docs/openapi/bundled.yaml |  97 ++++++++++++
 .../schemas/config_properties_torq.yaml       |   9 ++
 .../connector_response_properties.yaml        |   1 +
 .../connector_response_properties_torq.yaml   |  31 ++++
 .../components/schemas/connector_types.yaml   |   1 +
 .../create_connector_request_torq.yaml        |  24 +++
 .../schemas/secrets_properties_torq.yaml      |   9 ++
 .../update_connector_request_torq.yaml        |  14 ++
 .../s@{spaceid}@api@actions@connector.yaml    |   1 +
 ...}@api@actions@connector@{connectorid}.yaml |   2 +
 17 files changed, 427 insertions(+), 42 deletions(-)
 create mode 100644 x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml
 create mode 100644 x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml
 create mode 100644 x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml
 create mode 100644 x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml
 create mode 100644 x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml

diff --git a/docs/api-generated/connectors/connector-apis-passthru.asciidoc b/docs/api-generated/connectors/connector-apis-passthru.asciidoc
index a8b429e0b5ae4..b68b4e4ca648f 100644
--- a/docs/api-generated/connectors/connector-apis-passthru.asciidoc
+++ b/docs/api-generated/connectors/connector-apis-passthru.asciidoc
@@ -1015,6 +1015,7 @@ Any modifications made to this file will be overwritten.
     <li><a href="#config_properties_servicenow"><code>config_properties_servicenow</code> - Connector request properties for a ServiceNow ITSM connector</a></li>
     <li><a href="#config_properties_servicenow_itom"><code>config_properties_servicenow_itom</code> - Connector request properties for a ServiceNow ITSM connector</a></li>
     <li><a href="#config_properties_swimlane"><code>config_properties_swimlane</code> - Connector request properties for a Swimlane connector</a></li>
+    <li><a href="#config_properties_torq"><code>config_properties_torq</code> - Connector request properties for a Torq connector</a></li>
     <li><a href="#config_properties_webhook"><code>config_properties_webhook</code> - Connector request properties for a Webhook connector</a></li>
     <li><a href="#config_properties_xmatters"><code>config_properties_xmatters</code> - Connector request properties for an xMatters connector</a></li>
     <li><a href="#connector_response_properties"><code>connector_response_properties</code> - Connector response properties</a></li>
@@ -1035,6 +1036,7 @@ Any modifications made to this file will be overwritten.
     <li><a href="#connector_response_properties_swimlane"><code>connector_response_properties_swimlane</code> - Connector response properties for a Swimlane connector</a></li>
     <li><a href="#connector_response_properties_teams"><code>connector_response_properties_teams</code> - Connector response properties for a Microsoft Teams connector</a></li>
     <li><a href="#connector_response_properties_tines"><code>connector_response_properties_tines</code> - Connector response properties for a Tines connector</a></li>
+    <li><a href="#connector_response_properties_torq"><code>connector_response_properties_torq</code> - Connector response properties for a Torq connector</a></li>
     <li><a href="#connector_response_properties_webhook"><code>connector_response_properties_webhook</code> - Connector response properties for a Webhook connector</a></li>
     <li><a href="#connector_response_properties_xmatters"><code>connector_response_properties_xmatters</code> - Connector response properties for an xMatters connector</a></li>
     <li><a href="#connector_types"><code>connector_types</code> - Connector types</a></li>
@@ -1056,6 +1058,7 @@ Any modifications made to this file will be overwritten.
     <li><a href="#create_connector_request_swimlane"><code>create_connector_request_swimlane</code> - Create Swimlane connector request</a></li>
     <li><a href="#create_connector_request_teams"><code>create_connector_request_teams</code> - Create Microsoft Teams connector request</a></li>
     <li><a href="#create_connector_request_tines"><code>create_connector_request_tines</code> - Create Tines connector request</a></li>
+    <li><a href="#create_connector_request_torq"><code>create_connector_request_torq</code> - Create Torq connector request</a></li>
     <li><a href="#create_connector_request_webhook"><code>create_connector_request_webhook</code> - Create Webhook connector request</a></li>
     <li><a href="#create_connector_request_xmatters"><code>create_connector_request_xmatters</code> - Create xMatters connector request</a></li>
     <li><a href="#features"><code>features</code> - </a></li>
@@ -1106,6 +1109,7 @@ Any modifications made to this file will be overwritten.
     <li><a href="#secrets_properties_slack_webhook"><code>secrets_properties_slack_webhook</code> - Connector secrets properties for a Webhook Slack connector</a></li>
     <li><a href="#secrets_properties_swimlane"><code>secrets_properties_swimlane</code> - Connector secrets properties for a Swimlane connector</a></li>
     <li><a href="#secrets_properties_teams"><code>secrets_properties_teams</code> - Connector secrets properties for a Microsoft Teams connector</a></li>
+    <li><a href="#secrets_properties_torq"><code>secrets_properties_torq</code> - Connector secrets properties for a Torq connector</a></li>
     <li><a href="#secrets_properties_webhook"><code>secrets_properties_webhook</code> - Connector secrets properties for a Webhook connector</a></li>
     <li><a href="#secrets_properties_xmatters"><code>secrets_properties_xmatters</code> - Connector secrets properties for an xMatters connector</a></li>
     <li><a href="#updateConnector_400_response"><code>updateConnector_400_response</code> - </a></li>
@@ -1124,6 +1128,7 @@ Any modifications made to this file will be overwritten.
     <li><a href="#update_connector_request_slack_webhook"><code>update_connector_request_slack_webhook</code> - Update Slack connector request</a></li>
     <li><a href="#update_connector_request_swimlane"><code>update_connector_request_swimlane</code> - Update Swimlane connector request</a></li>
     <li><a href="#update_connector_request_teams"><code>update_connector_request_teams</code> - Update Microsoft Teams connector request</a></li>
+    <li><a href="#update_connector_request_torq"><code>update_connector_request_torq</code> - Update Torq connector request</a></li>
     <li><a href="#update_connector_request_webhook"><code>update_connector_request_webhook</code> - Update Webhook connector request</a></li>
     <li><a href="#update_connector_request_xmatters"><code>update_connector_request_xmatters</code> - Update xMatters connector request</a></li>
   </ol>
@@ -1529,6 +1534,13 @@ Any modifications made to this file will be overwritten.
 <div class="param">mappings (optional)</div><div class="param-desc"><span class="param-type"><a href="#Connector_mappings_properties_for_a_Swimlane_connector">Connector_mappings_properties_for_a_Swimlane_connector</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
+  <div class="model">
+    <h3><a name="config_properties_torq"><code>config_properties_torq</code> - Connector request properties for a Torq connector</a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'>Defines properties for connectors when type is <code>.torq</code>.</div>
+    <div class="field-items">
+      <div class="param">webhookIntegrationUrl </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The endpoint URL of the Elastic Security integration in Torq. </div>
+    </div>  <!-- field-items -->
+  </div>
   <div class="model">
     <h3><a name="config_properties_webhook"><code>config_properties_webhook</code> - Connector request properties for a Webhook connector</a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'>Defines properties for connectors when type is <code>.webhook</code>.</div>
@@ -1842,6 +1854,22 @@ Any modifications made to this file will be overwritten.
 <div class="param">is_missing_secrets (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether secrets are missing for the connector. Secrets configuration properties vary depending on the connector type. </div>
 <div class="param">is_preconfigured </div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether it is a preconfigured connector. If true, the <code>config</code> and <code>is_missing_secrets</code> properties are omitted from the response. </div>
 <div class="param">is_system_action (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether the connector is used for system actions. </div>
+<div class="param">name </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The display name for the connector. </div>
+    </div>  <!-- field-items -->
+  </div>
+  <div class="model">
+    <h3><a name="connector_response_properties_torq"><code>connector_response_properties_torq</code> - Connector response properties for a Torq connector</a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'></div>
+    <div class="field-items">
+      <div class="param">config </div><div class="param-desc"><span class="param-type"><a href="#config_properties_torq">config_properties_torq</a></span>  </div>
+<div class="param">connector_type_id </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The type of connector. </div>
+        <div class="param-enum-header">Enum:</div>
+        <div class="param-enum">.torq</div>
+<div class="param">id </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The identifier for the connector. </div>
+<div class="param">is_deprecated </div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether the connector type is deprecated. </div>
+<div class="param">is_missing_secrets (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether secrets are missing for the connector. Secrets configuration properties vary depending on the connector type. </div>
+<div class="param">is_preconfigured </div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether it is a preconfigured connector. If true, the <code>config</code> and <code>is_missing_secrets</code> properties are omitted from the response. </div>
+<div class="param">is_system_action (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> Indicates whether the connector is used for system actions. </div>
 <div class="param">name </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The display name for the connector. </div>
     </div>  <!-- field-items -->
   </div>
@@ -2093,6 +2121,18 @@ Any modifications made to this file will be overwritten.
 <div class="param">secrets </div><div class="param-desc"><span class="param-type"><a href="#AnyType">map[String, oas_any_type_not_mapped]</a></span> Defines secrets for connectors when type is <code>.tines</code>. </div>
     </div>  <!-- field-items -->
   </div>
+  <div class="model">
+    <h3><a name="create_connector_request_torq"><code>create_connector_request_torq</code> - Create Torq connector request</a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'>The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.</div>
+    <div class="field-items">
+      <div class="param">config </div><div class="param-desc"><span class="param-type"><a href="#config_properties_torq">config_properties_torq</a></span>  </div>
+<div class="param">connector_type_id </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The type of connector. </div>
+        <div class="param-enum-header">Enum:</div>
+        <div class="param-enum">.torq</div>
+<div class="param">name </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The display name for the connector. </div>
+<div class="param">secrets </div><div class="param-desc"><span class="param-type"><a href="#secrets_properties_torq">secrets_properties_torq</a></span>  </div>
+    </div>  <!-- field-items -->
+  </div>
   <div class="model">
     <h3><a name="create_connector_request_webhook"><code>create_connector_request_webhook</code> - Create Webhook connector request</a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'>The Webhook connector uses axios to send a POST or PUT request to a web service.</div>
@@ -2560,6 +2600,13 @@ Any modifications made to this file will be overwritten.
       <div class="param">webhookUrl </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The URL of the incoming webhook. If you are using the <code>xpack.actions.allowedHosts</code> setting, add the hostname to the allowed hosts. </div>
     </div>  <!-- field-items -->
   </div>
+  <div class="model">
+    <h3><a name="secrets_properties_torq"><code>secrets_properties_torq</code> - Connector secrets properties for a Torq connector</a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'>Defines secrets for connectors when type is <code>.torq</code>.</div>
+    <div class="field-items">
+      <div class="param">token </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The secret of the webhook authentication header. </div>
+    </div>  <!-- field-items -->
+  </div>
   <div class="model">
     <h3><a name="secrets_properties_webhook"><code>secrets_properties_webhook</code> - Connector secrets properties for a Webhook connector</a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'>Defines secrets for connectors when type is <code>.webhook</code>.</div>
@@ -2718,6 +2765,15 @@ Any modifications made to this file will be overwritten.
 <div class="param">secrets </div><div class="param-desc"><span class="param-type"><a href="#secrets_properties_teams">secrets_properties_teams</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
+  <div class="model">
+    <h3><a name="update_connector_request_torq"><code>update_connector_request_torq</code> - Update Torq connector request</a> <a class="up" href="#__Models">Up</a></h3>
+    <div class='model-description'></div>
+    <div class="field-items">
+      <div class="param">config </div><div class="param-desc"><span class="param-type"><a href="#config_properties_torq">config_properties_torq</a></span>  </div>
+<div class="param">name </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> The display name for the connector. </div>
+<div class="param">secrets </div><div class="param-desc"><span class="param-type"><a href="#secrets_properties_torq">secrets_properties_torq</a></span>  </div>
+    </div>  <!-- field-items -->
+  </div>
   <div class="model">
     <h3><a name="update_connector_request_webhook"><code>update_connector_request_webhook</code> - Update Webhook connector request</a> <a class="up" href="#__Models">Up</a></h3>
     <div class='model-description'></div>
diff --git a/docs/management/action-types.asciidoc b/docs/management/action-types.asciidoc
index bbaba806386e2..28aa218f3b6c8 100644
--- a/docs/management/action-types.asciidoc
+++ b/docs/management/action-types.asciidoc
@@ -7,10 +7,18 @@ Connectors provide a central place to store connection information for services
 [cols="2"]
 |===
 
+a| <<d3security-action-type,D3 Security>>
+
+| Send a request to D3 Security.
+
 a| <<email-action-type,Email>>
 
 | Send email from your server.
 
+a| <<gen-ai-action-type,Generative AI>>
+
+| Send a request to OpenAI.
+
 a| <<resilient-action-type,{ibm-r}>>
 
 | Create an incident in {ibm-r}.
@@ -63,6 +71,10 @@ a| <<tines-action-type,Tines>>
 
 | Send events to a Tines Story.
 
+a| <<torq-action-type,Torq>>
+
+| Trigger a Torq workflow.
+
 a| <<webhook-action-type, {webhook}>>
 
 | Send a request to a web service.
@@ -75,18 +87,6 @@ a| <<xmatters-action-type,xMatters>>
 
 | Send actionable alerts to on-call xMatters resources.
 
-a| <<torq-action-type,Torq>>
-
-| Trigger a Torq workflow.
-
-a| <<gen-ai-action-type,Generative AI>>
-
-| Send a request to OpenAI.
-
-a| <<d3security-action-type,D3 Security>>
-
-| Send a request to D3 Security.
-
 |===
 
 [NOTE]
diff --git a/docs/management/connectors/action-types/torq.asciidoc b/docs/management/connectors/action-types/torq.asciidoc
index 39ef2585e62b9..7b4b9712adcb9 100644
--- a/docs/management/connectors/action-types/torq.asciidoc
+++ b/docs/management/connectors/action-types/torq.asciidoc
@@ -3,6 +3,10 @@
 ++++
 <titleabbrev>Torq</titleabbrev>
 ++++
+:frontmatter-description: Add a connector that can use Torq to trigger workflows.
+:frontmatter-tags-products: [kibana] 
+:frontmatter-tags-content-type: [how-to]
+:frontmatter-tags-user-goals: [configure]
 
 The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.
 
@@ -27,34 +31,6 @@ Torq endpoint URL:: Endpoint URL (webhook) of the Elastic Security integration y
 
 Torq authentication header secret:: Secret of the webhook authentication header.
 
-[float]
-[[preconfigured-torq-configuration]]
-=== Create preconfigured connectors
-
-If you are running {kib} on-prem, you can define connectors by
-adding `xpack.actions.preconfigured` settings to your `kibana.yml` file.
-For example:
-
-[source,yaml]
---
-xpack.actions.preconfigured:
-  my-torq:
-    name: preconfigured-torq-connector-type
-    actionTypeId: .torq
-    config:
-      webhookIntegrationUrl: https://hooks.torq.io/v1/somehook
-    secrets:
-      token: mytorqtoken
---
-
-Config defines information for the connector type.
-
-`webhookIntegrationUrl`:: An address that corresponds to **Torq endpoint URL**.
-
-Secrets defines sensitive information for the connector type.
-
-`token`:: A string that corresponds to **Torq authentication header secret**.
-
 [float]
 [[torq-action-configuration]]
 === Test connectors
diff --git a/docs/management/connectors/index.asciidoc b/docs/management/connectors/index.asciidoc
index 63de5aca67dca..a30bc26ca2511 100644
--- a/docs/management/connectors/index.asciidoc
+++ b/docs/management/connectors/index.asciidoc
@@ -1,4 +1,6 @@
+include::action-types/d3security.asciidoc[leveloffset=+1]
 include::action-types/email.asciidoc[leveloffset=+1]
+include::action-types/gen-ai.asciidoc[leveloffset=+1]
 include::action-types/resilient.asciidoc[leveloffset=+1]
 include::action-types/index.asciidoc[leveloffset=+1]
 include::action-types/jira.asciidoc[leveloffset=+1]
@@ -16,6 +18,4 @@ include::action-types/torq.asciidoc[leveloffset=+1]
 include::action-types/webhook.asciidoc[leveloffset=+1]
 include::action-types/cases-webhook.asciidoc[leveloffset=+1]
 include::action-types/xmatters.asciidoc[leveloffset=+1]
-include::action-types/gen-ai.asciidoc[leveloffset=+1]
-include::action-types/d3security.asciidoc[leveloffset=+1]
 include::pre-configured-connectors.asciidoc[leveloffset=+1]
diff --git a/docs/management/connectors/pre-configured-connectors.asciidoc b/docs/management/connectors/pre-configured-connectors.asciidoc
index 72f5f78f6e728..1fc679facf423 100644
--- a/docs/management/connectors/pre-configured-connectors.asciidoc
+++ b/docs/management/connectors/pre-configured-connectors.asciidoc
@@ -113,11 +113,13 @@ Index names must start with `kibana-alert-history-` to take advantage of the pre
 * <<preconfigured-resilient-configuration>>
 * <<preconfigured-index-configuration>>
 * <<preconfigured-jira-configuration>>
+* <<preconfigured-teams-configuration>>
 * <<preconfigured-opsgenie-configuration>>
 * <<preconfigured-pagerduty-configuration>>
 * <<preconfigured-server-log-configuration>>
 * <<preconfigured-slack-configuration>>
 * <<preconfigured-swimlane-configuration>>
+* <<preconfigured-torq-configuration>>
 * <<preconfigured-webhook-configuration>>
 * <<preconfigured-cases-webhook-configuration>>
 * <<preconfigured-xmatters-configuration>>
@@ -528,6 +530,26 @@ xpack.actions.preconfigured:
 <3> Field mappings for properties such as the alert identifer, severity, and rule name.
 <4> The API authentication token for HTTP basic authentication. NOTE: This value should be stored in the <<creating-keystore, {kib} keystore>>.
 
+[float]
+[[preconfigured-torq-configuration]]
+==== Torq connectors
+
+The following example creates a <<torq-action-type,Torq connector>>:
+
+[source,yaml]
+--
+xpack.actions.preconfigured:
+  my-torq:
+    name: preconfigured-torq-connector-type
+    actionTypeId: .torq
+    config:
+      webhookIntegrationUrl: https://hooks.torq.io/v1/somehook <1>
+    secrets:
+      token: mytorqtoken <2>
+--
+<1> The endpoint URL of the Elastic Security integration in Torq.
+<2> The secret of the webhook authentication header.
+
 [float]
 [[preconfigured-webhook-configuration]]
 ==== Webhook connectors
diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc
index c9880bdade4dc..b3cd5777edd69 100644
--- a/docs/settings/alert-action-settings.asciidoc
+++ b/docs/settings/alert-action-settings.asciidoc
@@ -432,6 +432,9 @@ For an <<xmatters-action-type,xMatters connector>>, specifies whether it uses HT
 `xpack.actions.preconfigured.<connector-id>.config.viewIncidentUrl`::
 For a <<cases-webhook-action-type,{webhook-cm} connector>>, specifies a URL string with either the external service ID or external service title Mustache variable to view a case in the external system.
 
+`xpack.actions.preconfigured.<connector-id>.config.webhookIntegrationUrl`::
+For a <<torq-action-type,Torq connector>>, specifies the endpoint URL of the Elastic Security integration in Torq.
+
 `xpack.actions.preconfigured.<connector-id>.name`::
 The name of the preconfigured connector.
 
@@ -492,6 +495,7 @@ A token secret that varies by connector:
 --
 * For a <<d3security-action-type,D3 Security conector>>, specifies the D3 Security token.
 * For a <<slack-action-type,Slack connector>>, specifies the Slack bot user OAuth token.
+* For a <<torq-action-type,Torq connector>>, specifies the secret of the webhook authentication header.
 --
 
 `xpack.actions.preconfigured.<connector-id>.secrets.user`::
diff --git a/x-pack/plugins/actions/docs/openapi/bundled.json b/x-pack/plugins/actions/docs/openapi/bundled.json
index f79bb3ca9c0fe..4039ab6b5a0de 100644
--- a/x-pack/plugins/actions/docs/openapi/bundled.json
+++ b/x-pack/plugins/actions/docs/openapi/bundled.json
@@ -111,6 +111,9 @@
                   {
                     "$ref": "#/components/schemas/create_connector_request_tines"
                   },
+                  {
+                    "$ref": "#/components/schemas/create_connector_request_torq"
+                  },
                   {
                     "$ref": "#/components/schemas/create_connector_request_webhook"
                   },
@@ -389,6 +392,9 @@
                   {
                     "$ref": "#/components/schemas/create_connector_request_tines"
                   },
+                  {
+                    "$ref": "#/components/schemas/create_connector_request_torq"
+                  },
                   {
                     "$ref": "#/components/schemas/create_connector_request_webhook"
                   },
@@ -508,6 +514,9 @@
                   {
                     "$ref": "#/components/schemas/update_connector_request_teams"
                   },
+                  {
+                    "$ref": "#/components/schemas/update_connector_request_torq"
+                  },
                   {
                     "$ref": "#/components/schemas/update_connector_request_webhook"
                   },
@@ -2829,6 +2838,66 @@
           }
         }
       },
+      "config_properties_torq": {
+        "title": "Connector request properties for a Torq connector",
+        "description": "Defines properties for connectors when type is `.torq`.",
+        "type": "object",
+        "required": [
+          "webhookIntegrationUrl"
+        ],
+        "properties": {
+          "webhookIntegrationUrl": {
+            "description": "The endpoint URL of the Elastic Security integration in Torq.",
+            "type": "string"
+          }
+        }
+      },
+      "secrets_properties_torq": {
+        "title": "Connector secrets properties for a Torq connector",
+        "description": "Defines secrets for connectors when type is `.torq`.",
+        "type": "object",
+        "required": [
+          "token"
+        ],
+        "properties": {
+          "token": {
+            "description": "The secret of the webhook authentication header.",
+            "type": "string"
+          }
+        }
+      },
+      "create_connector_request_torq": {
+        "title": "Create Torq connector request",
+        "description": "The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.\n",
+        "type": "object",
+        "required": [
+          "config",
+          "connector_type_id",
+          "name",
+          "secrets"
+        ],
+        "properties": {
+          "config": {
+            "$ref": "#/components/schemas/config_properties_torq"
+          },
+          "connector_type_id": {
+            "type": "string",
+            "description": "The type of connector.",
+            "enum": [
+              ".torq"
+            ],
+            "example": ".torq"
+          },
+          "name": {
+            "type": "string",
+            "description": "The display name for the connector.",
+            "example": "my-connector"
+          },
+          "secrets": {
+            "$ref": "#/components/schemas/secrets_properties_torq"
+          }
+        }
+      },
       "config_properties_webhook": {
         "title": "Connector request properties for a Webhook connector",
         "description": "Defines properties for connectors when type is `.webhook`.",
@@ -3776,6 +3845,50 @@
           }
         }
       },
+      "connector_response_properties_torq": {
+        "title": "Connector response properties for a Torq connector",
+        "type": "object",
+        "required": [
+          "config",
+          "connector_type_id",
+          "id",
+          "is_deprecated",
+          "is_preconfigured",
+          "name"
+        ],
+        "properties": {
+          "config": {
+            "$ref": "#/components/schemas/config_properties_torq"
+          },
+          "connector_type_id": {
+            "type": "string",
+            "description": "The type of connector.",
+            "enum": [
+              ".torq"
+            ]
+          },
+          "id": {
+            "type": "string",
+            "description": "The identifier for the connector."
+          },
+          "is_deprecated": {
+            "$ref": "#/components/schemas/is_deprecated"
+          },
+          "is_missing_secrets": {
+            "$ref": "#/components/schemas/is_missing_secrets"
+          },
+          "is_preconfigured": {
+            "$ref": "#/components/schemas/is_preconfigured"
+          },
+          "is_system_action": {
+            "$ref": "#/components/schemas/is_system_action"
+          },
+          "name": {
+            "type": "string",
+            "description": "The display name for the connector."
+          }
+        }
+      },
       "connector_response_properties_webhook": {
         "title": "Connector response properties for a Webhook connector",
         "type": "object",
@@ -3919,6 +4032,9 @@
           {
             "$ref": "#/components/schemas/connector_response_properties_tines"
           },
+          {
+            "$ref": "#/components/schemas/connector_response_properties_torq"
+          },
           {
             "$ref": "#/components/schemas/connector_response_properties_webhook"
           },
@@ -4221,6 +4337,27 @@
           }
         }
       },
+      "update_connector_request_torq": {
+        "title": "Update Torq connector request",
+        "type": "object",
+        "required": [
+          "config",
+          "name",
+          "secrets"
+        ],
+        "properties": {
+          "config": {
+            "$ref": "#/components/schemas/config_properties_torq"
+          },
+          "name": {
+            "type": "string",
+            "description": "The display name for the connector."
+          },
+          "secrets": {
+            "$ref": "#/components/schemas/secrets_properties_torq"
+          }
+        }
+      },
       "update_connector_request_webhook": {
         "title": "Update Webhook connector request",
         "type": "object",
@@ -4286,6 +4423,7 @@
           ".swimlane",
           ".teams",
           ".tines",
+          ".torq",
           ".webhook",
           ".xmatters"
         ],
diff --git a/x-pack/plugins/actions/docs/openapi/bundled.yaml b/x-pack/plugins/actions/docs/openapi/bundled.yaml
index ed97fb7f31233..9d1842c4dace9 100644
--- a/x-pack/plugins/actions/docs/openapi/bundled.yaml
+++ b/x-pack/plugins/actions/docs/openapi/bundled.yaml
@@ -55,6 +55,7 @@ paths:
                 - $ref: '#/components/schemas/create_connector_request_swimlane'
                 - $ref: '#/components/schemas/create_connector_request_teams'
                 - $ref: '#/components/schemas/create_connector_request_tines'
+                - $ref: '#/components/schemas/create_connector_request_torq'
                 - $ref: '#/components/schemas/create_connector_request_webhook'
                 - $ref: '#/components/schemas/create_connector_request_xmatters'
               discriminator:
@@ -208,6 +209,7 @@ paths:
                 - $ref: '#/components/schemas/create_connector_request_swimlane'
                 - $ref: '#/components/schemas/create_connector_request_teams'
                 - $ref: '#/components/schemas/create_connector_request_tines'
+                - $ref: '#/components/schemas/create_connector_request_torq'
                 - $ref: '#/components/schemas/create_connector_request_webhook'
                 - $ref: '#/components/schemas/create_connector_request_xmatters'
               discriminator:
@@ -264,6 +266,7 @@ paths:
                 - $ref: '#/components/schemas/update_connector_request_slack_webhook'
                 - $ref: '#/components/schemas/update_connector_request_swimlane'
                 - $ref: '#/components/schemas/update_connector_request_teams'
+                - $ref: '#/components/schemas/update_connector_request_torq'
                 - $ref: '#/components/schemas/update_connector_request_webhook'
                 - $ref: '#/components/schemas/update_connector_request_xmatters'
             examples:
@@ -1916,6 +1919,51 @@ components:
           example: my-connector
         secrets:
           $ref: '#/components/schemas/secrets_properties_tines'
+    config_properties_torq:
+      title: Connector request properties for a Torq connector
+      description: Defines properties for connectors when type is `.torq`.
+      type: object
+      required:
+        - webhookIntegrationUrl
+      properties:
+        webhookIntegrationUrl:
+          description: The endpoint URL of the Elastic Security integration in Torq.
+          type: string
+    secrets_properties_torq:
+      title: Connector secrets properties for a Torq connector
+      description: Defines secrets for connectors when type is `.torq`.
+      type: object
+      required:
+        - token
+      properties:
+        token:
+          description: The secret of the webhook authentication header.
+          type: string
+    create_connector_request_torq:
+      title: Create Torq connector request
+      description: |
+        The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.
+      type: object
+      required:
+        - config
+        - connector_type_id
+        - name
+        - secrets
+      properties:
+        config:
+          $ref: '#/components/schemas/config_properties_torq'
+        connector_type_id:
+          type: string
+          description: The type of connector.
+          enum:
+            - .torq
+          example: .torq
+        name:
+          type: string
+          description: The display name for the connector.
+          example: my-connector
+        secrets:
+          $ref: '#/components/schemas/secrets_properties_torq'
     config_properties_webhook:
       title: Connector request properties for a Webhook connector
       description: Defines properties for connectors when type is `.webhook`.
@@ -2626,6 +2674,38 @@ components:
         name:
           type: string
           description: The display name for the connector.
+    connector_response_properties_torq:
+      title: Connector response properties for a Torq connector
+      type: object
+      required:
+        - config
+        - connector_type_id
+        - id
+        - is_deprecated
+        - is_preconfigured
+        - name
+      properties:
+        config:
+          $ref: '#/components/schemas/config_properties_torq'
+        connector_type_id:
+          type: string
+          description: The type of connector.
+          enum:
+            - .torq
+        id:
+          type: string
+          description: The identifier for the connector.
+        is_deprecated:
+          $ref: '#/components/schemas/is_deprecated'
+        is_missing_secrets:
+          $ref: '#/components/schemas/is_missing_secrets'
+        is_preconfigured:
+          $ref: '#/components/schemas/is_preconfigured'
+        is_system_action:
+          $ref: '#/components/schemas/is_system_action'
+        name:
+          type: string
+          description: The display name for the connector.
     connector_response_properties_webhook:
       title: Connector response properties for a Webhook connector
       type: object
@@ -2711,6 +2791,7 @@ components:
         - $ref: '#/components/schemas/connector_response_properties_swimlane'
         - $ref: '#/components/schemas/connector_response_properties_teams'
         - $ref: '#/components/schemas/connector_response_properties_tines'
+        - $ref: '#/components/schemas/connector_response_properties_torq'
         - $ref: '#/components/schemas/connector_response_properties_webhook'
         - $ref: '#/components/schemas/connector_response_properties_xmatters'
       discriminator:
@@ -2922,6 +3003,21 @@ components:
           description: The display name for the connector.
         secrets:
           $ref: '#/components/schemas/secrets_properties_teams'
+    update_connector_request_torq:
+      title: Update Torq connector request
+      type: object
+      required:
+        - config
+        - name
+        - secrets
+      properties:
+        config:
+          $ref: '#/components/schemas/config_properties_torq'
+        name:
+          type: string
+          description: The display name for the connector.
+        secrets:
+          $ref: '#/components/schemas/secrets_properties_torq'
     update_connector_request_webhook:
       title: Update Webhook connector request
       type: object
@@ -2975,6 +3071,7 @@ components:
         - .swimlane
         - .teams
         - .tines
+        - .torq
         - .webhook
         - .xmatters
       example: .server-log
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml
new file mode 100644
index 0000000000000..06808a37a75fc
--- /dev/null
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml
@@ -0,0 +1,9 @@
+title: Connector request properties for a Torq connector
+description: Defines properties for connectors when type is `.torq`.
+type: object
+required:
+  - webhookIntegrationUrl
+properties:
+  webhookIntegrationUrl:
+    description: The endpoint URL of the Elastic Security integration in Torq.
+    type: string
\ No newline at end of file
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml
index 334fe3fa5cdb3..edef270fd75ae 100644
--- a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml
@@ -18,6 +18,7 @@ oneOf:
   - $ref: 'connector_response_properties_swimlane.yaml'
   - $ref: 'connector_response_properties_teams.yaml'
   - $ref: 'connector_response_properties_tines.yaml'
+  - $ref: 'connector_response_properties_torq.yaml'
   - $ref: 'connector_response_properties_webhook.yaml'
   - $ref: 'connector_response_properties_xmatters.yaml'
 discriminator:
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml
new file mode 100644
index 0000000000000..135d5e9db6cb4
--- /dev/null
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml
@@ -0,0 +1,31 @@
+title: Connector response properties for a Torq connector
+type: object
+required:
+  - config
+  - connector_type_id
+  - id
+  - is_deprecated
+  - is_preconfigured
+  - name
+properties:
+  config:
+    $ref: 'config_properties_torq.yaml'
+  connector_type_id:
+    type: string
+    description: The type of connector.
+    enum:
+      - .torq
+  id:
+    type: string
+    description: The identifier for the connector.
+  is_deprecated:
+   $ref: 'is_deprecated.yaml'
+  is_missing_secrets:
+    $ref: 'is_missing_secrets.yaml'
+  is_preconfigured:
+    $ref: 'is_preconfigured.yaml'
+  is_system_action:
+    $ref: 'is_system_action.yaml'
+  name:
+    type: string
+    description: The display name for the connector.
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml
index 2bbc9f5dabac4..687648acd7141 100644
--- a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml
@@ -20,6 +20,7 @@ enum:
   - .swimlane
   - .teams
   - .tines
+  - .torq
   - .webhook
   - .xmatters
 example: .server-log
\ No newline at end of file
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml
new file mode 100644
index 0000000000000..934f9c9c1b395
--- /dev/null
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml
@@ -0,0 +1,24 @@
+title: Create Torq connector request
+description: >
+  The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.
+type: object
+required:
+  - config
+  - connector_type_id
+  - name
+  - secrets
+properties:
+  config:
+    $ref: 'config_properties_torq.yaml'
+  connector_type_id:
+    type: string
+    description: The type of connector.
+    enum:
+      - .torq
+    example: .torq
+  name:
+    type: string
+    description: The display name for the connector.
+    example: my-connector
+  secrets:
+    $ref: 'secrets_properties_torq.yaml' 
\ No newline at end of file
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml
new file mode 100644
index 0000000000000..ab79a0f672b42
--- /dev/null
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml
@@ -0,0 +1,9 @@
+title: Connector secrets properties for a Torq connector
+description: Defines secrets for connectors when type is `.torq`.
+type: object
+required:
+  - token
+properties:
+  token:
+    description:  The secret of the webhook authentication header.
+    type: string
\ No newline at end of file
diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml
new file mode 100644
index 0000000000000..f82de22f3e27b
--- /dev/null
+++ b/x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml
@@ -0,0 +1,14 @@
+title: Update Torq connector request
+type: object
+required:
+  - config
+  - name
+  - secrets
+properties:
+  config:
+    $ref: 'config_properties_torq.yaml'  
+  name:
+    type: string
+    description: The display name for the connector.
+  secrets:
+    $ref: 'secrets_properties_torq.yaml' 
\ No newline at end of file
diff --git a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml
index f32def50706b2..62f2e1821eb8f 100644
--- a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml
+++ b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml
@@ -34,6 +34,7 @@ post:
             - $ref: '../components/schemas/create_connector_request_swimlane.yaml'
             - $ref: '../components/schemas/create_connector_request_teams.yaml'
             - $ref: '../components/schemas/create_connector_request_tines.yaml'
+            - $ref: '../components/schemas/create_connector_request_torq.yaml'
             - $ref: '../components/schemas/create_connector_request_webhook.yaml'
             - $ref: '../components/schemas/create_connector_request_xmatters.yaml'
           discriminator:
diff --git a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml
index c6a2447c251a2..ad93b5076639a 100644
--- a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml
+++ b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml
@@ -118,6 +118,7 @@ post:
             - $ref: '../components/schemas/create_connector_request_swimlane.yaml'
             - $ref: '../components/schemas/create_connector_request_teams.yaml'
             - $ref: '../components/schemas/create_connector_request_tines.yaml'
+            - $ref: '../components/schemas/create_connector_request_torq.yaml'
             - $ref: '../components/schemas/create_connector_request_webhook.yaml'
             - $ref: '../components/schemas/create_connector_request_xmatters.yaml'
           discriminator:
@@ -176,6 +177,7 @@ put:
             - $ref: '../components/schemas/update_connector_request_swimlane.yaml' 
             - $ref: '../components/schemas/update_connector_request_teams.yaml'
 #            - $ref: '../components/schemas/update_connector_request_tines.yaml'
+            - $ref: '../components/schemas/update_connector_request_torq.yaml'
             - $ref: '../components/schemas/update_connector_request_webhook.yaml'
             - $ref: '../components/schemas/update_connector_request_xmatters.yaml'
         examples:

From 2f1b6ac89640e494ec592eee92a0471674ff5526 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= <aurelien.foucret@elastic.co>
Date: Mon, 25 Sep 2023 18:02:33 +0200
Subject: [PATCH 13/27] [Guided onboarding] Updating search solution items.
 (#166953)

---
 .../__snapshots__/guide_cards.test.tsx.snap   | 76 +++++++++----------
 .../landing_page/guide_cards.constants.tsx    | 42 +++++-----
 .../translations/translations/fr-FR.json      |  2 +-
 .../translations/translations/ja-JP.json      |  2 +-
 .../translations/translations/zh-CN.json      |  2 +-
 5 files changed, 62 insertions(+), 62 deletions(-)

diff --git a/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap b/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap
index ba2969ab4a0b3..f971ca2c40bbb 100644
--- a/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap
+++ b/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap
@@ -18,14 +18,20 @@ exports[`guide cards snapshots should render all cards 1`] = `
           activeFilter="all"
           card={
             Object {
-              "icon": "vector",
-              "navigateTo": Object {
-                "appId": "enterpriseSearchVectorSearch",
-              },
+              "guideId": "databaseSearch",
+              "icon": "database",
               "order": 1,
               "solution": "search",
-              "telemetryId": "onboarding--search--vector",
-              "title": "Set up vector search",
+              "telemetryId": "onboarding--search--database",
+              "title": <FormattedMessage
+                defaultMessage="Search across databases {lineBreak} and business systems"
+                id="guidedOnboardingPackage.gettingStarted.cards.databaseSearch.title"
+                values={
+                  Object {
+                    "lineBreak": <br />,
+                  }
+                }
+              />,
             }
           }
           guidesState={Array []}
@@ -44,14 +50,14 @@ exports[`guide cards snapshots should render all cards 1`] = `
           activeFilter="all"
           card={
             Object {
-              "icon": "magnifyWithPlus",
+              "icon": "vector",
               "navigateTo": Object {
-                "appId": "enterpriseSearchAISearch",
+                "appId": "enterpriseSearchVectorSearch",
               },
               "order": 4,
               "solution": "search",
-              "telemetryId": "onboarding--search--semantic",
-              "title": "Build a semantic search experience",
+              "telemetryId": "onboarding--search--vector",
+              "title": "Set up vector search",
             }
           }
           guidesState={Array []}
@@ -70,20 +76,14 @@ exports[`guide cards snapshots should render all cards 1`] = `
           activeFilter="all"
           card={
             Object {
-              "guideId": "appSearch",
-              "icon": "wrench",
+              "icon": "magnifyWithPlus",
+              "navigateTo": Object {
+                "appId": "enterpriseSearchAISearch",
+              },
               "order": 7,
               "solution": "search",
-              "telemetryId": "onboarding--search--application",
-              "title": <FormattedMessage
-                defaultMessage="Build an application on {lineBreak} top of Elasticsearch"
-                id="guidedOnboardingPackage.gettingStarted.cards.appSearch.title"
-                values={
-                  Object {
-                    "lineBreak": <br />,
-                  }
-                }
-              />,
+              "telemetryId": "onboarding--search--ai",
+              "title": "Build an AI-powered search experience",
             }
           }
           guidesState={Array []}
@@ -102,12 +102,20 @@ exports[`guide cards snapshots should render all cards 1`] = `
           activeFilter="all"
           card={
             Object {
-              "guideId": "websiteSearch",
-              "icon": "search",
+              "guideId": "appSearch",
+              "icon": "wrench",
               "order": 10,
               "solution": "search",
-              "telemetryId": "onboarding--search--website",
-              "title": "Add search to my website",
+              "telemetryId": "onboarding--search--application",
+              "title": <FormattedMessage
+                defaultMessage="Build an application on {lineBreak} top of Elasticsearch"
+                id="guidedOnboardingPackage.gettingStarted.cards.appSearch.title"
+                values={
+                  Object {
+                    "lineBreak": <br />,
+                  }
+                }
+              />,
             }
           }
           guidesState={Array []}
@@ -126,20 +134,12 @@ exports[`guide cards snapshots should render all cards 1`] = `
           activeFilter="all"
           card={
             Object {
-              "guideId": "databaseSearch",
-              "icon": "database",
+              "guideId": "websiteSearch",
+              "icon": "search",
               "order": 13,
               "solution": "search",
-              "telemetryId": "onboarding--search--database",
-              "title": <FormattedMessage
-                defaultMessage="Search across databases {lineBreak} and business systems"
-                id="guidedOnboardingPackage.gettingStarted.cards.databaseSearch.title"
-                values={
-                  Object {
-                    "lineBreak": <br />,
-                  }
-                }
-              />,
+              "telemetryId": "onboarding--search--website",
+              "title": "Add search to my website",
             }
           }
           guidesState={Array []}
diff --git a/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx b/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx
index c624f31d2848a..f570f5ea3b87b 100644
--- a/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx
+++ b/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx
@@ -31,6 +31,22 @@ export interface GuideCardConstants {
 }
 
 export const guideCards: GuideCardConstants[] = [
+  {
+    solution: 'search',
+    icon: 'database',
+    title: (
+      <FormattedMessage
+        id="guidedOnboardingPackage.gettingStarted.cards.databaseSearch.title"
+        defaultMessage="Search across databases {lineBreak} and business systems"
+        values={{
+          lineBreak: <br />,
+        }}
+      />
+    ),
+    guideId: 'databaseSearch',
+    telemetryId: 'onboarding--search--database',
+    order: 1,
+  },
   {
     solution: 'search',
     icon: 'vector',
@@ -41,19 +57,19 @@ export const guideCards: GuideCardConstants[] = [
       appId: 'enterpriseSearchVectorSearch',
     },
     telemetryId: 'onboarding--search--vector',
-    order: 1,
+    order: 4,
   },
   {
     solution: 'search',
     icon: 'magnifyWithPlus',
     title: i18n.translate('guidedOnboardingPackage.gettingStarted.cards.aiSearch.title', {
-      defaultMessage: 'Build a semantic search experience',
+      defaultMessage: 'Build an AI-powered search experience',
     }),
     navigateTo: {
       appId: 'enterpriseSearchAISearch',
     },
-    telemetryId: 'onboarding--search--semantic',
-    order: 4,
+    telemetryId: 'onboarding--search--ai',
+    order: 7,
   },
   {
     solution: 'search',
@@ -69,7 +85,7 @@ export const guideCards: GuideCardConstants[] = [
     ),
     guideId: 'appSearch',
     telemetryId: 'onboarding--search--application',
-    order: 7,
+    order: 10,
   },
   {
     solution: 'search',
@@ -79,22 +95,6 @@ export const guideCards: GuideCardConstants[] = [
     }),
     guideId: 'websiteSearch',
     telemetryId: 'onboarding--search--website',
-    order: 10,
-  },
-  {
-    solution: 'search',
-    icon: 'database',
-    title: (
-      <FormattedMessage
-        id="guidedOnboardingPackage.gettingStarted.cards.databaseSearch.title"
-        defaultMessage="Search across databases {lineBreak} and business systems"
-        values={{
-          lineBreak: <br />,
-        }}
-      />
-    ),
-    guideId: 'databaseSearch',
-    telemetryId: 'onboarding--search--database',
     order: 13,
   },
   {
diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json
index 150f72792a861..0e167e1ce60d3 100644
--- a/x-pack/plugins/translations/translations/fr-FR.json
+++ b/x-pack/plugins/translations/translations/fr-FR.json
@@ -3212,7 +3212,7 @@
     "guidedOnboardingPackage.gettingStarted.cards.progressLabel": "{numberCompleteSteps} étape(s) terminée(s) sur {numberSteps}",
     "guidedOnboardingPackage.gettingStarted.cards.siemSecurity.title": "Détecter les menaces dans {lineBreak} mes données avec SIEM",
     "guidedOnboardingPackage.gettingStarted.cards.completeLabel": "Guide terminé",
-    "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "Créer une expérience de recherche sémantique",
+    "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "Créer une expérience de recherche basée sur l'IA",
     "guidedOnboardingPackage.gettingStarted.cards.hostsObservability.title": "Monitorer mes indicateurs d'hôte",
     "guidedOnboardingPackage.gettingStarted.cards.kubernetesObservability.title": "Monitorer les clusters Kubernetes",
     "guidedOnboardingPackage.gettingStarted.cards.logsObservability.title": "Collecter et analyser mes logs",
diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json
index 41096d8c2e990..f0e47690da767 100644
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@@ -3227,7 +3227,7 @@
     "guidedOnboardingPackage.gettingStarted.cards.progressLabel": "{numberSteps}ステップ中{numberCompleteSteps}ステップ完了",
     "guidedOnboardingPackage.gettingStarted.cards.siemSecurity.title": "SIEMで{lineBreak}データの脅威を検出",
     "guidedOnboardingPackage.gettingStarted.cards.completeLabel": "ガイド完了",
-    "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "セマンティック検索エクスペリエンスを構築",
+    "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "AI を活用した検索エクスペリエンスを構築する",
     "guidedOnboardingPackage.gettingStarted.cards.hostsObservability.title": "ホストメトリックを監視",
     "guidedOnboardingPackage.gettingStarted.cards.kubernetesObservability.title": "Kubernetesクラスターの監視",
     "guidedOnboardingPackage.gettingStarted.cards.logsObservability.title": "ログを収集して分析",
diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json
index b17bf98b8ad3a..17beacb3184e2 100644
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@@ -3226,7 +3226,7 @@
     "guidedOnboardingPackage.gettingStarted.cards.progressLabel": "已完成 {numberCompleteSteps} 个（共 {numberSteps} 个）步骤",
     "guidedOnboardingPackage.gettingStarted.cards.siemSecurity.title": "通过 SIEM{lineBreak}在我的数据中检测威胁",
     "guidedOnboardingPackage.gettingStarted.cards.completeLabel": "指南完成",
-    "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "构建语义搜索体验",
+    "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "打造人工智能驱动的搜索体验",
     "guidedOnboardingPackage.gettingStarted.cards.hostsObservability.title": "监测我的主机指标",
     "guidedOnboardingPackage.gettingStarted.cards.kubernetesObservability.title": "监测 Kubernetes 集群",
     "guidedOnboardingPackage.gettingStarted.cards.logsObservability.title": "收集并分析我的日志",

From 2bce7bbcbeba1d093ee4825d0de8d6d3b78c7bc4 Mon Sep 17 00:00:00 2001
From: Dima Arnautov <dmitrii.arnautov@elastic.co>
Date: Mon, 25 Sep 2023 18:15:35 +0200
Subject: [PATCH 14/27] [ML] Shared service for elastic curated models
 (#167000)

## Summary

Adds a shared service for elastic curated models. The first use case is
to provide a default/recommended ELSER version based on the hardware of
the current cluster.

#### Why?
In 8.11 we'll provide a platform-specific version of the ELSER v2
alongside the portable one. At the moment several solutions refer to
ELSER for download/inference purposes with a `.elser_model_1` constant.
Starting 8.11 the model ID will vary, so using the `ElastcModels`
service allows retrieving the recommended version of ELSER for the
current cluster without any changes by solution teams in future
releases. It is still possible to request an older version of the model
if necessary.

#### Implementation
- Adds a new Kibana API endpoint `/trained_models/model_downloads` that
provides a list of model definitions, with the `default` and
`recommended` flags.
- Adds a new Kibana API endpoint `/trained_models/elser_config` that
provides an ELSER configuration based on the cluster architecture.
- `getELSER` method is exposed from the plugin `setup` server-side as
part of our shared services and plugin `start` client-side.

### Checklist

- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
---
 .../packages/ml/trained_models_utils/index.ts |  6 ++
 .../src/constants/trained_models.ts           | 53 ++++++++++-
 .../services/elastic_models_service.ts        | 23 +++++
 .../services/get_shared_ml_services.ts        | 25 ++++++
 .../application/services/http_service.ts      | 64 +------------
 .../services/ml_api_service/trained_models.ts | 26 +++++-
 x-pack/plugins/ml/public/mocks.ts             | 18 +++-
 x-pack/plugins/ml/public/plugin.ts            | 10 +++
 .../apidoc_scripts/apidoc_config/apidoc.json  |  2 +
 .../model_management/model_provider.test.ts   | 88 ++++++++++++++++++
 .../model_management/models_provider.ts       | 89 ++++++++++++++++++-
 x-pack/plugins/ml/server/plugin.ts            |  2 +-
 .../server/routes/schemas/inference_schema.ts |  4 +
 .../ml/server/routes/trained_models.ts        | 86 ++++++++++++++++--
 .../providers/trained_models.ts               | 16 +++-
 .../server/shared_services/shared_services.ts |  2 +-
 x-pack/plugins/ml/tsconfig.json               |  1 +
 17 files changed, 438 insertions(+), 77 deletions(-)
 create mode 100644 x-pack/plugins/ml/public/application/services/elastic_models_service.ts
 create mode 100644 x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts
 create mode 100644 x-pack/plugins/ml/server/models/model_management/model_provider.test.ts

diff --git a/x-pack/packages/ml/trained_models_utils/index.ts b/x-pack/packages/ml/trained_models_utils/index.ts
index ba67911f3f8ab..22b808bdc7b5e 100644
--- a/x-pack/packages/ml/trained_models_utils/index.ts
+++ b/x-pack/packages/ml/trained_models_utils/index.ts
@@ -14,4 +14,10 @@ export {
   type DeploymentState,
   type SupportedPytorchTasksType,
   type TrainedModelType,
+  ELASTIC_MODEL_DEFINITIONS,
+  type ElasticModelId,
+  type ModelDefinition,
+  type ModelDefinitionResponse,
+  type ElserVersion,
+  type GetElserOptions,
 } from './src/constants/trained_models';
diff --git a/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts b/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts
index 86b88f51a66c4..4580330119ddd 100644
--- a/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts
+++ b/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts
@@ -46,8 +46,9 @@ export const BUILT_IN_MODEL_TAG = 'prepackaged';
 
 export const ELASTIC_MODEL_TAG = 'elastic';
 
-export const ELASTIC_MODEL_DEFINITIONS = {
+export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object.freeze({
   '.elser_model_1': {
+    version: 1,
     config: {
       input: {
         field_names: ['text_field'],
@@ -57,7 +58,49 @@ export const ELASTIC_MODEL_DEFINITIONS = {
       defaultMessage: 'Elastic Learned Sparse EncodeR v1 (Tech Preview)',
     }),
   },
-} as const;
+  '.elser_model_2_SNAPSHOT': {
+    version: 2,
+    default: true,
+    config: {
+      input: {
+        field_names: ['text_field'],
+      },
+    },
+    description: i18n.translate('xpack.ml.trainedModels.modelsList.elserV2Description', {
+      defaultMessage: 'Elastic Learned Sparse EncodeR v2 (Tech Preview)',
+    }),
+  },
+  '.elser_model_2_linux-x86_64_SNAPSHOT': {
+    version: 2,
+    os: 'Linux',
+    arch: 'amd64',
+    config: {
+      input: {
+        field_names: ['text_field'],
+      },
+    },
+    description: i18n.translate('xpack.ml.trainedModels.modelsList.elserV2x86Description', {
+      defaultMessage:
+        'Elastic Learned Sparse EncodeR v2, optimized for linux-x86_64 (Tech Preview)',
+    }),
+  },
+} as const);
+
+export interface ModelDefinition {
+  version: number;
+  config: object;
+  description: string;
+  os?: string;
+  arch?: string;
+  default?: boolean;
+  recommended?: boolean;
+}
+
+export type ModelDefinitionResponse = ModelDefinition & {
+  name: string;
+};
+
+export type ElasticModelId = keyof typeof ELASTIC_MODEL_DEFINITIONS;
 
 export const MODEL_STATE = {
   ...DEPLOYMENT_STATE,
@@ -66,3 +109,9 @@ export const MODEL_STATE = {
 } as const;
 
 export type ModelState = typeof MODEL_STATE[keyof typeof MODEL_STATE] | null;
+
+export type ElserVersion = 1 | 2;
+
+export interface GetElserOptions {
+  version?: ElserVersion;
+}
diff --git a/x-pack/plugins/ml/public/application/services/elastic_models_service.ts b/x-pack/plugins/ml/public/application/services/elastic_models_service.ts
new file mode 100644
index 0000000000000..2591fb6d82e7d
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/services/elastic_models_service.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ModelDefinitionResponse, GetElserOptions } from '@kbn/ml-trained-models-utils';
+import { type TrainedModelsApiService } from './ml_api_service/trained_models';
+
+export class ElasticModels {
+  constructor(private readonly trainedModels: TrainedModelsApiService) {}
+
+  /**
+   * Provides an ELSER model name and configuration for download based on the current cluster architecture.
+   * The current default version is 2. If running on Cloud it returns the Linux x86_64 optimized version.
+   * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64,
+   * a portable version of the model is returned.
+   */
+  public async getELSER(options?: GetElserOptions): Promise<ModelDefinitionResponse> {
+    return await this.trainedModels.getElserConfig(options);
+  }
+}
diff --git a/x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts b/x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts
new file mode 100644
index 0000000000000..23ac82737044f
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type HttpStart } from '@kbn/core-http-browser';
+import { ElasticModels } from './elastic_models_service';
+import { HttpService } from './http_service';
+import { mlApiServicesProvider } from './ml_api_service';
+
+export type MlSharedServices = ReturnType<typeof getMlSharedServices>;
+
+/**
+ * Provides ML services exposed from the plugin start.
+ */
+export function getMlSharedServices(httpStart: HttpStart) {
+  const httpService = new HttpService(httpStart);
+  const mlApiServices = mlApiServicesProvider(httpService);
+
+  return {
+    elasticModels: new ElasticModels(mlApiServices.trainedModels),
+  };
+}
diff --git a/x-pack/plugins/ml/public/application/services/http_service.ts b/x-pack/plugins/ml/public/application/services/http_service.ts
index c43b6126f147f..cd283c5d58652 100644
--- a/x-pack/plugins/ml/public/application/services/http_service.ts
+++ b/x-pack/plugins/ml/public/application/services/http_service.ts
@@ -6,7 +6,7 @@
  */
 
 import { Observable } from 'rxjs';
-import { HttpFetchOptionsWithPath, HttpFetchOptions, HttpStart } from '@kbn/core/public';
+import type { HttpFetchOptionsWithPath, HttpFetchOptions, HttpStart } from '@kbn/core/public';
 import { getHttp } from '../util/dependency_cache';
 
 function getResultHeaders(headers: HeadersInit) {
@@ -59,68 +59,6 @@ export async function http<T>(options: HttpFetchOptionsWithPath): Promise<T> {
   return getHttp().fetch<T>(path, fetchOptions);
 }
 
-/**
- * Function for making HTTP requests to Kibana's backend which returns an Observable
- * with request cancellation support.
- *
- * @deprecated use {@link HttpService} instead
- */
-export function http$<T>(options: HttpFetchOptionsWithPath): Observable<T> {
-  const { path, fetchOptions } = getFetchOptions(options);
-  return fromHttpHandler<T>(path, fetchOptions);
-}
-
-/**
- * Creates an Observable from Kibana's HttpHandler.
- */
-function fromHttpHandler<T>(input: string, init?: RequestInit): Observable<T> {
-  return new Observable<T>((subscriber) => {
-    const controller = new AbortController();
-    const signal = controller.signal;
-
-    let abortable = true;
-    let unsubscribed = false;
-
-    if (init?.signal) {
-      if (init.signal.aborted) {
-        controller.abort();
-      } else {
-        init.signal.addEventListener('abort', () => {
-          if (!signal.aborted) {
-            controller.abort();
-          }
-        });
-      }
-    }
-
-    const perSubscriberInit: RequestInit = {
-      ...(init ? init : {}),
-      signal,
-    };
-
-    getHttp()
-      .fetch<T>(input, perSubscriberInit)
-      .then((response) => {
-        abortable = false;
-        subscriber.next(response);
-        subscriber.complete();
-      })
-      .catch((err) => {
-        abortable = false;
-        if (!unsubscribed) {
-          subscriber.error(err);
-        }
-      });
-
-    return () => {
-      unsubscribed = true;
-      if (abortable) {
-        controller.abort();
-      }
-    };
-  });
-}
-
 /**
  * ML Http Service
  */
diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts
index e6b9c1a5badc3..c10867af0011b 100644
--- a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts
+++ b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts
@@ -6,11 +6,12 @@
  */
 
 import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { IngestPipeline } from '@elastic/elasticsearch/lib/api/types';
+import type { IngestPipeline } from '@elastic/elasticsearch/lib/api/types';
 
 import { useMemo } from 'react';
 import type { HttpFetchQuery } from '@kbn/core/public';
 import type { ErrorType } from '@kbn/ml-error-utils';
+import type { GetElserOptions, ModelDefinitionResponse } from '@kbn/ml-trained-models-utils';
 import { ML_INTERNAL_BASE_PATH } from '../../../../common/constants/app';
 import type { MlSavedObjectType } from '../../../../common/types/saved_objects';
 import { HttpService } from '../http_service';
@@ -57,6 +58,29 @@ export interface InferenceStatsResponse {
  */
 export function trainedModelsApiProvider(httpService: HttpService) {
   return {
+    /**
+     * Fetches the trained models list available for download.
+     */
+    getTrainedModelDownloads() {
+      return httpService.http<ModelDefinitionResponse[]>({
+        path: `${ML_INTERNAL_BASE_PATH}/trained_models/model_downloads`,
+        method: 'GET',
+        version: '1',
+      });
+    },
+
+    /**
+     * Gets ELSER config for download based on the cluster OS and CPU architecture.
+     */
+    getElserConfig(options?: GetElserOptions) {
+      return httpService.http<ModelDefinitionResponse>({
+        path: `${ML_INTERNAL_BASE_PATH}/trained_models/elser_config`,
+        method: 'GET',
+        ...(options ? { query: options as HttpFetchQuery } : {}),
+        version: '1',
+      });
+    },
+
     /**
      * Fetches configuration information for a trained inference model.
      * @param modelId - Model ID, collection of Model IDs or Model ID pattern.
diff --git a/x-pack/plugins/ml/public/mocks.ts b/x-pack/plugins/ml/public/mocks.ts
index 13f8952dbad21..77cdefdb2f1c9 100644
--- a/x-pack/plugins/ml/public/mocks.ts
+++ b/x-pack/plugins/ml/public/mocks.ts
@@ -6,7 +6,8 @@
  */
 
 import { sharePluginMock } from '@kbn/share-plugin/public/mocks';
-import { MlPluginSetup, MlPluginStart } from './plugin';
+import { type ElasticModels } from './application/services/elastic_models_service';
+import type { MlPluginSetup, MlPluginStart } from './plugin';
 
 const createSetupContract = (): jest.Mocked<MlPluginSetup> => {
   return {
@@ -17,6 +18,21 @@ const createSetupContract = (): jest.Mocked<MlPluginSetup> => {
 const createStartContract = (): jest.Mocked<MlPluginStart> => {
   return {
     locator: sharePluginMock.createLocator(),
+    elasticModels: {
+      getELSER: jest.fn(() =>
+        Promise.resolve({
+          version: 2,
+          default: true,
+          config: {
+            input: {
+              field_names: ['text_field'],
+            },
+          },
+          description: 'Elastic Learned Sparse EncodeR v2 (Tech Preview)',
+          name: '.elser_model_2',
+        })
+      ),
+    } as unknown as jest.Mocked<ElasticModels>,
   };
 };
 
diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts
index 3a32f8b25ae89..4eae00a53d401 100644
--- a/x-pack/plugins/ml/public/plugin.ts
+++ b/x-pack/plugins/ml/public/plugin.ts
@@ -48,6 +48,10 @@ import type { ChartsPluginStart } from '@kbn/charts-plugin/public';
 import type { CasesUiSetup, CasesUiStart } from '@kbn/cases-plugin/public';
 import type { SavedSearchPublicPluginStart } from '@kbn/saved-search-plugin/public';
 import type { PresentationUtilPluginStart } from '@kbn/presentation-util-plugin/public';
+import {
+  getMlSharedServices,
+  MlSharedServices,
+} from './application/services/get_shared_ml_services';
 import { registerManagementSection } from './application/management';
 import { MlLocatorDefinition, type MlLocator } from './locator';
 import { setDependencyCache } from './application/util/dependency_cache';
@@ -103,6 +107,9 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
   private appUpdater$ = new BehaviorSubject<AppUpdater>(() => ({}));
 
   private locator: undefined | MlLocator;
+
+  private sharedMlServices: MlSharedServices | undefined;
+
   private isServerless: boolean = false;
 
   constructor(private initializerContext: PluginInitializerContext) {
@@ -110,6 +117,8 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
   }
 
   setup(core: MlCoreSetup, pluginsSetup: MlSetupDependencies) {
+    this.sharedMlServices = getMlSharedServices(core.http);
+
     core.application.register({
       id: PLUGIN_ID,
       title: i18n.translate('xpack.ml.plugin.title', {
@@ -249,6 +258,7 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
 
     return {
       locator: this.locator,
+      elasticModels: this.sharedMlServices?.elasticModels,
     };
   }
 
diff --git a/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json b/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json
index 6ec23bc13c559..a6e647a60fe9f 100644
--- a/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json
+++ b/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json
@@ -180,6 +180,8 @@
     "InferTrainedModelDeployment",
     "CreateInferencePipeline",
     "GetIngestPipelines",
+    "GetTrainedModelDownloadList",
+    "GetElserConfig",
 
     "Alerting",
     "PreviewAlert",
diff --git a/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts b/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts
new file mode 100644
index 0000000000000..7e66d03033b66
--- /dev/null
+++ b/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts
@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { modelsProvider } from './models_provider';
+import { type IScopedClusterClient } from '@kbn/core/server';
+import { cloudMock } from '@kbn/cloud-plugin/server/mocks';
+
+describe('modelsProvider', () => {
+  const mockClient = {
+    asInternalUser: {
+      transport: {
+        request: jest.fn().mockResolvedValue({
+          _nodes: {
+            total: 1,
+            successful: 1,
+            failed: 0,
+          },
+          cluster_name: 'default',
+          nodes: {
+            yYmqBqjpQG2rXsmMSPb9pQ: {
+              name: 'node-0',
+              roles: ['ml'],
+              attributes: {},
+              os: {
+                name: 'Linux',
+                arch: 'amd64',
+              },
+            },
+          },
+        }),
+      },
+    },
+  } as unknown as jest.Mocked<IScopedClusterClient>;
+
+  const mockCloud = cloudMock.createSetup();
+  const modelService = modelsProvider(mockClient, mockCloud);
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('getELSER', () => {
+    test('provides a recommended definition by default', async () => {
+      const result = await modelService.getELSER();
+      expect(result.name).toEqual('.elser_model_2_linux-x86_64_SNAPSHOT');
+    });
+
+    test('provides a default version if there is no recommended', async () => {
+      mockCloud.cloudId = undefined;
+      (mockClient.asInternalUser.transport.request as jest.Mock).mockResolvedValueOnce({
+        _nodes: {
+          total: 1,
+          successful: 1,
+          failed: 0,
+        },
+        cluster_name: 'default',
+        nodes: {
+          yYmqBqjpQG2rXsmMSPb9pQ: {
+            name: 'node-0',
+            roles: ['ml'],
+            attributes: {},
+            os: {
+              name: 'Mac OS X',
+              arch: 'aarch64',
+            },
+          },
+        },
+      });
+
+      const result = await modelService.getELSER();
+      expect(result.name).toEqual('.elser_model_2_SNAPSHOT');
+    });
+
+    test('provides the requested version', async () => {
+      const result = await modelService.getELSER({ version: 1 });
+      expect(result.name).toEqual('.elser_model_1');
+    });
+
+    test('provides the requested version of a recommended architecture', async () => {
+      const result = await modelService.getELSER({ version: 2 });
+      expect(result.name).toEqual('.elser_model_2_linux-x86_64_SNAPSHOT');
+    });
+  });
+});
diff --git a/x-pack/plugins/ml/server/models/model_management/models_provider.ts b/x-pack/plugins/ml/server/models/model_management/models_provider.ts
index e7cfcbe7fd50d..f6164ad6e65ca 100644
--- a/x-pack/plugins/ml/server/models/model_management/models_provider.ts
+++ b/x-pack/plugins/ml/server/models/model_management/models_provider.ts
@@ -6,16 +6,23 @@
  */
 
 import type { IScopedClusterClient } from '@kbn/core/server';
-import {
+import type {
   IngestPipeline,
   IngestSimulateDocument,
   IngestSimulateRequest,
+  NodesInfoResponseBase,
 } from '@elastic/elasticsearch/lib/api/types';
+import {
+  ELASTIC_MODEL_DEFINITIONS,
+  type GetElserOptions,
+  type ModelDefinitionResponse,
+} from '@kbn/ml-trained-models-utils';
+import type { CloudSetup } from '@kbn/cloud-plugin/server';
 import type { PipelineDefinition } from '../../../common/types/trained_models';
 
 export type ModelService = ReturnType<typeof modelsProvider>;
 
-export function modelsProvider(client: IScopedClusterClient) {
+export function modelsProvider(client: IScopedClusterClient, cloud?: CloudSetup) {
   return {
     /**
      * Retrieves the map of model ids and aliases with associated pipelines.
@@ -128,5 +135,83 @@ export function modelsProvider(client: IScopedClusterClient) {
 
       return result;
     },
+
+    /**
+     * Returns a list of elastic curated models available for download.
+     */
+    async getModelDownloads(): Promise<ModelDefinitionResponse[]> {
+      // We assume that ML nodes in Cloud are always on linux-x86_64, even if other node types aren't.
+      const isCloud = !!cloud?.cloudId;
+
+      const nodesInfoResponse =
+        await client.asInternalUser.transport.request<NodesInfoResponseBase>({
+          method: 'GET',
+          path: `/_nodes/ml:true/os`,
+        });
+
+      let osName: string | undefined;
+      let arch: string | undefined;
+      // Indicates that all ML nodes have the same architecture
+      let sameArch = true;
+      for (const node of Object.values(nodesInfoResponse.nodes)) {
+        if (!osName) {
+          osName = node.os?.name;
+        }
+        if (!arch) {
+          arch = node.os?.arch;
+        }
+        if (node.os?.name !== osName || node.os?.arch !== arch) {
+          sameArch = false;
+          break;
+        }
+      }
+
+      const result = Object.entries(ELASTIC_MODEL_DEFINITIONS).map(([name, def]) => {
+        const recommended =
+          (isCloud && def.os === 'Linux' && def.arch === 'amd64') ||
+          (sameArch && !!def?.os && def?.os === osName && def?.arch === arch);
+        return {
+          ...def,
+          name,
+          ...(recommended ? { recommended } : {}),
+        };
+      });
+
+      return result;
+    },
+
+    /**
+     * Provides an ELSER model name and configuration for download based on the current cluster architecture.
+     * The current default version is 2. If running on Cloud it returns the Linux x86_64 optimized version.
+     * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64,
+     * a portable version of the model is returned.
+     */
+    async getELSER(options?: GetElserOptions): Promise<ModelDefinitionResponse> | never {
+      const modelDownloadConfig = await this.getModelDownloads();
+
+      let requestedModel: ModelDefinitionResponse | undefined;
+      let recommendedModel: ModelDefinitionResponse | undefined;
+      let defaultModel: ModelDefinitionResponse | undefined;
+
+      for (const model of modelDownloadConfig) {
+        if (options?.version === model.version) {
+          requestedModel = model;
+          if (model.recommended) {
+            requestedModel = model;
+            break;
+          }
+        } else if (model.recommended) {
+          recommendedModel = model;
+        } else if (model.default) {
+          defaultModel = model;
+        }
+      }
+
+      if (!requestedModel && !defaultModel && !recommendedModel) {
+        throw new Error('Requested model not found');
+      }
+
+      return requestedModel || recommendedModel || defaultModel!;
+    },
   };
 }
diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts
index 342350fac998a..dcd97acabcbd8 100644
--- a/x-pack/plugins/ml/server/plugin.ts
+++ b/x-pack/plugins/ml/server/plugin.ts
@@ -241,7 +241,7 @@ export class MlServerPlugin
     // Register Trained Model Management routes
     if (this.enabledFeatures.dfa || this.enabledFeatures.nlp) {
       modelManagementRoutes(routeInit);
-      trainedModelsRoutes(routeInit);
+      trainedModelsRoutes(routeInit, plugins.cloud);
     }
 
     // Register Miscellaneous routes
diff --git a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts
index 21b62c6f5ce42..1b48a49c8d82f 100644
--- a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts
+++ b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts
@@ -87,3 +87,7 @@ export const createIngestPipelineSchema = schema.object({
     })
   ),
 });
+
+export const modelDownloadsQuery = schema.object({
+  version: schema.maybe(schema.oneOf([schema.literal('1'), schema.literal('2')])),
+});
diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts
index ab5d3a87e8f46..8685652ab3189 100644
--- a/x-pack/plugins/ml/server/routes/trained_models.ts
+++ b/x-pack/plugins/ml/server/routes/trained_models.ts
@@ -9,6 +9,8 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { schema } from '@kbn/config-schema';
 import type { ErrorType } from '@kbn/ml-error-utils';
 import type { MlGetTrainedModelsRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { type ElserVersion } from '@kbn/ml-trained-models-utils';
+import type { CloudSetup } from '@kbn/cloud-plugin/server';
 import { ML_INTERNAL_BASE_PATH } from '../../common/constants/app';
 import type { MlFeatures, RouteInitialization } from '../types';
 import { wrapError } from '../client/error_wrapper';
@@ -25,6 +27,7 @@ import {
   threadingParamsSchema,
   updateDeploymentParamsSchema,
   createIngestPipelineSchema,
+  modelDownloadsQuery,
 } from './schemas/inference_schema';
 import type { TrainedModelConfigResponse } from '../../common/types/trained_models';
 import { mlLog } from '../lib/log';
@@ -49,11 +52,10 @@ export function filterForEnabledFeatureModels(
   return filteredModels;
 }
 
-export function trainedModelsRoutes({
-  router,
-  routeGuard,
-  getEnabledFeatures,
-}: RouteInitialization) {
+export function trainedModelsRoutes(
+  { router, routeGuard, getEnabledFeatures }: RouteInitialization,
+  cloud: CloudSetup
+) {
   /**
    * @apiGroup TrainedModels
    *
@@ -652,4 +654,78 @@ export function trainedModelsRoutes({
         }
       })
     );
+
+  /**
+   * @apiGroup TrainedModels
+   *
+   * @api {get} /internal/ml/trained_models/model_downloads Gets available models for download
+   * @apiName GetTrainedModelDownloadList
+   * @apiDescription Gets available models for download with default and recommended flags based on the cluster OS and CPU architecture.
+   */
+  router.versioned
+    .get({
+      path: `${ML_INTERNAL_BASE_PATH}/trained_models/model_downloads`,
+      access: 'internal',
+      options: {
+        tags: ['access:ml:canGetTrainedModels'],
+      },
+    })
+    .addVersion(
+      {
+        version: '1',
+        validate: false,
+      },
+      routeGuard.fullLicenseAPIGuard(async ({ response, client }) => {
+        try {
+          const body = await modelsProvider(client, cloud).getModelDownloads();
+
+          return response.ok({
+            body,
+          });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
+      })
+    );
+
+  /**
+   * @apiGroup TrainedModels
+   *
+   * @api {get} /internal/ml/trained_models/elser_config Gets ELSER config for download
+   * @apiName GetElserConfig
+   * @apiDescription Gets ELSER config for download based on the cluster OS and CPU architecture.
+   */
+  router.versioned
+    .get({
+      path: `${ML_INTERNAL_BASE_PATH}/trained_models/elser_config`,
+      access: 'internal',
+      options: {
+        tags: ['access:ml:canGetTrainedModels'],
+      },
+    })
+    .addVersion(
+      {
+        version: '1',
+        validate: {
+          request: {
+            query: modelDownloadsQuery,
+          },
+        },
+      },
+      routeGuard.fullLicenseAPIGuard(async ({ response, client, request }) => {
+        try {
+          const { version } = request.query;
+
+          const body = await modelsProvider(client, cloud).getELSER(
+            version ? { version: Number(version) as ElserVersion } : undefined
+          );
+
+          return response.ok({
+            body,
+          });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
+      })
+    );
 }
diff --git a/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts b/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts
index c2b3f41551afd..b884edd99c22d 100644
--- a/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts
+++ b/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts
@@ -6,13 +6,16 @@
  */
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { CloudSetup } from '@kbn/cloud-plugin/server';
 import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server';
+import type { GetElserOptions } from '@kbn/ml-trained-models-utils';
 import type {
   MlInferTrainedModelRequest,
   MlStopTrainedModelDeploymentRequest,
   UpdateTrainedModelDeploymentRequest,
   UpdateTrainedModelDeploymentResponse,
 } from '../../lib/ml_client/types';
+import { modelsProvider } from '../../models/model_management';
 import type { GetGuards } from '../shared_services';
 
 export interface TrainedModelsProvider {
@@ -47,7 +50,10 @@ export interface TrainedModelsProvider {
   };
 }
 
-export function getTrainedModelsProvider(getGuards: GetGuards): TrainedModelsProvider {
+export function getTrainedModelsProvider(
+  getGuards: GetGuards,
+  cloud: CloudSetup
+): TrainedModelsProvider {
   return {
     trainedModelsProvider(request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract) {
       const guards = getGuards(request, savedObjectsClient);
@@ -116,6 +122,14 @@ export function getTrainedModelsProvider(getGuards: GetGuards): TrainedModelsPro
               return mlClient.putTrainedModel(params);
             });
         },
+        async getELSER(params: GetElserOptions) {
+          return await guards
+            .isFullLicense()
+            .hasMlCapabilities(['canGetTrainedModels'])
+            .ok(async ({ scopedClient }) => {
+              return modelsProvider(scopedClient, cloud).getELSER(params);
+            });
+        },
       };
     },
   };
diff --git a/x-pack/plugins/ml/server/shared_services/shared_services.ts b/x-pack/plugins/ml/server/shared_services/shared_services.ts
index 752820f57cf2a..235a30f541984 100644
--- a/x-pack/plugins/ml/server/shared_services/shared_services.ts
+++ b/x-pack/plugins/ml/server/shared_services/shared_services.ts
@@ -186,7 +186,7 @@ export function createSharedServices(
       ...getResultsServiceProvider(getGuards),
       ...getMlSystemProvider(getGuards, mlLicense, getSpaces, cloud, resolveMlCapabilities),
       ...getAlertingServiceProvider(getGuards),
-      ...getTrainedModelsProvider(getGuards),
+      ...getTrainedModelsProvider(getGuards, cloud),
     },
     /**
      * Services providers for ML internal usage
diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json
index 57811ff1201fe..2532a6b7824ee 100644
--- a/x-pack/plugins/ml/tsconfig.json
+++ b/x-pack/plugins/ml/tsconfig.json
@@ -104,5 +104,6 @@
     "@kbn/ml-in-memory-table",
     "@kbn/presentation-util-plugin",
     "@kbn/react-kibana-mount",
+    "@kbn/core-http-browser",
   ],
 }

From 88fdebdc8114394f0f87758187292003c6548007 Mon Sep 17 00:00:00 2001
From: Kevin Lacabane <kevin.lacabane@elastic.co>
Date: Mon, 25 Sep 2023 18:28:09 +0200
Subject: [PATCH 15/27] [apm] allow retrieval of metric indices (#167041)

### Summary

Closes https://github.com/elastic/kibana/issues/166961

`/internal/apm/services/{serviceName}/infrastructure_attributes` route
was disabled in serverless as it relied on an infra API to function.
Since the infra plugin dependency was removed in
https://github.com/elastic/kibana/pull/164094 we can reenable the route

### Testing
I used a ccs cluster connected to edge-oblt and had to update the apm
indices to also search the remote_cluster
```
xpack.apm.indices.metric: remote_cluster:metrics-apm*,remote_cluster:apm*,metrics-apm*,apm*
xpack.apm.indices.transaction: remote_cluster:traces-apm*,remote_cluster:apm*,traces-apm*,apm*
xpack.apm.indices.span: remote_cluster:traces-apm*,remote_cluster:apm*,traces-apm*,apm*
xpack.apm.indices.error: remote_cluster:logs-apm*,remote_cluster:apm*,logs-apm*,apm*
```
- start serverless kibana
- navigate to Applications -> Services, we need to select a [service
linked to a
container](https://github.com/elastic/kibana/blob/main/x-pack/plugins/apm/server/routes/infrastructure/get_host_names.ts#L23)
to fully trigger the route logic (you can pick `quoteservice` if
connected to edge-oblt data)
- navigate to Logs tab
- call to `/infrastructure_attributes` is successful
---
 .../create_infra_metrics_client.ts            |  2 +-
 .../lib/helpers/get_infra_metric_indices.ts   | 25 -------------------
 .../apm/server/routes/infrastructure/route.ts |  5 ----
 .../apm/server/routes/services/route.ts       |  7 ++----
 .../server/client/client.test.ts              | 25 +++++++++++++------
 .../server/client/client.ts                   |  8 +++---
 6 files changed, 24 insertions(+), 48 deletions(-)
 delete mode 100644 x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts

diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts
index f7ae6ea53147a..eff505b22bd4a 100644
--- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts
+++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts
@@ -6,7 +6,7 @@
  */
 
 import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
-import { APMRouteHandlerResources } from '../../../../routes/typings';
+import { APMRouteHandlerResources } from '../../../../routes/apm_routes/register_apm_server_routes';
 
 type InfraMetricsSearchParams = Omit<ESSearchRequest, 'index'> & {
   size: number;
diff --git a/x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts b/x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts
deleted file mode 100644
index 24b76edb4d887..0000000000000
--- a/x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { SavedObjectsClientContract } from '@kbn/core/server';
-import { APMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes';
-
-export async function getInfraMetricIndices({
-  infraPlugin,
-  savedObjectsClient,
-}: {
-  infraPlugin: Required<APMRouteHandlerResources['plugins']['infra']>;
-  savedObjectsClient: SavedObjectsClientContract;
-}): Promise<string> {
-  if (!infraPlugin) {
-    throw new Error('Infra Plugin needs to be setup');
-  }
-  const infra = await infraPlugin.start();
-  const infraMetricIndices = await infra.getMetricIndices(savedObjectsClient);
-
-  return infraMetricIndices;
-}
diff --git a/x-pack/plugins/apm/server/routes/infrastructure/route.ts b/x-pack/plugins/apm/server/routes/infrastructure/route.ts
index 4117a43ce1e3f..9050f1a46622c 100644
--- a/x-pack/plugins/apm/server/routes/infrastructure/route.ts
+++ b/x-pack/plugins/apm/server/routes/infrastructure/route.ts
@@ -5,7 +5,6 @@
  * 2.0.
  */
 import * as t from 'io-ts';
-import Boom from '@hapi/boom';
 import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
 import { getApmEventClient } from '../../lib/helpers/get_apm_event_client';
 import { environmentRt, kueryRt, rangeRt } from '../default_api_types';
@@ -30,10 +29,6 @@ const infrastructureRoute = createApmServerRoute({
     hostNames: string[];
     podNames: string[];
   }> => {
-    if (!resources.plugins.infra) {
-      throw Boom.notFound();
-    }
-
     const apmEventClient = await getApmEventClient(resources);
     const infraMetricsClient = createInfraMetricsClient(resources);
     const { params } = resources;
diff --git a/x-pack/plugins/apm/server/routes/services/route.ts b/x-pack/plugins/apm/server/routes/services/route.ts
index 4ac0a37b3d10d..970a72d478f72 100644
--- a/x-pack/plugins/apm/server/routes/services/route.ts
+++ b/x-pack/plugins/apm/server/routes/services/route.ts
@@ -250,7 +250,7 @@ const serviceMetadataDetailsRoute = createApmServerRoute({
       end,
     });
 
-    if (serviceMetadataDetails?.container?.ids && resources.plugins.infra) {
+    if (serviceMetadataDetails?.container?.ids) {
       const infraMetricsClient = createInfraMetricsClient(resources);
       const containerMetadata = await getServiceOverviewContainerMetadata({
         infraMetricsClient,
@@ -761,10 +761,7 @@ export const serviceInstancesMetadataDetails = createApmServerRoute({
         end,
       });
 
-    if (
-      serviceInstanceMetadataDetails?.container?.id &&
-      resources.plugins.infra
-    ) {
+    if (serviceInstanceMetadataDetails?.container?.id) {
       const infraMetricsClient = createInfraMetricsClient(resources);
       const containerMetadata = await getServiceInstanceContainerMetadata({
         infraMetricsClient,
diff --git a/x-pack/plugins/metrics_data_access/server/client/client.test.ts b/x-pack/plugins/metrics_data_access/server/client/client.test.ts
index 72449cf47132b..d96d8efecf52f 100644
--- a/x-pack/plugins/metrics_data_access/server/client/client.test.ts
+++ b/x-pack/plugins/metrics_data_access/server/client/client.test.ts
@@ -7,18 +7,13 @@
 
 import { SavedObjectsErrorHelpers } from '@kbn/core/server';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
-import { MetricsDataClient } from './client';
+import { MetricsDataClient, DEFAULT_METRIC_INDICES } from './client';
 import { metricsDataSourceSavedObjectName } from '../saved_objects/metrics_data_source';
 
 describe('MetricsDataClient', () => {
-  const client = new MetricsDataClient();
-
-  client.setDefaultMetricIndicesHandler(async () => {
-    return 'fallback-indices*';
-  });
-
   describe('metric indices', () => {
     it('retrieves metrics saved object', async () => {
+      const client = new MetricsDataClient();
       const savedObjectsClient = {
         get: jest.fn().mockResolvedValue({ attributes: { metricIndices: 'foo,bar' } }),
       };
@@ -36,6 +31,10 @@ describe('MetricsDataClient', () => {
     });
 
     it('falls back to provided handler when no metrics saved object exists', async () => {
+      const client = new MetricsDataClient();
+      client.setDefaultMetricIndicesHandler(async () => {
+        return 'fallback-indices*';
+      });
       const savedObjectsClient = {
         get: jest.fn().mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()),
       };
@@ -51,5 +50,17 @@ describe('MetricsDataClient', () => {
       ]);
       expect(indices).toEqual('fallback-indices*');
     });
+
+    it('falls back to static indices when no fallback exists', async () => {
+      const client = new MetricsDataClient();
+      const savedObjectsClient = {
+        get: jest.fn().mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()),
+      };
+
+      const indices = await client.getMetricIndices({
+        savedObjectsClient: savedObjectsClient as unknown as SavedObjectsClientContract,
+      });
+      expect(indices).toEqual(DEFAULT_METRIC_INDICES);
+    });
   });
 });
diff --git a/x-pack/plugins/metrics_data_access/server/client/client.ts b/x-pack/plugins/metrics_data_access/server/client/client.ts
index 30d367cea0293..26359cae578a7 100644
--- a/x-pack/plugins/metrics_data_access/server/client/client.ts
+++ b/x-pack/plugins/metrics_data_access/server/client/client.ts
@@ -16,21 +16,19 @@ import {
   metricsDataSourceSavedObjectName,
 } from '../saved_objects/metrics_data_source';
 
+export const DEFAULT_METRIC_INDICES = 'metrics-*,metricbeat-*';
+
 export class MetricsDataClient {
   private readonly defaultSavedObjectId = 'default';
   private getDefaultMetricIndices: DefaultMetricIndicesHandler = null;
 
   async getMetricIndices(options: GetMetricIndicesOptions): Promise<string> {
-    if (!this.getDefaultMetricIndices) {
-      throw new Error('Missing getMetricsIndices fallback');
-    }
-
     const metricIndices = await options.savedObjectsClient
       .get<MetricsDataSavedObject>(metricsDataSourceSavedObjectName, this.defaultSavedObjectId)
       .then(({ attributes }) => attributes.metricIndices)
       .catch((err) => {
         if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
-          return this.getDefaultMetricIndices!(options);
+          return this.getDefaultMetricIndices?.(options) ?? DEFAULT_METRIC_INDICES;
         }
 
         throw err;

From 8c853b6ca77735b17db4b4f384e5d426189a755b Mon Sep 17 00:00:00 2001
From: Paulo Henrique <paulo.henrique@elastic.co>
Date: Mon, 25 Sep 2023 09:32:12 -0700
Subject: [PATCH 16/27] [Cloud Security] [CSPM] Update cloud native deployment
 instructions (#166419)

---
 .../aws_credentials_form/aws_credentials_form.tsx           | 6 ++++++
 .../components/fleet_extensions/gcp_credential_form.tsx     | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/aws_credentials_form/aws_credentials_form.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/aws_credentials_form/aws_credentials_form.tsx
index b1774547a6e6d..fdc7d8c0f328f 100644
--- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/aws_credentials_form/aws_credentials_form.tsx
+++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/aws_credentials_form/aws_credentials_form.tsx
@@ -133,6 +133,12 @@ const CloudFormationSetup = ({
             list-style: auto;
           `}
         >
+          <li>
+            <FormattedMessage
+              id="xpack.csp.awsIntegration.cloudFormationSetupStep.hostRequirement"
+              defaultMessage='Ensure "New hosts" is selected in the "Where to add this integration?" section below'
+            />
+          </li>
           {accountType === AWS_ORGANIZATION_ACCOUNT ? (
             <li>
               <FormattedMessage
diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx
index b4af438818b04..18ac1247d4b1a 100644
--- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx
+++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx
@@ -84,6 +84,12 @@ const GoogleCloudShellSetup = () => {
             list-style: auto;
           `}
         >
+          <li>
+            <FormattedMessage
+              id="xpack.csp.gcpIntegration.cloudShellSetupStep.hostRequirement"
+              defaultMessage='Ensure "New hosts" is selected in the "Where to add this integration?" section below'
+            />
+          </li>
           <li>
             <FormattedMessage
               id="xpack.csp.gcpIntegration.cloudShellSetupStep.login"

From f9c35e4971d629c47af7d23f0ea38c28261f033e Mon Sep 17 00:00:00 2001
From: Paulo Henrique <paulo.henrique@elastic.co>
Date: Mon, 25 Sep 2023 09:32:53 -0700
Subject: [PATCH 17/27] [Cloud Security] [Dashboard Navigation] Fix edit filter
 when navigating from dashboard (#166500)

---
 .../common/constants.ts                       |  2 +
 .../hooks/use_navigate_findings.test.ts       | 40 ++++++++++++++++---
 .../common/hooks/use_navigate_findings.ts     | 27 +++++++++----
 3 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts
index e1bcdc1f9a95f..7f0b4f62fb216 100644
--- a/x-pack/plugins/cloud_security_posture/common/constants.ts
+++ b/x-pack/plugins/cloud_security_posture/common/constants.ts
@@ -65,6 +65,8 @@ export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d';
 
 export const DATA_VIEW_INDEX_PATTERN = 'logs-*';
 
+export const SECURITY_DEFAULT_DATA_VIEW_ID = 'security-solution-default';
+
 export const CSP_INGEST_TIMESTAMP_PIPELINE = 'cloud_security_posture_add_ingest_timestamp_pipeline';
 export const CSP_LATEST_FINDINGS_INGEST_TIMESTAMP_PIPELINE =
   'cloud_security_posture_latest_index_add_ingest_timestamp_pipeline';
diff --git a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts
index 9bdd3bfada098..e3d213118dd51 100644
--- a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts
+++ b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts
@@ -6,7 +6,11 @@
  */
 
 import { renderHook, act } from '@testing-library/react-hooks/dom';
-import { useNavigateFindings, useNavigateFindingsByResource } from './use_navigate_findings';
+import {
+  useNavigateFindings,
+  useNavigateFindingsByResource,
+  useNavigateVulnerabilities,
+} from './use_navigate_findings';
 import { useHistory } from 'react-router-dom';
 
 jest.mock('react-router-dom', () => ({
@@ -29,9 +33,17 @@ jest.mock('./use_kibana', () => ({
     },
   }),
 }));
+jest.mock('../api/use_latest_findings_data_view', () => ({
+  useLatestFindingsDataView: jest.fn().mockReturnValue({
+    status: 'success',
+    data: {
+      id: 'data-view-id',
+    },
+  }),
+}));
 
 describe('useNavigateFindings', () => {
-  it('creates a URL to findings page with correct path and filter', () => {
+  it('creates a URL to findings page with correct path, filter and dataViewId', () => {
     const push = jest.fn();
     (useHistory as jest.Mock).mockReturnValueOnce({ push });
 
@@ -44,7 +56,7 @@ describe('useNavigateFindings', () => {
     expect(push).toHaveBeenCalledWith({
       pathname: '/cloud_security_posture/findings/configurations',
       search:
-        "cspq=(filters:!((meta:(alias:!n,disabled:!f,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
+        "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:data-view-id,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
     });
     expect(push).toHaveBeenCalledTimes(1);
   });
@@ -62,7 +74,7 @@ describe('useNavigateFindings', () => {
     expect(push).toHaveBeenCalledWith({
       pathname: '/cloud_security_posture/findings/configurations',
       search:
-        "cspq=(filters:!((meta:(alias:!n,disabled:!f,key:foo,negate:!t,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
+        "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:data-view-id,key:foo,negate:!t,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
     });
     expect(push).toHaveBeenCalledTimes(1);
   });
@@ -80,7 +92,25 @@ describe('useNavigateFindings', () => {
     expect(push).toHaveBeenCalledWith({
       pathname: '/cloud_security_posture/findings/resource',
       search:
-        "cspq=(filters:!((meta:(alias:!n,disabled:!f,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
+        "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:data-view-id,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
+    });
+    expect(push).toHaveBeenCalledTimes(1);
+  });
+
+  it('creates a URL to vulnerabilities page with correct path, filter and dataViewId', () => {
+    const push = jest.fn();
+    (useHistory as jest.Mock).mockReturnValueOnce({ push });
+
+    const { result } = renderHook(() => useNavigateVulnerabilities());
+
+    act(() => {
+      result.current({ foo: 1 });
+    });
+
+    expect(push).toHaveBeenCalledWith({
+      pathname: '/cloud_security_posture/findings/vulnerabilities',
+      search:
+        "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:security-solution-default,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))",
     });
     expect(push).toHaveBeenCalledTimes(1);
   });
diff --git a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts
index 48b16f62cbaf5..fbeeeb32a0c2e 100644
--- a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts
+++ b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts
@@ -8,9 +8,14 @@
 import { useCallback } from 'react';
 import { useHistory } from 'react-router-dom';
 import { Filter } from '@kbn/es-query';
+import {
+  LATEST_FINDINGS_INDEX_PATTERN,
+  SECURITY_DEFAULT_DATA_VIEW_ID,
+} from '../../../common/constants';
 import { findingsNavigation } from '../navigation/constants';
 import { encodeQuery } from '../navigation/query_utils';
 import { useKibana } from './use_kibana';
+import { useLatestFindingsDataView } from '../api/use_latest_findings_data_view';
 
 interface NegatedValue {
   value: string | number;
@@ -21,7 +26,7 @@ type FilterValue = string | number | NegatedValue;
 
 export type NavFilter = Record<string, FilterValue>;
 
-const createFilter = (key: string, filterValue: FilterValue): Filter => {
+const createFilter = (key: string, filterValue: FilterValue, dataViewId: string): Filter => {
   let negate = false;
   let value = filterValue;
   if (typeof filterValue === 'object') {
@@ -32,7 +37,7 @@ const createFilter = (key: string, filterValue: FilterValue): Filter => {
   if (value === '*') {
     return {
       query: { exists: { field: key } },
-      meta: { type: 'exists' },
+      meta: { type: 'exists', index: dataViewId },
     };
   }
   return {
@@ -42,18 +47,19 @@ const createFilter = (key: string, filterValue: FilterValue): Filter => {
       disabled: false,
       type: 'phrase',
       key,
+      index: dataViewId,
     },
     query: { match_phrase: { [key]: value } },
   };
 };
-const useNavigate = (pathname: string) => {
+const useNavigate = (pathname: string, dataViewId = SECURITY_DEFAULT_DATA_VIEW_ID) => {
   const history = useHistory();
   const { services } = useKibana();
 
   return useCallback(
     (filterParams: NavFilter = {}) => {
       const filters = Object.entries(filterParams).map(([key, filterValue]) =>
-        createFilter(key, filterValue)
+        createFilter(key, filterValue, dataViewId)
       );
 
       history.push({
@@ -65,14 +71,19 @@ const useNavigate = (pathname: string) => {
         }),
       });
     },
-    [pathname, history, services.data.query.queryString]
+    [pathname, history, services.data.query.queryString, dataViewId]
   );
 };
 
-export const useNavigateFindings = () => useNavigate(findingsNavigation.findings_default.path);
+export const useNavigateFindings = () => {
+  const { data } = useLatestFindingsDataView(LATEST_FINDINGS_INDEX_PATTERN);
+  return useNavigate(findingsNavigation.findings_default.path, data?.id);
+};
 
-export const useNavigateFindingsByResource = () =>
-  useNavigate(findingsNavigation.findings_by_resource.path);
+export const useNavigateFindingsByResource = () => {
+  const { data } = useLatestFindingsDataView(LATEST_FINDINGS_INDEX_PATTERN);
+  return useNavigate(findingsNavigation.findings_by_resource.path, data?.id);
+};
 
 export const useNavigateVulnerabilities = () =>
   useNavigate(findingsNavigation.vulnerabilities.path);

From 778dbf26b9bcd5b43e53a41b1646974dfe9352ea Mon Sep 17 00:00:00 2001
From: Paulo Henrique <paulo.henrique@elastic.co>
Date: Mon, 25 Sep 2023 09:33:13 -0700
Subject: [PATCH 18/27] [Cloud Security] [Misconfigurations] Test coverage for
 the Alerts workflow (#166788)

---
 .../detection_rule_counter.test.tsx           | 138 ++++++++++
 .../components/detection_rule_counter.tsx     |  24 +-
 .../public/components/take_action.tsx         |   9 +-
 .../page_objects/findings_page.ts             |  70 +++++-
 .../pages/findings_alerts.ts                  | 236 ++++++++++++++++++
 .../pages/index.ts                            |   1 +
 6 files changed, 468 insertions(+), 10 deletions(-)
 create mode 100644 x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx
 create mode 100644 x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts

diff --git a/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx
new file mode 100644
index 0000000000000..1e2f2f52fd02a
--- /dev/null
+++ b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { render, waitFor } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import { DetectionRuleCounter } from './detection_rule_counter';
+import { TestProvider } from '../test/test_provider';
+import { useFetchDetectionRulesByTags } from '../common/api/use_fetch_detection_rules_by_tags';
+import { useFetchDetectionRulesAlertsStatus } from '../common/api/use_fetch_detection_rules_alerts_status';
+import { RuleResponse } from '../common/types';
+
+jest.mock('../common/api/use_fetch_detection_rules_by_tags', () => ({
+  useFetchDetectionRulesByTags: jest.fn(),
+}));
+jest.mock('../common/api/use_fetch_detection_rules_alerts_status', () => ({
+  useFetchDetectionRulesAlertsStatus: jest.fn(),
+}));
+
+describe('DetectionRuleCounter', () => {
+  beforeEach(() => {
+    jest.restoreAllMocks();
+  });
+  it('should render loading skeleton when both rules and alerts are loading', () => {
+    (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({
+      data: undefined,
+      isLoading: true,
+    });
+
+    (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({
+      data: undefined,
+      isLoading: true,
+    });
+    const { getByTestId } = render(
+      <TestProvider>
+        <DetectionRuleCounter tags={['tag1', 'tag2']} createRuleFn={jest.fn()} />
+      </TestProvider>
+    );
+
+    const skeletonText = getByTestId('csp:detection-rule-counter-loading');
+    expect(skeletonText).toBeInTheDocument();
+  });
+
+  it('should render create rule link when no rules exist', () => {
+    (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({
+      data: { total: 0 },
+      isLoading: false,
+    });
+
+    (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({
+      data: null,
+      isLoading: false,
+      isFetching: false,
+    });
+
+    const { getByText, getByTestId } = render(
+      <TestProvider>
+        <DetectionRuleCounter tags={['tag1', 'tag2']} createRuleFn={jest.fn()} />
+      </TestProvider>
+    );
+
+    const createRuleLink = getByTestId('csp:findings-flyout-create-detection-rule-link');
+    expect(createRuleLink).toBeInTheDocument();
+    expect(getByText('Create a detection rule')).toBeInTheDocument();
+  });
+
+  it('should render alert and rule count when rules exist', () => {
+    (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({
+      data: { total: 5 },
+      isLoading: false,
+    });
+
+    (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({
+      data: { total: 10 },
+      isLoading: false,
+      isFetching: false,
+    });
+
+    const { getByText, getByTestId } = render(
+      <TestProvider>
+        <DetectionRuleCounter tags={['tag1', 'tag2']} createRuleFn={jest.fn()} />
+      </TestProvider>
+    );
+
+    const alertCountLink = getByTestId('csp:findings-flyout-alert-count');
+    const ruleCountLink = getByTestId('csp:findings-flyout-detection-rule-count');
+
+    expect(alertCountLink).toBeInTheDocument();
+    expect(getByText(/10 alerts/i)).toBeInTheDocument();
+    expect(ruleCountLink).toBeInTheDocument();
+    expect(getByText(/5 detection rules/i)).toBeInTheDocument();
+  });
+
+  it('should show loading spinner when creating a rule', async () => {
+    (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({
+      data: { total: 0 },
+      isLoading: false,
+    });
+
+    (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({
+      data: null,
+      isLoading: false,
+      isFetching: false,
+    });
+    const createRuleFn = jest.fn(() => Promise.resolve({} as RuleResponse));
+    const { getByTestId, queryByTestId } = render(
+      <TestProvider>
+        <DetectionRuleCounter tags={['tag1', 'tag2']} createRuleFn={createRuleFn} />
+      </TestProvider>
+    );
+
+    // Trigger createDetectionRuleOnClick
+    const createRuleLink = getByTestId('csp:findings-flyout-create-detection-rule-link');
+    userEvent.click(createRuleLink);
+
+    const loadingSpinner = getByTestId('csp:findings-flyout-detection-rule-counter-loading');
+    expect(loadingSpinner).toBeInTheDocument();
+
+    (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({
+      data: { total: 1 },
+      isLoading: false,
+    });
+
+    (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({
+      data: { total: 0 },
+      isLoading: false,
+      isFetching: false,
+    });
+
+    // Wait for the loading spinner to disappear
+    await waitFor(() => {
+      expect(queryByTestId('csp:findings-flyout-detection-rule-counter-loading')).toBeNull();
+    });
+  });
+});
diff --git a/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx
index eeea89f9a310f..b2a79710d09a2 100644
--- a/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx
+++ b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx
@@ -68,7 +68,12 @@ export const DetectionRuleCounter = ({ tags, createRuleFn }: DetectionRuleCounte
   }, [createRuleFn, http, notifications, queryClient]);
 
   return (
-    <EuiSkeletonText lines={1} size="m" isLoading={ruleIsLoading || alertsIsLoading}>
+    <EuiSkeletonText
+      data-test-subj="csp:detection-rule-counter-loading"
+      lines={1}
+      size="m"
+      isLoading={ruleIsLoading || alertsIsLoading}
+    >
       {rulesData?.total === 0 ? (
         <>
           <EuiText size="s">
@@ -78,11 +83,17 @@ export const DetectionRuleCounter = ({ tags, createRuleFn }: DetectionRuleCounte
                   id="xpack.csp.findingsFlyout.alerts.creatingRule"
                   defaultMessage="Creating detection rule"
                 />{' '}
-                <EuiLoadingSpinner size="s" />
+                <EuiLoadingSpinner
+                  size="s"
+                  data-test-subj="csp:findings-flyout-detection-rule-counter-loading"
+                />
               </>
             ) : (
               <>
-                <EuiLink onClick={createDetectionRuleOnClick}>
+                <EuiLink
+                  onClick={createDetectionRuleOnClick}
+                  data-test-subj="csp:findings-flyout-create-detection-rule-link"
+                >
                   <FormattedMessage
                     id="xpack.csp.findingsFlyout.alerts.createRuleAction"
                     defaultMessage="Create a detection rule"
@@ -98,7 +109,7 @@ export const DetectionRuleCounter = ({ tags, createRuleFn }: DetectionRuleCounte
         </>
       ) : (
         <>
-          <EuiLink onClick={alertsPageNavigation}>
+          <EuiLink onClick={alertsPageNavigation} data-test-subj="csp:findings-flyout-alert-count">
             <FormattedMessage
               id="xpack.csp.findingsFlyout.alerts.alertCount"
               defaultMessage="{alertCount, plural, one {# alert} other {# alerts}}"
@@ -109,7 +120,10 @@ export const DetectionRuleCounter = ({ tags, createRuleFn }: DetectionRuleCounte
             id="xpack.csp.findingsFlyout.alerts.detectedBy"
             defaultMessage="detected by"
           />{' '}
-          <EuiLink onClick={rulePageNavigation}>
+          <EuiLink
+            onClick={rulePageNavigation}
+            data-test-subj="csp:findings-flyout-detection-rule-count"
+          >
             <FormattedMessage
               id="xpack.csp.findingsFlyout.alerts.detectionRuleCount"
               defaultMessage="{ruleCount, plural, one {# detection rule} other {# detection rules}}"
diff --git a/x-pack/plugins/cloud_security_posture/public/components/take_action.tsx b/x-pack/plugins/cloud_security_posture/public/components/take_action.tsx
index f4035f1532b7c..f72a4f7114be3 100644
--- a/x-pack/plugins/cloud_security_posture/public/components/take_action.tsx
+++ b/x-pack/plugins/cloud_security_posture/public/components/take_action.tsx
@@ -40,10 +40,11 @@ export const showSuccessToast = (
     toastLifeTimeMs: 10000,
     color: 'success',
     iconType: '',
+    'data-test-subj': 'csp:toast-success',
     text: toMountPoint(
       <div>
         <EuiText size="m">
-          <strong>{ruleResponse.name}</strong>
+          <strong data-test-subj="csp:toast-success-title">{ruleResponse.name}</strong>
           {` `}
           <FormattedMessage
             id="xpack.csp.flyout.ruleCreatedToastTitle"
@@ -58,7 +59,11 @@ export const showSuccessToast = (
         </EuiText>
         <EuiFlexGroup justifyContent="flexEnd" gutterSize="s">
           <EuiFlexItem grow={false}>
-            <EuiButton size="s" href={http.basePath.prepend(RULE_PAGE_PATH + ruleResponse.id)}>
+            <EuiButton
+              data-test-subj="csp:toast-success-link"
+              size="s"
+              href={http.basePath.prepend(RULE_PAGE_PATH + ruleResponse.id)}
+            >
               <FormattedMessage
                 id="xpack.csp.flyout.ruleCreatedToastViewRuleButton"
                 defaultMessage="View rule"
diff --git a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
index 94b023757bf19..ab0e0302e163d 100644
--- a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
+++ b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
@@ -55,14 +55,20 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
           refresh: true,
         }),
       ]),
-    add: async <T>(findingsMock: T[]) => {
+    add: async <
+      T extends {
+        '@timestamp'?: string;
+      }
+    >(
+      findingsMock: T[]
+    ) => {
       await Promise.all([
         ...findingsMock.map((finding) =>
           es.index({
             index: FINDINGS_INDEX,
             body: {
               ...finding,
-              '@timestamp': new Date().toISOString(),
+              '@timestamp': finding['@timestamp'] ?? new Date().toISOString(),
             },
             refresh: true,
           })
@@ -72,7 +78,7 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
             index: FINDINGS_LATEST_INDEX,
             body: {
               ...finding,
-              '@timestamp': new Date().toISOString(),
+              '@timestamp': finding['@timestamp'] ?? new Date().toISOString(),
             },
             refresh: true,
           })
@@ -81,6 +87,20 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
     },
   };
 
+  const detectionRuleApi = {
+    remove: async () => {
+      await supertest
+        .post('/api/detection_engine/rules/_bulk_action?dry_run=false')
+        .set('kbn-xsrf', 'true')
+        .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31')
+        .send({
+          action: 'delete',
+          query: '',
+        })
+        .expect(200);
+    },
+  };
+
   const distributionBar = {
     filterBy: async (type: 'passed' | 'failed') =>
       testSubjects.click(type === 'failed' ? 'distribution_bar_failed' : 'distribution_bar_passed'),
@@ -203,6 +223,12 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
         await nonStaleElement.click();
       }
     },
+
+    async openFlyoutAt(rowIndex: number) {
+      const table = await this.getElement();
+      const flyoutButton = await table.findAllByTestSubject('findings_table_expand_column');
+      await flyoutButton[rowIndex].click();
+    },
   });
 
   const navigateToLatestFindingsPage = async () => {
@@ -247,6 +273,41 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
   const notInstalledVulnerabilities = createNotInstalledObject('cnvm-integration-not-installed');
   const notInstalledCSP = createNotInstalledObject('cloud_posture_page_package_not_installed');
 
+  const createFlyoutObject = (tableTestSubject: string) => ({
+    async getElement() {
+      return await testSubjects.find(tableTestSubject);
+    },
+    async clickTakeActionButton() {
+      const element = await this.getElement();
+      const button = await element.findByCssSelector('[data-test-subj="csp:take_action"] button');
+      await button.click();
+      return button;
+    },
+    async clickTakeActionCreateRuleButton() {
+      await this.clickTakeActionButton();
+      const button = await testSubjects.find('csp:create_rule');
+      await button.click();
+      return button;
+    },
+    async getVisibleText(testSubj: string) {
+      const element = await this.getElement();
+      return await (await element.findByTestSubject(testSubj)).getVisibleText();
+    },
+  });
+
+  const misconfigurationsFlyout = createFlyoutObject('findings_flyout');
+
+  const toastMessage = async (testSubj = 'csp:toast-success') => ({
+    async getElement() {
+      return await testSubjects.find(testSubj);
+    },
+    async clickToastMessageLink(linkTestSubj = 'csp:toast-success-link') {
+      const element = await this.getElement();
+      const link = await element.findByTestSubject(linkTestSubj);
+      await link.click();
+    },
+  });
+
   return {
     navigateToLatestFindingsPage,
     navigateToVulnerabilities,
@@ -259,5 +320,8 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
     index,
     waitForPluginInitialized,
     distributionBar,
+    misconfigurationsFlyout,
+    toastMessage,
+    detectionRuleApi,
   };
 }
diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts b/x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts
new file mode 100644
index 0000000000000..53f0b765165ef
--- /dev/null
+++ b/x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts
@@ -0,0 +1,236 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import Chance from 'chance';
+import type { FtrProviderContext } from '../ftr_provider_context';
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const testSubjects = getService('testSubjects');
+  const retry = getService('retry');
+  const pageObjects = getPageObjects(['common', 'findings', 'header']);
+  const chance = new Chance();
+
+  // We need to use a dataset for the tests to run
+  const data = [
+    {
+      resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
+      result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
+      rule: {
+        tags: ['CIS', 'CIS K8S'],
+        rationale: 'rationale steps for rule 1.1',
+        references: '1. https://elastic.co/rules/1.1',
+        name: 'Upper case rule name',
+        section: 'Upper case section',
+        benchmark: {
+          rule_number: '1.1',
+          id: 'cis_k8s',
+          posture_type: 'kspm',
+          name: 'CIS Kubernetes V1.23',
+          version: 'v1.0.0',
+          remediation: 'remediation guide',
+        },
+        type: 'process',
+      },
+      cluster_id: 'Upper case cluster id',
+    },
+    {
+      '@timestamp': '2023-09-10T14:01:00.000Z',
+      resource: { id: chance.guid(), name: `Pod`, sub_type: 'Upper case sub type' },
+      result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
+      rule: {
+        tags: ['CIS', 'CIS K8S'],
+        rationale: 'rationale steps',
+        references: '1. https://elastic.co',
+        name: 'lower case rule name',
+        section: 'Another upper case section',
+        benchmark: {
+          rule_number: '1.2',
+          id: 'cis_k8s',
+          posture_type: 'kspm',
+          name: 'CIS Kubernetes V1.23',
+          version: 'v1.0.0',
+          remediation: 'remediation guide',
+        },
+        type: 'process',
+      },
+      cluster_id: 'Another Upper case cluster id',
+    },
+    {
+      '@timestamp': '2023-09-10T14:02:00.000Z',
+      resource: { id: chance.guid(), name: `process`, sub_type: 'another lower case type' },
+      result: { evaluation: 'passed' },
+      rule: {
+        tags: ['CIS', 'CIS K8S'],
+        rationale: 'rationale steps',
+        references: '1. https://elastic.co',
+        name: 'Another upper case rule name',
+        section: 'lower case section',
+        benchmark: {
+          rule_number: '1.3',
+          id: 'cis_k8s',
+          posture_type: 'kspm',
+          name: 'CIS Kubernetes V1.23',
+          version: 'v1.0.0',
+          remediation: 'remediation guide',
+        },
+        type: 'process',
+      },
+      cluster_id: 'lower case cluster id',
+    },
+    {
+      '@timestamp': '2023-09-10T14:03:00.000Z',
+      resource: { id: chance.guid(), name: `process`, sub_type: 'Upper case type again' },
+      result: { evaluation: 'failed' },
+      rule: {
+        tags: ['CIS', 'CIS K8S'],
+        rationale: 'rationale steps',
+        references: '1. https://elastic.co',
+        name: 'some lower case rule name',
+        section: 'another lower case section',
+        benchmark: {
+          rule_number: '1.4',
+          id: 'cis_k8s',
+          posture_type: 'kspm',
+          name: 'CIS Kubernetes V1.23',
+          version: 'v1.0.0',
+          remediation: 'remediation guide',
+        },
+        type: 'process',
+      },
+      cluster_id: 'another lower case cluster id',
+    },
+  ];
+
+  const ruleName1 = data[0].rule.name;
+
+  describe('Findings Page - Alerts', function () {
+    this.tags(['cloud_security_posture_findings_alerts']);
+    let findings: typeof pageObjects.findings;
+    let latestFindingsTable: typeof findings.latestFindingsTable;
+    let misconfigurationsFlyout: typeof findings.misconfigurationsFlyout;
+
+    before(async () => {
+      findings = pageObjects.findings;
+      latestFindingsTable = findings.latestFindingsTable;
+      misconfigurationsFlyout = findings.misconfigurationsFlyout;
+      // Before we start any test we must wait for cloud_security_posture plugin to complete its initialization
+      await findings.waitForPluginInitialized();
+      // Prepare mocked findings
+      await findings.index.remove();
+      await findings.index.add(data);
+    });
+
+    after(async () => {
+      await findings.index.remove();
+      await findings.detectionRuleApi.remove();
+    });
+
+    beforeEach(async () => {
+      await findings.detectionRuleApi.remove();
+      await findings.navigateToLatestFindingsPage();
+      await retry.waitFor(
+        'Findings table to be loaded',
+        async () => (await latestFindingsTable.getRowsCount()) === data.length
+      );
+      pageObjects.header.waitUntilLoadingHasFinished();
+    });
+
+    describe('Create detection rule', () => {
+      it('Creates a detection rule from the Take Action button and navigates to rule page', async () => {
+        await latestFindingsTable.openFlyoutAt(0);
+        await misconfigurationsFlyout.clickTakeActionCreateRuleButton();
+
+        expect(
+          await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-alert-count')
+        ).to.be('0 alerts');
+
+        expect(
+          await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-detection-rule-count')
+        ).to.be('1 detection rule');
+
+        const toastMessage = await (await findings.toastMessage()).getElement();
+        expect(toastMessage).to.be.ok();
+
+        const toastMessageTitle = await toastMessage.findByTestSubject('csp:toast-success-title');
+        expect(await toastMessageTitle.getVisibleText()).to.be(ruleName1);
+
+        await (await findings.toastMessage()).clickToastMessageLink();
+
+        const rulePageTitle = await testSubjects.find('header-page-title');
+        expect(await rulePageTitle.getVisibleText()).to.be(ruleName1);
+      });
+      it('Creates a detection rule from the Alerts section and navigates to rule page', async () => {
+        await latestFindingsTable.openFlyoutAt(0);
+        const flyout = await misconfigurationsFlyout.getElement();
+
+        await (
+          await flyout.findByTestSubject('csp:findings-flyout-create-detection-rule-link')
+        ).click();
+
+        expect(
+          await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-alert-count')
+        ).to.be('0 alerts');
+
+        expect(
+          await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-detection-rule-count')
+        ).to.be('1 detection rule');
+
+        const toastMessage = await (await findings.toastMessage()).getElement();
+        expect(toastMessage).to.be.ok();
+
+        const toastMessageTitle = await toastMessage.findByTestSubject('csp:toast-success-title');
+        expect(await toastMessageTitle.getVisibleText()).to.be(ruleName1);
+
+        await (await findings.toastMessage()).clickToastMessageLink();
+
+        const rulePageTitle = await testSubjects.find('header-page-title');
+        expect(await rulePageTitle.getVisibleText()).to.be(ruleName1);
+      });
+    });
+    describe('Rule details', () => {
+      it('The rule page contains the expected matching data', async () => {
+        await latestFindingsTable.openFlyoutAt(0);
+        await misconfigurationsFlyout.clickTakeActionCreateRuleButton();
+
+        await (await findings.toastMessage()).clickToastMessageLink();
+
+        const rulePageDescription = await testSubjects.find(
+          'stepAboutRuleDetailsToggleDescriptionText'
+        );
+        expect(await rulePageDescription.getVisibleText()).to.be(data[0].rule.rationale);
+
+        const severity = await testSubjects.find('severity');
+        expect(await severity.getVisibleText()).to.be('Low');
+
+        const referenceUrls = await testSubjects.find('urlsDescriptionReferenceLinkItem');
+        expect(await referenceUrls.getVisibleText()).to.contain('https://elastic.co/rules/1.1');
+      });
+    });
+    describe('Navigation', () => {
+      it('Clicking on count of Rules should navigate to the rules page with benchmark tags as a filter', async () => {
+        await latestFindingsTable.openFlyoutAt(0);
+        await misconfigurationsFlyout.clickTakeActionCreateRuleButton();
+        const flyout = await misconfigurationsFlyout.getElement();
+        await (await flyout.findByTestSubject('csp:findings-flyout-detection-rule-count')).click();
+
+        expect(await (await testSubjects.find('ruleName')).getVisibleText()).to.be(ruleName1);
+      });
+      it('Clicking on count of Alerts should navigate to the alerts page', async () => {
+        await latestFindingsTable.openFlyoutAt(0);
+        await misconfigurationsFlyout.clickTakeActionCreateRuleButton();
+        const flyout = await misconfigurationsFlyout.getElement();
+        await (await flyout.findByTestSubject('csp:findings-flyout-alert-count')).click();
+
+        expect(await (await testSubjects.find('header-page-title')).getVisibleText()).to.be(
+          'Alerts'
+        );
+      });
+    });
+  });
+}
diff --git a/x-pack/test/cloud_security_posture_functional/pages/index.ts b/x-pack/test/cloud_security_posture_functional/pages/index.ts
index 1d30a3b27fda9..81e905ddaca35 100644
--- a/x-pack/test/cloud_security_posture_functional/pages/index.ts
+++ b/x-pack/test/cloud_security_posture_functional/pages/index.ts
@@ -12,6 +12,7 @@ export default function ({ loadTestFile }: FtrProviderContext) {
   describe('Cloud Security Posture', function () {
     loadTestFile(require.resolve('./findings_onboarding'));
     loadTestFile(require.resolve('./findings'));
+    loadTestFile(require.resolve('./findings_alerts'));
     loadTestFile(require.resolve('./compliance_dashboard'));
   });
 }

From 7f82102d720b5dffb5dafe2fd48a8ba760b9b4ec Mon Sep 17 00:00:00 2001
From: Alex Szabo <alex.szabo@elastic.co>
Date: Mon, 25 Sep 2023 18:49:20 +0200
Subject: [PATCH 19/27] [Ops] ES Serverless image verification pipeline
 (#166054)

## Summary
Prepares the serverless FTR tests to be runnable with a custom ES image.
(`--esServerlessImage` cli arg)
Creates a pipeline for testing and promoting ES Serverless docker
releases.

The job can be triggered here:
https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote
The three main env variables it takes:
- BUILDKITE_BRANCH: the kibana branch to test with (maybe not as
important)
 - BUILDKITE_COMMIT: the kibana commit to test with
- ES_SERVERLESS_IMAGE: the elasticsearch serverless image, or tag to use
from this repo:
`docker.elastic.co/elasticsearch-ci/elasticsearch-serverless`

## TODOS:
 - [x] set `latest_verified` with full img path as default
- [x] ~~find other CLIs that might need the `esServerlessImage` argument
(if the docker runner has multiple usages)~~ | I confused the `yarn es
docker` with this, because I thought we only run ES serverless in a
docker container, but `elasticsearch` can also be run in docker.
- [x] set `latest-compatible` or similar flag in a manifest in gcs for
Elastic's use-case
- [ ] ensure we can only verify "forward" (ie.: to avoid a
parameterization on old versions to set our pointers back) [on a second
thought, this might be kept as a feature to roll back (if we should ever
need that)]


There are two confusing things I couldn't sort out just yet:
#### Ambiguity in --esServerlessImage
We can either have 2 CLI args: one for an image tag, one for an image
repo/image url, or we can have one (like I have it now) and interpret
that in the code, it can be either the image url, or the tag. It's more
flexible, but it's two things in one. Is it ok this way, or is it too
confusing?
e.g.:
```
node scripts/functional_tests --esFrom serverless --esServerlessImage docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:git-8fc8f941bd4d --bail --config x-pack/test_serverless/functional/test_suites/security/config.ts

# or
node scripts/functional_tests  --esFrom serverless --esServerlessImage latest --bail --config x-pack/test_serverless/functional/test_suites/security/config.ts
```

#### Ambiguity in the default image path
The published ES Serverless images will sit on this image path:
`docker.elastic.co/elasticsearch-ci/elasticsearch-serverless`, however,
our one exception is the `latest-verified` which we will be tagging
under a different path, where we have write rights:
`docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified`.

Is it okay, that by default, we're searching in the `elasticsearch-ci`
images for any tags as parameters (after all, all the new images will be
published there), however our grand default will ultimately be
`docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified`.


## Links
Buildkite:
https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote
eg.:
https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/builds/24
Closes: https://github.com/elastic/kibana/issues/162931

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../verify_es_serverless_image.yml            | 58 ++++++++++++++
 .../promote_es_serverless_image.sh            | 75 +++++++++++++++++++
 packages/kbn-dev-cli-runner/src/flags.ts      |  2 +
 packages/kbn-dev-cli-runner/src/help.ts       |  8 +-
 packages/kbn-dev-cli-runner/src/run.ts        |  1 +
 .../kbn-es/src/cli_commands/serverless.ts     | 16 ++--
 packages/kbn-es/src/utils/docker.test.ts      | 10 ++-
 packages/kbn-es/src/utils/docker.ts           | 49 ++++++++----
 packages/kbn-test/src/es/es_test_config.ts    |  4 +
 packages/kbn-test/src/es/test_es_cluster.ts   | 16 +++-
 .../functional_tests/lib/run_elasticsearch.ts | 28 ++++++-
 .../src/functional_tests/run_tests/cli.ts     |  1 +
 .../functional_tests/run_tests/flags.test.ts  |  1 +
 .../src/functional_tests/run_tests/flags.ts   | 10 +++
 14 files changed, 246 insertions(+), 33 deletions(-)
 create mode 100644 .buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
 create mode 100755 .buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
new file mode 100644
index 0000000000000..f6cb21abdb682
--- /dev/null
+++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@@ -0,0 +1,58 @@
+# https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/
+agents:
+  queue: kibana-default
+
+steps:
+  - label: "Annotate runtime parameters"
+    command: |
+      buildkite-agent annotate --context es-serverless-image --style info "ES Serverless image: $ES_SERVERLESS_IMAGE"
+      buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT"
+
+  - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite"
+    if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'"
+    steps:
+      - label: "Pre-Build"
+        command: .buildkite/scripts/lifecycle/pre_build.sh
+        key: pre-build
+        timeout_in_minutes: 10
+        agents:
+          queue: kibana-default
+
+      - label: "Build Kibana Distribution and Plugins"
+        command: .buildkite/scripts/steps/build_kibana.sh
+        agents:
+          queue: n2-16-spot
+        key: build
+        depends_on: pre-build
+        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - label: "Pick Test Group Run Order"
+        command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+        agents:
+          queue: kibana-default
+        env:
+          FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
+          FTR_CONFIG_PATTERNS: '**/test_serverless/**'
+          LIMIT_CONFIG_TYPE: 'functional'
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+  - wait: ~
+
+  - label: ":arrow_up::elastic::arrow_up: Promote docker image"
+    command: .buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh $ES_SERVERLESS_IMAGE
+
+  - wait: ~
+
+  - label: 'Post-Build'
+    command: .buildkite/scripts/lifecycle/post_build.sh
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
diff --git a/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh b/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh
new file mode 100755
index 0000000000000..c6bf1738fe144
--- /dev/null
+++ b/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+BASE_ES_SERVERLESS_REPO=docker.elastic.co/elasticsearch-ci/elasticsearch-serverless
+TARGET_IMAGE=docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
+
+ES_SERVERLESS_BUCKET=kibana-ci-es-serverless-images
+MANIFEST_FILE_NAME=latest-verified.json
+
+SOURCE_IMAGE_OR_TAG=$1
+if [[ $SOURCE_IMAGE_OR_TAG =~ :[a-zA-Z_-]+$ ]]; then
+  # $SOURCE_IMAGE_OR_TAG was a full image
+  SOURCE_IMAGE=$SOURCE_IMAGE_OR_TAG
+else
+  # $SOURCE_IMAGE_OR_TAG was an image tag
+  SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG"
+fi
+
+echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+
+echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+docker pull "$SOURCE_IMAGE"
+docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE"
+docker push "$TARGET_IMAGE"
+
+ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE")
+ELASTIC_COMMIT_HASH=$(echo $ORIG_IMG_DATA | jq -r '.[].Config.Labels["org.opencontainers.image.revision"]')
+
+docker logout docker.elastic.co
+
+echo "Image push to $TARGET_IMAGE successful."
+echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
+
+MANIFEST_UPLOAD_PATH="Skipped"
+if [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Uploading latest-verified manifest to GCS"
+  cat << EOT >> $MANIFEST_FILE_NAME
+{
+  "build_url": "$BUILDKITE_BUILD_URL",
+  "kibana_commit": "$BUILDKITE_COMMIT",
+  "kibana_branch": "$BUILDKITE_BRANCH",
+  "elasticsearch_serverless_tag": "$SOURCE_IMAGE_OR_TAG",
+  "elasticsearch_serverless_image_url: "$SOURCE_IMAGE",
+  "elasticsearch_serverless_commit": "TODO: this currently can't be decided",
+  "elasticsearch_commit": "$ELASTIC_COMMIT_HASH",
+  "created_at": "`date`",
+  "timestamp": "`FORCE_COLOR=0 node -p 'Date.now()'`"
+}
+EOT
+
+  gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" \
+    cp $MANIFEST_FILE_NAME "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
+
+elif [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
+elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"
+fi
+
+echo "--- Annotating build with info"
+cat << EOT | buildkite-agent annotate --style "success"
+  <h2>Promotion successful!</h2>
+  <br/>New image: $TARGET_IMAGE
+  <br/>Source image: $SOURCE_IMAGE
+  <br/>Kibana commit: <a href="https://github.com/elastic/kibana/commit/$BUILDKITE_COMMIT">$BUILDKITE_COMMIT</a>
+  <br/>Elasticsearch commit: <a href="https://github.com/elastic/elasticsearch/commit/$ELASTIC_COMMIT_HASH">$ELASTIC_COMMIT_HASH</a>
+  <br/>Manifest file: $MANIFEST_UPLOAD_PATH
+EOT
diff --git a/packages/kbn-dev-cli-runner/src/flags.ts b/packages/kbn-dev-cli-runner/src/flags.ts
index 595205c3e0333..d7b352333ae1b 100644
--- a/packages/kbn-dev-cli-runner/src/flags.ts
+++ b/packages/kbn-dev-cli-runner/src/flags.ts
@@ -27,6 +27,7 @@ export interface FlagOptions {
   allowUnexpected?: boolean;
   guessTypesForUnexpectedFlags?: boolean;
   help?: string;
+  examples?: string;
   alias?: { [key: string]: string | string[] };
   boolean?: string[];
   string?: string[];
@@ -47,6 +48,7 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions =
     },
 
     help: local.help,
+    examples: local.examples,
 
     allowUnexpected: !!(global.allowUnexpected || local.allowUnexpected),
     guessTypesForUnexpectedFlags: !!(global.allowUnexpected || local.allowUnexpected),
diff --git a/packages/kbn-dev-cli-runner/src/help.ts b/packages/kbn-dev-cli-runner/src/help.ts
index a7dc17aa43f17..f3e0e2c78e97f 100644
--- a/packages/kbn-dev-cli-runner/src/help.ts
+++ b/packages/kbn-dev-cli-runner/src/help.ts
@@ -36,11 +36,13 @@ export function getHelp({
   usage,
   flagHelp,
   defaultLogLevel,
+  examples,
 }: {
   description?: string;
   usage?: string;
   flagHelp?: string;
   defaultLogLevel?: string;
+  examples?: string;
 }) {
   const optionHelp = joinAndTrimLines(
     dedent(flagHelp || ''),
@@ -48,13 +50,17 @@ export function getHelp({
     GLOBAL_FLAGS
   );
 
+  const examplesHelp = examples ? joinAndTrimLines('Examples:', examples) : '';
+
   return `
   ${dedent(usage || '') || DEFAULT_GLOBAL_USAGE}
 
   ${indent(dedent(description || 'Runs a dev task'), 2)}
 
   Options:
-    ${indent(optionHelp, 4)}\n\n`;
+    ${indent(optionHelp, 4)}
+${examplesHelp ? `\n  ${indent(examplesHelp, 4)}` : ''}
+`;
 }
 
 export function getCommandLevelHelp({
diff --git a/packages/kbn-dev-cli-runner/src/run.ts b/packages/kbn-dev-cli-runner/src/run.ts
index 08457caaebfd4..2ef90c2e2c27a 100644
--- a/packages/kbn-dev-cli-runner/src/run.ts
+++ b/packages/kbn-dev-cli-runner/src/run.ts
@@ -50,6 +50,7 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
     usage: options.usage,
     flagHelp: options.flags?.help,
     defaultLogLevel: options.log?.defaultLevel,
+    examples: options.flags?.examples,
   });
 
   if (flags.help) {
diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts
index 7ee4f08fb94fe..c8b3018e6f669 100644
--- a/packages/kbn-es/src/cli_commands/serverless.ts
+++ b/packages/kbn-es/src/cli_commands/serverless.ts
@@ -13,9 +13,8 @@ import { getTimeReporter } from '@kbn/ci-stats-reporter';
 
 import { Cluster } from '../cluster';
 import {
-  SERVERLESS_REPO,
-  SERVERLESS_TAG,
-  SERVERLESS_IMG,
+  ES_SERVERLESS_REPO_ELASTICSEARCH,
+  ES_SERVERLESS_DEFAULT_IMAGE,
   DEFAULT_PORT,
   ServerlessOptions,
 } from '../utils';
@@ -28,9 +27,8 @@ export const serverless: Command = {
     return dedent`
     Options:
 
-      --tag               Image tag of ES serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}]
-      --image             Full path of ES serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}]
-      
+      --tag               Image tag of ES serverless to run from ${ES_SERVERLESS_REPO_ELASTICSEARCH}
+      --image             Full path of ES serverless image to run, has precedence over tag. [default: ${ES_SERVERLESS_DEFAULT_IMAGE}]
       --background        Start ES serverless without attaching to the first node's logs
       --basePath          Path to the directory where the ES cluster will store data
       --clean             Remove existing file system object store before running
@@ -39,14 +37,14 @@ export const serverless: Command = {
       --ssl               Enable HTTP SSL on the ES cluster
       --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
-      
+
       -E                  Additional key=value settings to pass to ES
       -F                  Absolute paths for files to mount into containers
 
     Examples:
 
-      es serverless --tag git-fec36430fba2-x86_64
-      es serverless --image docker.elastic.co/repo:tag
+      es serverless --tag git-fec36430fba2-x86_64 # loads ${ES_SERVERLESS_REPO_ELASTICSEARCH}:git-fec36430fba2-x86_64
+      es serverless --image docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
     `;
   },
   run: async (defaults = {}) => {
diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts
index d48cddd6fdb6d..08edc2a17521d 100644
--- a/packages/kbn-es/src/utils/docker.test.ts
+++ b/packages/kbn-es/src/utils/docker.test.ts
@@ -23,7 +23,7 @@ import {
   runDockerContainer,
   runServerlessCluster,
   runServerlessEsNode,
-  SERVERLESS_IMG,
+  ES_SERVERLESS_DEFAULT_IMAGE,
   setupServerlessVolumes,
   stopServerlessCluster,
   teardownServerlessClusterSync,
@@ -451,7 +451,7 @@ describe('runServerlessEsNode()', () => {
   const node = {
     params: ['--env', 'foo=bar', '--volume', 'foo/bar'],
     name: 'es01',
-    image: SERVERLESS_IMG,
+    image: ES_SERVERLESS_DEFAULT_IMAGE,
   };
 
   test('should call the correct Docker command', async () => {
@@ -462,7 +462,7 @@ describe('runServerlessEsNode()', () => {
     expect(execa.mock.calls[0][0]).toEqual('docker');
     expect(execa.mock.calls[0][1]).toEqual(
       expect.arrayContaining([
-        SERVERLESS_IMG,
+        ES_SERVERLESS_DEFAULT_IMAGE,
         ...node.params,
         '--name',
         node.name,
@@ -530,7 +530,9 @@ describe('teardownServerlessClusterSync()', () => {
     teardownServerlessClusterSync(log, defaultOptions);
 
     expect(execa.commandSync.mock.calls).toHaveLength(2);
-    expect(execa.commandSync.mock.calls[0][0]).toEqual(expect.stringContaining(SERVERLESS_IMG));
+    expect(execa.commandSync.mock.calls[0][0]).toEqual(
+      expect.stringContaining(ES_SERVERLESS_DEFAULT_IMAGE)
+    );
     expect(execa.commandSync.mock.calls[1][0]).toEqual(`docker kill ${nodes.join(' ')}`);
   });
 
diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts
index 00a1d7ce9dc54..5ed22e094e6f8 100644
--- a/packages/kbn-es/src/utils/docker.ts
+++ b/packages/kbn-es/src/utils/docker.ts
@@ -38,9 +38,12 @@ import {
 import { SYSTEM_INDICES_SUPERUSER } from './native_realm';
 import { waitUntilClusterReady } from './wait_until_cluster_ready';
 
-interface BaseOptions {
-  tag?: string;
+interface ImageOptions {
   image?: string;
+  tag?: string;
+}
+
+interface BaseOptions extends ImageOptions {
   port?: number;
   ssl?: boolean;
   /** Kill running cluster before starting a new cluster  */
@@ -106,9 +109,10 @@ export const DOCKER_REPO = `${DOCKER_REGISTRY}/elasticsearch/elasticsearch`;
 export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`;
 export const DOCKER_IMG = `${DOCKER_REPO}:${DOCKER_TAG}`;
 
-export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`;
-export const SERVERLESS_TAG = 'latest';
-export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`;
+export const ES_SERVERLESS_REPO_KIBANA = `${DOCKER_REGISTRY}/kibana-ci/elasticsearch-serverless`;
+export const ES_SERVERLESS_REPO_ELASTICSEARCH = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`;
+export const ES_SERVERLESS_LATEST_VERIFIED_TAG = 'latest-verified';
+export const ES_SERVERLESS_DEFAULT_IMAGE = `${ES_SERVERLESS_REPO_KIBANA}:${ES_SERVERLESS_LATEST_VERIFIED_TAG}`;
 
 // See for default cluster settings
 // https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/kotlin/elasticsearch.serverless-run.gradle.kts
@@ -275,7 +279,12 @@ export function resolveDockerImage({
   image,
   repo,
   defaultImg,
-}: (ServerlessOptions | DockerOptions) & { repo: string; defaultImg: string }) {
+}: {
+  tag?: string;
+  image?: string;
+  repo: string;
+  defaultImg: string;
+}) {
   if (image) {
     if (!image.includes(DOCKER_REGISTRY)) {
       throw createCliError(
@@ -525,11 +534,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
 /**
  * Resolve the Serverless ES image based on defaults and CLI options
  */
-function getServerlessImage(options: ServerlessOptions) {
+function getServerlessImage({ image, tag }: ImageOptions) {
   return resolveDockerImage({
-    ...options,
-    repo: SERVERLESS_REPO,
-    defaultImg: SERVERLESS_IMG,
+    image,
+    tag,
+    repo: ES_SERVERLESS_REPO_ELASTICSEARCH,
+    defaultImg: ES_SERVERLESS_DEFAULT_IMAGE,
   });
 }
 
@@ -573,7 +583,10 @@ function getESClient(clientOptions: ClientOptions): Client {
  * Runs an ES Serverless Cluster through Docker
  */
 export async function runServerlessCluster(log: ToolingLog, options: ServerlessOptions) {
-  const image = getServerlessImage(options);
+  const image = getServerlessImage({
+    image: options.image,
+    tag: options.tag,
+  });
   await setupDocker({ log, image, options });
 
   const volumeCmd = await setupServerlessVolumes(log, options);
@@ -686,8 +699,13 @@ export function teardownServerlessClusterSync(log: ToolingLog, options: Serverle
 /**
  * Resolve the Elasticsearch image based on defaults and CLI options
  */
-function getDockerImage(options: DockerOptions) {
-  return resolveDockerImage({ ...options, repo: DOCKER_REPO, defaultImg: DOCKER_IMG });
+function getDockerImage({ image, tag }: ImageOptions) {
+  return resolveDockerImage({
+    image,
+    tag,
+    repo: DOCKER_REPO,
+    defaultImg: DOCKER_IMG,
+  });
 }
 
 /**
@@ -713,7 +731,10 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions
   let image;
 
   if (!options.dockerCmd) {
-    image = getDockerImage(options);
+    image = getDockerImage({
+      image: options.image,
+      tag: options.tag,
+    });
     await setupDocker({ log, image, options });
   }
 
diff --git a/packages/kbn-test/src/es/es_test_config.ts b/packages/kbn-test/src/es/es_test_config.ts
index c31728b0fd7d8..26827558b7bd3 100644
--- a/packages/kbn-test/src/es/es_test_config.ts
+++ b/packages/kbn-test/src/es/es_test_config.ts
@@ -27,6 +27,10 @@ class EsTestConfig {
     return process.env.TEST_ES_FROM || 'snapshot';
   }
 
+  getESServerlessImage() {
+    return process.env.TEST_ES_SERVERLESS_IMAGE;
+  }
+
   getTransportPort() {
     return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
   }
diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts
index 3cd90d21d1d91..3c63960bdc0e5 100644
--- a/packages/kbn-test/src/es/test_es_cluster.ts
+++ b/packages/kbn-test/src/es/test_es_cluster.ts
@@ -70,6 +70,10 @@ export interface CreateTestEsClusterOptions {
    */
   esArgs?: string[];
   esFrom?: string;
+  esServerlessOptions?: {
+    image?: string;
+    tag?: string;
+  };
   esJavaOpts?: string;
   /**
    * License to run your cluster under. Keep in mind that a `trial` license
@@ -164,6 +168,7 @@ export function createTestEsCluster<
     writeLogsToPath,
     basePath = Path.resolve(REPO_ROOT, '.es'),
     esFrom = esTestConfig.getBuildFrom(),
+    esServerlessOptions,
     dataArchive,
     nodes = [{ name: 'node-01' }],
     esArgs: customEsArgs = [],
@@ -236,9 +241,11 @@ export function createTestEsCluster<
       } else if (esFrom === 'snapshot') {
         installPath = (await firstNode.installSnapshot(config)).installPath;
       } else if (esFrom === 'serverless') {
-        return await firstNode.runServerless({
+        await firstNode.runServerless({
           basePath,
           esArgs: customEsArgs,
+          image: esServerlessOptions?.image,
+          tag: esServerlessOptions?.tag,
           port,
           clean: true,
           background: true,
@@ -247,6 +254,7 @@ export function createTestEsCluster<
           kill: true, // likely don't need this but avoids any issues where the ESS cluster wasn't cleaned up
           waitForReady: true,
         });
+        return;
       } else if (Path.isAbsolute(esFrom)) {
         installPath = esFrom;
       } else {
@@ -275,9 +283,9 @@ export function createTestEsCluster<
           });
         }
 
-        nodeStartPromises.push(async () => {
+        nodeStartPromises.push(() => {
           log.info(`[es] starting node ${node.name} on port ${nodePort}`);
-          return await this.nodes[i].start(installPath, {
+          return this.nodes[i].start(installPath, {
             password: config.password,
             esArgs: assignArgs(esArgs, overriddenArgs),
             esJavaOpts,
@@ -292,7 +300,7 @@ export function createTestEsCluster<
         });
       }
 
-      await Promise.all(extractDirectoryPromises.map(async (extract) => await extract()));
+      await Promise.all(extractDirectoryPromises.map((extract) => extract()));
       for (const start of nodeStartPromises) {
         await start();
       }
diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts
index f9c83161b521b..d298a1c1abaa4 100644
--- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts
+++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts
@@ -12,11 +12,12 @@ import getPort from 'get-port';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { ArtifactLicense } from '@kbn/es';
 import type { Config } from '../../functional_test_runner';
-import { createTestEsCluster } from '../../es';
+import { createTestEsCluster, esTestConfig } from '../../es';
 
 interface RunElasticsearchOptions {
   log: ToolingLog;
   esFrom?: string;
+  esServerlessImage?: string;
   config: Config;
   onEarlyExit?: (msg: string) => void;
   logsDir?: string;
@@ -32,6 +33,7 @@ type EsConfig = ReturnType<typeof getEsConfig>;
 function getEsConfig({
   config,
   esFrom = config.get('esTestCluster.from'),
+  esServerlessImage,
 }: RunElasticsearchOptions) {
   const ssl = !!config.get('esTestCluster.ssl');
   const license: ArtifactLicense = config.get('esTestCluster.license');
@@ -50,6 +52,8 @@ function getEsConfig({
   const serverless: boolean = config.get('serverless');
   const files: string[] | undefined = config.get('esTestCluster.files');
 
+  const esServerlessOptions = getESServerlessOptions(esServerlessImage, config);
+
   return {
     ssl,
     license,
@@ -57,6 +61,7 @@ function getEsConfig({
     esJavaOpts,
     isSecurityEnabled,
     esFrom,
+    esServerlessOptions,
     port,
     password,
     dataArchive,
@@ -129,6 +134,7 @@ async function startEsNode({
     clusterName: `cluster-${name}`,
     esArgs: config.esArgs,
     esFrom: config.esFrom,
+    esServerlessOptions: config.esServerlessOptions,
     esJavaOpts: config.esJavaOpts,
     license: config.license,
     password: config.password,
@@ -153,3 +159,23 @@ async function startEsNode({
 
   return cluster;
 }
+
+function getESServerlessOptions(esServerlessImageFromArg: string | undefined, config: Config) {
+  const esServerlessImageUrlOrTag =
+    esServerlessImageFromArg ||
+    esTestConfig.getESServerlessImage() ||
+    (config.has('esTestCluster.esServerlessImage') &&
+      config.get('esTestCluster.esServerlessImage'));
+
+  if (esServerlessImageUrlOrTag) {
+    if (esServerlessImageUrlOrTag.includes(':')) {
+      return {
+        image: esServerlessImageUrlOrTag,
+      };
+    } else {
+      return {
+        tag: esServerlessImageUrlOrTag,
+      };
+    }
+  }
+}
diff --git a/packages/kbn-test/src/functional_tests/run_tests/cli.ts b/packages/kbn-test/src/functional_tests/run_tests/cli.ts
index 19a003dd973cf..40a711ca9f6c2 100644
--- a/packages/kbn-test/src/functional_tests/run_tests/cli.ts
+++ b/packages/kbn-test/src/functional_tests/run_tests/cli.ts
@@ -26,6 +26,7 @@ export function runTestsCli() {
     {
       description: `Run Functional Tests`,
       usage: `
+      Usage:
         node scripts/functional_tests --help
         node scripts/functional_tests [--config <file1> [--config <file2> ...]]
         node scripts/functional_tests [options] [-- --<other args>]
diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts
index 0c9dae3a25794..77399605b29f4 100644
--- a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts
+++ b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts
@@ -42,6 +42,7 @@ describe('parse runTest flags', () => {
         ],
         "dryRun": false,
         "esFrom": undefined,
+        "esServerlessImage": undefined,
         "esVersion": <EsVersion 9.9.9>,
         "grep": undefined,
         "installDir": undefined,
diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.ts
index f4dd6beb26e80..3ba86999d3802 100644
--- a/packages/kbn-test/src/functional_tests/run_tests/flags.ts
+++ b/packages/kbn-test/src/functional_tests/run_tests/flags.ts
@@ -23,6 +23,7 @@ export const FLAG_OPTIONS: FlagOptions = {
     'config',
     'journey',
     'esFrom',
+    'esServerlessImage',
     'kibana-install-dir',
     'grep',
     'include-tag',
@@ -37,6 +38,7 @@ export const FLAG_OPTIONS: FlagOptions = {
     --config             Define a FTR config that should be executed. Can be specified multiple times
     --journey            Define a Journey that should be executed. Can be specified multiple times
     --esFrom             Build Elasticsearch from source or run snapshot or serverless. Default: $TEST_ES_FROM or "snapshot"
+    --esServerlessImage  When 'esFrom' is "serverless", this argument will be interpreted either as a tag within the ES Serverless repo, OR a full docker image path.
     --include-tag        Tags that suites must include to be run, can be included multiple times
     --exclude-tag        Tags that suites must NOT include to be run, can be included multiple times
     --include            Files that must included to be run, can be included multiple times
@@ -50,6 +52,13 @@ export const FLAG_OPTIONS: FlagOptions = {
     --updateSnapshots    Replace inline and file snapshots with whatever is generated from the test
     --updateAll, -u      Replace both baseline screenshots and snapshots
   `,
+  examples: `
+Run the latest verified, kibana-compatible ES Serverless image:
+  node scripts/functional_tests --config ./config.ts --esFrom serverless --esServerlessImage docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
+
+Run with a specific ES Serverless tag from the docker.elastic.co/elasticsearch-ci/elasticsearch-serverless repo:
+  node scripts/functional_tests --config ./config.ts --esFrom serverless --esServerlessImage git-fec36430fba2
+  `,
 };
 
 export function parseFlags(flags: FlagsReader) {
@@ -75,6 +84,7 @@ export function parseFlags(flags: FlagsReader) {
       ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4())
       : undefined,
     esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']),
+    esServerlessImage: flags.string('esServerlessImage'),
     installDir: flags.path('kibana-install-dir'),
     grep: flags.string('grep'),
     suiteTags: {

From 5b24da796d3a992cd0e3eeb0262bd37564ba995b Mon Sep 17 00:00:00 2001
From: Kevin Delemme <kevin.delemme@elastic.co>
Date: Mon, 25 Sep 2023 13:09:03 -0400
Subject: [PATCH 20/27] fix(slo): date range filter format (#166989)

---
 .../apm_transaction_duration.test.ts.snap        | 16 ++++++++--------
 .../apm_transaction_error_rate.test.ts.snap      | 16 ++++++++--------
 .../__snapshots__/histogram.test.ts.snap         |  6 +++---
 .../__snapshots__/kql_custom.test.ts.snap        |  6 +++---
 .../__snapshots__/metric_custom.test.ts.snap     |  6 +++---
 .../apm_transaction_duration.ts                  |  2 +-
 .../apm_transaction_error_rate.ts                |  2 +-
 .../slo/transform_generators/histogram.ts        |  2 +-
 .../slo/transform_generators/kql_custom.ts       |  2 +-
 .../slo/transform_generators/metric_custom.ts    |  2 +-
 10 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap
index 5f828ee51b625..aea9ba75f2863 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap
@@ -24,7 +24,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -130,7 +130,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -163,7 +163,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -277,7 +277,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -391,7 +391,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -505,7 +505,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -765,7 +765,7 @@ Object {
           Object {
             "range": Object {
               "@timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
@@ -1039,7 +1039,7 @@ Object {
           Object {
             "range": Object {
               "@timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap
index dc9278511a864..b7c4ebf0f6b8c 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap
@@ -20,7 +20,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -122,7 +122,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -151,7 +151,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -261,7 +261,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -371,7 +371,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -481,7 +481,7 @@ Object {
       Object {
         "range": Object {
           "@timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -730,7 +730,7 @@ Object {
           Object {
             "range": Object {
               "@timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
@@ -993,7 +993,7 @@ Object {
           Object {
             "range": Object {
               "@timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap
index ee4001303fec5..ad100c4662fe9 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap
@@ -51,7 +51,7 @@ Object {
       Object {
         "range": Object {
           "log_timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -232,7 +232,7 @@ Object {
           Object {
             "range": Object {
               "log_timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
@@ -488,7 +488,7 @@ Object {
           Object {
             "range": Object {
               "log_timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap
index 126437173f84a..3297a8c513821 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap
@@ -92,7 +92,7 @@ Object {
       Object {
         "range": Object {
           "log_timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -247,7 +247,7 @@ Object {
           Object {
             "range": Object {
               "log_timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
@@ -477,7 +477,7 @@ Object {
           Object {
             "range": Object {
               "log_timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap
index ced0801d859d4..362b1d4a9e01e 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap
@@ -63,7 +63,7 @@ Object {
       Object {
         "range": Object {
           "log_timestamp": Object {
-            "gte": "now-7d",
+            "gte": "now-7d/d",
           },
         },
       },
@@ -256,7 +256,7 @@ Object {
           Object {
             "range": Object {
               "log_timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
@@ -524,7 +524,7 @@ Object {
           Object {
             "range": Object {
               "log_timestamp": Object {
-                "gte": "now-7d",
+                "gte": "now-7d/d",
               },
             },
           },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts
index 3dd6469a7f2c5..171a5ca15e2de 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts
@@ -72,7 +72,7 @@ export class ApmTransactionDurationTransformGenerator extends TransformGenerator
       {
         range: {
           '@timestamp': {
-            gte: `now-${slo.timeWindow.duration.format()}`,
+            gte: `now-${slo.timeWindow.duration.format()}/d`,
           },
         },
       },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts
index 3818111c70df2..d038876f5ee9d 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts
@@ -72,7 +72,7 @@ export class ApmTransactionErrorRateTransformGenerator extends TransformGenerato
       {
         range: {
           '@timestamp': {
-            gte: `now-${slo.timeWindow.duration.format()}`,
+            gte: `now-${slo.timeWindow.duration.format()}/d`,
           },
         },
       },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts
index 654f8d67a3673..844703003257a 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts
@@ -54,7 +54,7 @@ export class HistogramTransformGenerator extends TransformGenerator {
             {
               range: {
                 [indicator.params.timestampField]: {
-                  gte: `now-${slo.timeWindow.duration.format()}`,
+                  gte: `now-${slo.timeWindow.duration.format()}/d`,
                 },
               },
             },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts
index 8321a0cb7172e..02c7757ec1362 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts
@@ -49,7 +49,7 @@ export class KQLCustomTransformGenerator extends TransformGenerator {
             {
               range: {
                 [indicator.params.timestampField]: {
-                  gte: `now-${slo.timeWindow.duration.format()}`,
+                  gte: `now-${slo.timeWindow.duration.format()}/d`,
                 },
               },
             },
diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts
index 52209533f828c..063e6fbe1e3dc 100644
--- a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts
+++ b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts
@@ -52,7 +52,7 @@ export class MetricCustomTransformGenerator extends TransformGenerator {
             {
               range: {
                 [indicator.params.timestampField]: {
-                  gte: `now-${slo.timeWindow.duration.format()}`,
+                  gte: `now-${slo.timeWindow.duration.format()}/d`,
                 },
               },
             },

From 93ce98831a900cc7f7cd1e91ac6f51bc57a67006 Mon Sep 17 00:00:00 2001
From: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
Date: Mon, 25 Sep 2023 20:38:22 +0300
Subject: [PATCH 21/27] [ES|QL] Hides system indices (#166909)

## Summary

Closes https://github.com/elastic/kibana/issues/166874

Hides indices starting with . (and considered as system from the
autocomplete)

<img width="785" alt="image"
src="https://github.com/elastic/kibana/assets/17003240/9c4cce79-c844-41b6-a30e-06dad49f7c52">


Followed the exact pattern that the dataview management page is using.
---
 .../kbn-text-based-editor/src/helpers.test.ts | 31 +++++++++++++++++--
 packages/kbn-text-based-editor/src/helpers.ts | 10 ++++++
 .../src/text_based_languages_editor.tsx       |  8 ++---
 3 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/packages/kbn-text-based-editor/src/helpers.test.ts b/packages/kbn-text-based-editor/src/helpers.test.ts
index 5f1546ccc138e..f7b100419b2c7 100644
--- a/packages/kbn-text-based-editor/src/helpers.test.ts
+++ b/packages/kbn-text-based-editor/src/helpers.test.ts
@@ -5,8 +5,14 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
-import { parseErrors, parseWarning, getInlineEditorText, getWrappedInPipesCode } from './helpers';
+import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
+import {
+  parseErrors,
+  parseWarning,
+  getInlineEditorText,
+  getWrappedInPipesCode,
+  getIndicesForAutocomplete,
+} from './helpers';
 
 describe('helpers', function () {
   describe('parseErrors', function () {
@@ -159,4 +165,25 @@ describe('helpers', function () {
       expect(code).toEqual('FROM index1 | keep field1, field2 | order field1');
     });
   });
+
+  describe('getIndicesForAutocomplete', function () {
+    it('should not return system indices', async function () {
+      const dataViewsMock = dataViewPluginMocks.createStartContract();
+      const updatedDataViewsMock = {
+        ...dataViewsMock,
+        getIndices: jest.fn().mockResolvedValue([
+          {
+            name: '.system1',
+            title: 'system1',
+          },
+          {
+            name: 'logs',
+            title: 'logs',
+          },
+        ]),
+      };
+      const indices = await getIndicesForAutocomplete(updatedDataViewsMock);
+      expect(indices).toStrictEqual(['logs']);
+    });
+  });
 });
diff --git a/packages/kbn-text-based-editor/src/helpers.ts b/packages/kbn-text-based-editor/src/helpers.ts
index 6b4e99f6e3093..8932276695117 100644
--- a/packages/kbn-text-based-editor/src/helpers.ts
+++ b/packages/kbn-text-based-editor/src/helpers.ts
@@ -10,6 +10,7 @@ import { useRef } from 'react';
 import useDebounce from 'react-use/lib/useDebounce';
 import { monaco } from '@kbn/monaco';
 import { i18n } from '@kbn/i18n';
+import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 
 export interface MonacoError {
   message: string;
@@ -172,3 +173,12 @@ export const getWrappedInPipesCode = (code: string, isWrapped: boolean): string
   });
   return codeNoLines.join(isWrapped ? ' | ' : '\n| ');
 };
+
+export const getIndicesForAutocomplete = async (dataViews: DataViewsPublicPluginStart) => {
+  const indices = await dataViews.getIndices({
+    showAllIndices: false,
+    pattern: '*',
+    isRollupIndex: () => false,
+  });
+  return indices.filter((index) => !index.name.startsWith('.')).map((i) => i.name);
+};
diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx
index fc86f7ca1cac6..b1f1708262743 100644
--- a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx
+++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx
@@ -56,6 +56,7 @@ import {
   getDocumentationSections,
   MonacoError,
   getWrappedInPipesCode,
+  getIndicesForAutocomplete,
 } from './helpers';
 import { EditorFooter } from './editor_footer';
 import { ResizableButton } from './resizable_button';
@@ -371,12 +372,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
 
   const getSourceIdentifiers: ESQLCustomAutocompleteCallbacks['getSourceIdentifiers'] =
     useCallback(async () => {
-      const indices = await dataViews.getIndices({
-        showAllIndices: false,
-        pattern: '*',
-        isRollupIndex: () => false,
-      });
-      return indices.map((i) => i.name);
+      return await getIndicesForAutocomplete(dataViews);
     }, [dataViews]);
 
   const getFieldsIdentifiers: ESQLCustomAutocompleteCallbacks['getFieldsIdentifiers'] = useCallback(

From 3f03264dc024a0e2abdd610c774048df861d6c38 Mon Sep 17 00:00:00 2001
From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com>
Date: Mon, 25 Sep 2023 13:46:35 -0400
Subject: [PATCH 22/27] [Security Solution][Endpoint] Refactor Cypress `login`
 task and ensure consistent use of users across ESS and Serverless tests
 (#166958)

## Summary

- Cypress `login` task refactored:
- `login(user?)` : logs use in using the default `user` or one of the
users supported by security solution and endpoint management tests
- `login.with(username, password)` : Logs a user in by using `username`
and `password`
- `login.withCustomRole(role)` : creates the provided `role`, creates a
user for it by the same role name and logs in with it
- The Cypress process for loading users into Kibana only applies to
non-serverless (at the moment). For serverless, it only validates that
the `username` being used is one of the approved user names that applies
to serverless
- FYI: the creation/availability of serverless roles/users for testing
is an ongoing effort by the kibana ops team
- New generic `RoleAndUserLoader` class. Is initialized with an map of
`Roles` and provide a standard interface for loading them.
- A sub-class (`EndpointSecurityTestRolesLoader`) was also created for
the endpoint security test users, which uses the existing set of role
definitions
- The `resolver_generator_script` was also updated to use the new
`EndpointSecurityTestRolesLoader` class for handling the `--rbacUser`
argument

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
---
 .../management/cypress/common/constants.ts    |  21 ++
 .../public/management/cypress/cypress.d.ts    |  20 +-
 .../artifact_tabs_in_policy_details.cy.ts     |  15 +-
 .../e2e/artifacts/artifacts_mocked_data.cy.ts |  15 +-
 .../e2e/automated_response_actions/form.cy.ts |  12 +-
 .../no_license.cy.ts                          |   4 +-
 ...dpoint_list_with_security_essentials.cy.ts |   4 +-
 .../serverless/feature_access/complete.cy.ts  |   4 +-
 .../complete_with_endpoint.cy.ts              |   4 +-
 .../feature_access/essentials.cy.ts           |   4 +-
 .../essentials_with_endpoint.cy.ts            |   4 +-
 ...icy_details_with_security_essentials.cy.ts |   4 +-
 .../roles/complete_with_endpoint_roles.cy.ts  |  20 +-
 .../essentials_with_endpoint.roles.cy.ts      |  22 +-
 .../role_with_artifact_read_privilege.ts      |  30 ++
 .../cypress/support/data_loaders.ts           | 116 ++++++-
 .../public/management/cypress/tasks/login.ts  | 295 +++++-------------
 .../cypress/tasks/login_serverless.ts         | 103 ------
 .../public/management/cypress/tsconfig.json   |   1 +
 .../public/management/cypress/types.ts        |  10 +
 .../scripts/endpoint/common/constants.ts      |   5 +
 .../endpoint/common/role_and_user_loader.ts   | 175 +++++++++++
 .../endpoint/common/roles_users/index.ts      | 139 +++++++++
 .../common/roles_users/rule_author.ts         |  36 +++
 .../endpoint/common/roles_users/t3_analyst.ts |  39 +++
 .../endpoint/resolver_generator_script.ts     |  73 +----
 26 files changed, 714 insertions(+), 461 deletions(-)
 create mode 100644 x-pack/plugins/security_solution/public/management/cypress/common/constants.ts
 create mode 100644 x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts
 delete mode 100644 x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts
 create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts
 create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts
 create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts
 create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts

diff --git a/x-pack/plugins/security_solution/public/management/cypress/common/constants.ts b/x-pack/plugins/security_solution/public/management/cypress/common/constants.ts
new file mode 100644
index 0000000000000..41f08f438e3f8
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/management/cypress/common/constants.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EndpointSecurityRoleNames } from '../../../../scripts/endpoint/common/roles_users';
+
+export type KibanaKnownUserAccounts = keyof typeof KIBANA_KNOWN_DEFAULT_ACCOUNTS;
+
+export type SecurityTestUser = EndpointSecurityRoleNames | KibanaKnownUserAccounts;
+
+/**
+ * List of kibana system accounts
+ */
+export const KIBANA_KNOWN_DEFAULT_ACCOUNTS = {
+  elastic: 'elastic',
+  elastic_serverless: 'elastic_serverless',
+  system_indices_superuser: 'system_indices_superuser',
+} as const;
diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts
index 4d84dc88af9c9..a4037e32632a5 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts
@@ -17,7 +17,12 @@ import type {
   HostPolicyResponse,
   LogsEndpointActionResponse,
 } from '../../../common/endpoint/types';
-import type { IndexEndpointHostsCyTaskOptions, HostActionResponse } from './types';
+import type {
+  HostActionResponse,
+  IndexEndpointHostsCyTaskOptions,
+  LoadUserAndRoleCyTaskOptions,
+  CreateUserAndRoleCyTaskOptions,
+} from './types';
 import type {
   DeleteIndexedFleetEndpointPoliciesResponse,
   IndexedFleetEndpointPolicyResponse,
@@ -32,6 +37,7 @@ import type {
   DeletedIndexedEndpointRuleAlerts,
   IndexedEndpointRuleAlerts,
 } from '../../../common/endpoint/data_loaders/index_endpoint_rule_alerts';
+import type { LoadedRoleAndUser } from '../../../scripts/endpoint/common/role_and_user_loader';
 
 declare global {
   namespace Cypress {
@@ -185,6 +191,18 @@ declare global {
         arg: { hostname: string; path: string; password?: string },
         options?: Partial<Loggable & Timeoutable>
       ): Chainable<string>;
+
+      task(
+        name: 'loadUserAndRole',
+        arg: LoadUserAndRoleCyTaskOptions,
+        options?: Partial<Loggable & Timeoutable>
+      ): Chainable<LoadedRoleAndUser>;
+
+      task(
+        name: 'createUserAndRole',
+        arg: CreateUserAndRoleCyTaskOptions,
+        options?: Partial<Loggable & Timeoutable>
+      ): Chainable<LoadedRoleAndUser>;
     }
   }
 }
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts
index 39ebae1825365..8610ac1fc3ba5 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { getRoleWithArtifactReadPrivilege } from '../../fixtures/role_with_artifact_read_privilege';
 import { getEndpointSecurityPolicyManager } from '../../../../../scripts/endpoint/common/roles_users/endpoint_security_policy_manager';
 import { getArtifactsListTestsData } from '../../fixtures/artifacts_page';
 import { visitPolicyDetailsPage } from '../../screens/policy_details';
@@ -16,27 +17,21 @@ import {
   yieldFirstPolicyID,
 } from '../../tasks/artifacts';
 import { loadEndpointDataForEventFiltersIfNeeded } from '../../tasks/load_endpoint_data';
-import {
-  getRoleWithArtifactReadPrivilege,
-  login,
-  loginWithCustomRole,
-  loginWithRole,
-  ROLE,
-} from '../../tasks/login';
+import { login, ROLE } from '../../tasks/login';
 import { performUserActions } from '../../tasks/perform_user_actions';
 
 const loginWithPrivilegeAll = () => {
-  loginWithRole(ROLE.endpoint_security_policy_manager);
+  login(ROLE.endpoint_policy_manager);
 };
 
 const loginWithPrivilegeRead = (privilegePrefix: string) => {
   const roleWithArtifactReadPrivilege = getRoleWithArtifactReadPrivilege(privilegePrefix);
-  loginWithCustomRole('roleWithArtifactReadPrivilege', roleWithArtifactReadPrivilege);
+  login.withCustomRole({ name: 'roleWithArtifactReadPrivilege', ...roleWithArtifactReadPrivilege });
 };
 
 const loginWithPrivilegeNone = (privilegePrefix: string) => {
   const roleWithoutArtifactPrivilege = getRoleWithoutArtifactPrivilege(privilegePrefix);
-  loginWithCustomRole('roleWithoutArtifactPrivilege', roleWithoutArtifactPrivilege);
+  login.withCustomRole({ name: 'roleWithoutArtifactPrivilege', ...roleWithoutArtifactPrivilege });
 };
 
 const getRoleWithoutArtifactPrivilege = (privilegePrefix: string) => {
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts
index 8f575282ad3f7..86cd86dd797b7 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts
@@ -5,13 +5,8 @@
  * 2.0.
  */
 
-import {
-  getRoleWithArtifactReadPrivilege,
-  login,
-  loginWithCustomRole,
-  loginWithRole,
-  ROLE,
-} from '../../tasks/login';
+import { getRoleWithArtifactReadPrivilege } from '../../fixtures/role_with_artifact_read_privilege';
+import { login, ROLE } from '../../tasks/login';
 import { loadPage } from '../../tasks/common';
 
 import { getArtifactsListTestsData } from '../../fixtures/artifacts_page';
@@ -20,18 +15,18 @@ import { performUserActions } from '../../tasks/perform_user_actions';
 import { loadEndpointDataForEventFiltersIfNeeded } from '../../tasks/load_endpoint_data';
 
 const loginWithWriteAccess = (url: string) => {
-  loginWithRole(ROLE.endpoint_security_policy_manager);
+  login(ROLE.endpoint_policy_manager);
   loadPage(url);
 };
 
 const loginWithReadAccess = (privilegePrefix: string, url: string) => {
   const roleWithArtifactReadPrivilege = getRoleWithArtifactReadPrivilege(privilegePrefix);
-  loginWithCustomRole('roleWithArtifactReadPrivilege', roleWithArtifactReadPrivilege);
+  login.withCustomRole({ name: 'roleWithArtifactReadPrivilege', ...roleWithArtifactReadPrivilege });
   loadPage(url);
 };
 
 const loginWithoutAccess = (url: string) => {
-  loginWithRole(ROLE.t1_analyst);
+  login(ROLE.t1_analyst);
   loadPage(url);
 };
 
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts
index 84f6903c35a9b..eb5bae8624475 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts
@@ -16,12 +16,12 @@ import {
 } from '../../tasks/response_actions';
 import { cleanupRule, generateRandomStringName, loadRule } from '../../tasks/api_fixtures';
 import { RESPONSE_ACTION_TYPES } from '../../../../../common/api/detection_engine';
-import { loginWithRole, ROLE } from '../../tasks/login';
+import { login, ROLE } from '../../tasks/login';
 
 describe('Form', { tags: '@ess' }, () => {
   describe('User with no access can not create an endpoint response action', () => {
     before(() => {
-      loginWithRole(ROLE.endpoint_response_actions_no_access);
+      login(ROLE.endpoint_response_actions_no_access);
     });
 
     it('no endpoint response action option during rule creation', () => {
@@ -36,7 +36,7 @@ describe('Form', { tags: '@ess' }, () => {
     const [ruleName, ruleDescription] = generateRandomStringName(2);
 
     before(() => {
-      loginWithRole(ROLE.endpoint_response_actions_access);
+      login(ROLE.endpoint_response_actions_access);
     });
     after(() => {
       cleanupRule(ruleId);
@@ -94,7 +94,7 @@ describe('Form', { tags: '@ess' }, () => {
       });
     });
     beforeEach(() => {
-      loginWithRole(ROLE.endpoint_response_actions_access);
+      login(ROLE.endpoint_response_actions_access);
     });
     after(() => {
       cleanupRule(ruleId);
@@ -146,7 +146,7 @@ describe('Form', { tags: '@ess' }, () => {
     const [ruleName, ruleDescription] = generateRandomStringName(2);
 
     before(() => {
-      loginWithRole(ROLE.endpoint_response_actions_no_access);
+      login(ROLE.endpoint_response_actions_no_access);
     });
 
     it('response actions are disabled', () => {
@@ -166,7 +166,7 @@ describe('Form', { tags: '@ess' }, () => {
       loadRule().then((res) => {
         ruleId = res.id;
       });
-      loginWithRole(ROLE.endpoint_response_actions_no_access);
+      login(ROLE.endpoint_response_actions_no_access);
     });
     after(() => {
       cleanupRule(ruleId);
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts
index edbaa90d3200c..eb6191ece0460 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts
@@ -9,7 +9,7 @@ import { disableExpandableFlyoutAdvancedSettings } from '../../tasks/common';
 import { APP_ALERTS_PATH } from '../../../../../common/constants';
 import { closeAllToasts } from '../../tasks/toasts';
 import { fillUpNewRule } from '../../tasks/response_actions';
-import { login, loginWithRole, ROLE } from '../../tasks/login';
+import { login, ROLE } from '../../tasks/login';
 import { generateRandomStringName } from '../../tasks/utils';
 import type { ReturnTypeFromChainable } from '../../types';
 import { indexEndpointHosts } from '../../tasks/index_endpoint_hosts';
@@ -20,7 +20,7 @@ describe('No License', { tags: '@ess', env: { ftrConfig: { license: 'basic' } }
     const [ruleName, ruleDescription] = generateRandomStringName(2);
 
     before(() => {
-      loginWithRole(ROLE.endpoint_response_actions_access);
+      login(ROLE.endpoint_response_actions_access);
     });
 
     it('response actions are disabled', () => {
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts
index 58c41539361c5..32800978a968e 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts
@@ -7,7 +7,7 @@
 
 import type { CyIndexEndpointHosts } from '../../tasks/index_endpoint_hosts';
 import { indexEndpointHosts } from '../../tasks/index_endpoint_hosts';
-import { loginServerless } from '../../tasks/login_serverless';
+import { login } from '../../tasks/login';
 import {
   getConsoleActionMenuItem,
   getUnIsolateActionMenuItem,
@@ -42,7 +42,7 @@ describe(
       });
 
       beforeEach(() => {
-        loginServerless();
+        login();
         visitEndpointList();
         openRowActionMenu();
       });
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts
index 7c4323b2aa689..dba7166b9a9e8 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts
@@ -6,7 +6,7 @@
  */
 
 import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions';
-import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless';
+import { login, ROLE } from '../../../tasks/login';
 import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants';
 import { getNoPrivilegesPage } from '../../../screens/common';
 import { getEndpointManagementPageList } from '../../../screens';
@@ -31,7 +31,7 @@ describe(
     let password: string;
 
     beforeEach(() => {
-      loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => {
+      login(ROLE.endpoint_operations_analyst).then((response) => {
         username = response.username;
         password = response.password;
       });
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts
index f3ae8b85a9f24..3c028b9e25040 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts
@@ -6,7 +6,7 @@
  */
 
 import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions';
-import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless';
+import { login, ROLE } from '../../../tasks/login';
 import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants';
 import {
   getEndpointManagementPageList,
@@ -33,7 +33,7 @@ describe(
     let password: string;
 
     beforeEach(() => {
-      loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => {
+      login(ROLE.endpoint_operations_analyst).then((response) => {
         username = response.username;
         password = response.password;
       });
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts
index 900a5d81a9f46..fed4494722df5 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts
@@ -6,7 +6,7 @@
  */
 
 import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions';
-import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless';
+import { login, ROLE } from '../../../tasks/login';
 import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants';
 import { getNoPrivilegesPage } from '../../../screens/common';
 import { getEndpointManagementPageList } from '../../../screens';
@@ -33,7 +33,7 @@ describe(
     let password: string;
 
     beforeEach(() => {
-      loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => {
+      login(ROLE.endpoint_operations_analyst).then((response) => {
         username = response.username;
         password = response.password;
       });
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts
index 7196b73f6813a..172f850e44b7c 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts
@@ -6,7 +6,7 @@
  */
 
 import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions';
-import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless';
+import { login, ROLE } from '../../../tasks/login';
 import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants';
 import {
   getEndpointManagementPageMap,
@@ -41,7 +41,7 @@ describe(
     let password: string;
 
     beforeEach(() => {
-      loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => {
+      login(ROLE.endpoint_operations_analyst).then((response) => {
         username = response.username;
         password = response.password;
       });
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts
index faf9aba6237b3..e1516bb08e10e 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { loginServerless } from '../../tasks/login_serverless';
+import { login } from '../../tasks/login';
 import { visitPolicyDetailsPage } from '../../screens/policy_details';
 import type { IndexedFleetEndpointPolicyResponse } from '../../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
 
@@ -35,7 +35,7 @@ describe(
     });
 
     beforeEach(() => {
-      loginServerless();
+      login();
       visitPolicyDetailsPage(loadedPolicyData.integrationPolicies[0].id);
     });
 
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts
index 879948a65c47d..534b681c2aeb4 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts
@@ -8,7 +8,7 @@
 import { pick } from 'lodash';
 import type { CyIndexEndpointHosts } from '../../../tasks/index_endpoint_hosts';
 import { indexEndpointHosts } from '../../../tasks/index_endpoint_hosts';
-import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless';
+import { login, ROLE } from '../../../tasks/login';
 import { ensurePolicyDetailsPageAuthzAccess } from '../../../screens/policy_details';
 import type { EndpointArtifactPageId } from '../../../screens';
 import {
@@ -63,12 +63,12 @@ describe(
     });
 
     // roles `t1_analyst` and `t2_analyst` are very similar with exception of one page
-    (['t1_analyst', `t2_analyst`] as ServerlessUser[]).forEach((roleName) => {
+    (['t1_analyst', `t2_analyst`] as ROLE[]).forEach((roleName) => {
       describe(`for role: ${roleName}`, () => {
         const deniedPages = allPages.filter((page) => page.id !== 'endpointList');
 
         beforeEach(() => {
-          loginServerless(roleName);
+          login(roleName);
         });
 
         it('should have READ access to Endpoint list page', () => {
@@ -124,7 +124,7 @@ describe(
       const deniedResponseActions = pick(consoleHelpPanelResponseActionsTestSubj, 'execute');
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.T3_ANALYST);
+        login(ROLE.t3_analyst);
       });
 
       it('should have access to Endpoint list page', () => {
@@ -176,7 +176,7 @@ describe(
       const deniedPages = allPages.filter(({ id }) => id !== 'blocklist' && id !== 'endpointList');
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.THREAT_INTELLIGENCE_ANALYST);
+        login(ROLE.threat_intelligence_analyst);
       });
 
       it('should have access to Endpoint list page', () => {
@@ -221,7 +221,7 @@ describe(
       ];
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.RULE_AUTHOR);
+        login(ROLE.rule_author);
       });
 
       for (const { id, title } of artifactPagesFullAccess) {
@@ -272,7 +272,7 @@ describe(
       const grantedAccessPages = [pageById.endpointList, pageById.policyList];
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.SOC_MANAGER);
+        login(ROLE.soc_manager);
       });
 
       for (const { id, title } of artifactPagesFullAccess) {
@@ -319,7 +319,7 @@ describe(
       const grantedAccessPages = [pageById.endpointList, pageById.policyList];
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST);
+        login(ROLE.endpoint_operations_analyst);
       });
 
       for (const { id, title } of artifactPagesFullAccess) {
@@ -350,7 +350,7 @@ describe(
       });
     });
 
-    (['platform_engineer', 'endpoint_policy_manager'] as ServerlessUser[]).forEach((roleName) => {
+    (['platform_engineer', 'endpoint_policy_manager'] as ROLE[]).forEach((roleName) => {
       describe(`for role: ${roleName}`, () => {
         const artifactPagesFullAccess = [
           pageById.trustedApps,
@@ -361,7 +361,7 @@ describe(
         const grantedAccessPages = [pageById.endpointList, pageById.policyList];
 
         beforeEach(() => {
-          loginServerless(roleName);
+          login(roleName);
         });
 
         for (const { id, title } of artifactPagesFullAccess) {
diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts
index fec6a0f803afb..6cf3ab727980a 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts
@@ -7,7 +7,7 @@
 
 import type { CyIndexEndpointHosts } from '../../../tasks/index_endpoint_hosts';
 import { indexEndpointHosts } from '../../../tasks/index_endpoint_hosts';
-import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless';
+import { login, ROLE } from '../../../tasks/login';
 import type { EndpointArtifactPageId } from '../../../screens';
 import {
   getNoPrivilegesPage,
@@ -55,12 +55,12 @@ describe(
     });
 
     // roles `t1_analyst` and `t2_analyst` are the same as far as endpoint access
-    (['t1_analyst', `t2_analyst`] as ServerlessUser[]).forEach((roleName) => {
+    (['t1_analyst', `t2_analyst`] as ROLE[]).forEach((roleName) => {
       describe(`for role: ${roleName}`, () => {
         const deniedPages = allPages.filter((page) => page.id !== 'endpointList');
 
         beforeEach(() => {
-          loginServerless(roleName);
+          login(roleName);
         });
 
         it('should have READ access to Endpoint list page', () => {
@@ -89,7 +89,7 @@ describe(
       ];
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.T3_ANALYST);
+        login(ROLE.t3_analyst);
       });
 
       it('should have access to Endpoint list page', () => {
@@ -128,7 +128,7 @@ describe(
       const deniedPages = allPages.filter(({ id }) => id !== 'blocklist' && id !== 'endpointList');
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.THREAT_INTELLIGENCE_ANALYST);
+        login(ROLE.threat_intelligence_analyst);
       });
 
       it('should have access to Endpoint list page', () => {
@@ -163,7 +163,7 @@ describe(
       ];
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.RULE_AUTHOR);
+        login(ROLE.rule_author);
       });
 
       for (const { id, title } of artifactPagesFullAccess) {
@@ -207,7 +207,7 @@ describe(
       const grantedAccessPages = [pageById.endpointList, pageById.policyList];
 
       beforeEach(() => {
-        loginServerless(ServerlessUser.SOC_MANAGER);
+        login(ROLE.soc_manager);
       });
 
       for (const { id, title } of artifactPagesFullAccess) {
@@ -238,11 +238,7 @@ describe(
 
     // Endpoint Operations Manager, Endpoint Policy Manager and Platform Engineer currently have the same level of access
     (
-      [
-        'platform_engineer',
-        `endpoint_operations_analyst`,
-        'endpoint_policy_manager',
-      ] as ServerlessUser[]
+      ['platform_engineer', `endpoint_operations_analyst`, 'endpoint_policy_manager'] as ROLE[]
     ).forEach((roleName) => {
       describe(`for role: ${roleName}`, () => {
         const artifactPagesFullAccess = [
@@ -253,7 +249,7 @@ describe(
         const grantedAccessPages = [pageById.endpointList, pageById.policyList];
 
         beforeEach(() => {
-          loginServerless(roleName);
+          login(roleName);
         });
 
         for (const { id, title } of artifactPagesFullAccess) {
diff --git a/x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts b/x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts
new file mode 100644
index 0000000000000..247b491f04632
--- /dev/null
+++ b/x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEndpointSecurityPolicyManager } from '../../../../scripts/endpoint/common/roles_users';
+
+export const getRoleWithArtifactReadPrivilege = (privilegePrefix: string) => {
+  const endpointSecurityPolicyManagerRole = getEndpointSecurityPolicyManager();
+
+  return {
+    ...endpointSecurityPolicyManagerRole,
+    kibana: [
+      {
+        ...endpointSecurityPolicyManagerRole.kibana[0],
+        feature: {
+          ...endpointSecurityPolicyManagerRole.kibana[0].feature,
+          siem: [
+            ...endpointSecurityPolicyManagerRole.kibana[0].feature.siem.filter(
+              (privilege) => privilege !== `${privilegePrefix}all`
+            ),
+            `${privilegePrefix}read`,
+          ],
+        },
+      },
+    ],
+  };
+};
diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts
index 4d8164391cb11..93614b6dbb86d 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts
@@ -9,6 +9,14 @@
 
 import type { CasePostRequest } from '@kbn/cases-plugin/common';
 import execa from 'execa';
+import type { KbnClient } from '@kbn/test';
+import type { ToolingLog } from '@kbn/tooling-log';
+import type { KibanaKnownUserAccounts } from '../common/constants';
+import { KIBANA_KNOWN_DEFAULT_ACCOUNTS } from '../common/constants';
+import type { EndpointSecurityRoleNames } from '../../../../scripts/endpoint/common/roles_users';
+import { SECURITY_SERVERLESS_ROLE_NAMES } from '../../../../scripts/endpoint/common/roles_users';
+import type { LoadedRoleAndUser } from '../../../../scripts/endpoint/common/role_and_user_loader';
+import { EndpointSecurityTestRolesLoader } from '../../../../scripts/endpoint/common/role_and_user_loader';
 import { startRuntimeServices } from '../../../../scripts/endpoint/endpoint_agent_runner/runtime';
 import { runFleetServerIfNeeded } from '../../../../scripts/endpoint/endpoint_agent_runner/fleet_server';
 import {
@@ -22,39 +30,89 @@ import type {
   CreateAndEnrollEndpointHostOptions,
   CreateAndEnrollEndpointHostResponse,
 } from '../../../../scripts/endpoint/common/endpoint_host_services';
+import {
+  createAndEnrollEndpointHost,
+  destroyEndpointHost,
+  startEndpointHost,
+  stopEndpointHost,
+  VAGRANT_CWD,
+} from '../../../../scripts/endpoint/common/endpoint_host_services';
 import type { IndexedEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_endpoint_policy_response';
 import {
   deleteIndexedEndpointPolicyResponse,
   indexEndpointPolicyResponse,
 } from '../../../../common/endpoint/data_loaders/index_endpoint_policy_response';
 import type { ActionDetails, HostPolicyResponse } from '../../../../common/endpoint/types';
-import type { IndexEndpointHostsCyTaskOptions } from '../types';
 import type {
-  IndexedEndpointRuleAlerts,
+  IndexEndpointHostsCyTaskOptions,
+  LoadUserAndRoleCyTaskOptions,
+  CreateUserAndRoleCyTaskOptions,
+} from '../types';
+import type {
   DeletedIndexedEndpointRuleAlerts,
+  IndexedEndpointRuleAlerts,
+} from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts';
+import {
+  deleteIndexedEndpointRuleAlerts,
+  indexEndpointRuleAlerts,
 } from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts';
 import type { IndexedHostsAndAlertsResponse } from '../../../../common/endpoint/index_data';
+import { deleteIndexedHostsAndAlerts } from '../../../../common/endpoint/index_data';
 import type { IndexedCase } from '../../../../common/endpoint/data_loaders/index_case';
+import { deleteIndexedCase, indexCase } from '../../../../common/endpoint/data_loaders/index_case';
 import { createRuntimeServices } from '../../../../scripts/endpoint/common/stack_services';
 import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
 import {
-  indexFleetEndpointPolicy,
   deleteIndexedFleetEndpointPolicies,
+  indexFleetEndpointPolicy,
 } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
-import { deleteIndexedCase, indexCase } from '../../../../common/endpoint/data_loaders/index_case';
 import { cyLoadEndpointDataHandler } from './plugin_handlers/endpoint_data_loader';
-import { deleteIndexedHostsAndAlerts } from '../../../../common/endpoint/index_data';
-import {
-  deleteIndexedEndpointRuleAlerts,
-  indexEndpointRuleAlerts,
-} from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts';
-import {
-  startEndpointHost,
-  createAndEnrollEndpointHost,
-  destroyEndpointHost,
-  stopEndpointHost,
-  VAGRANT_CWD,
-} from '../../../../scripts/endpoint/common/endpoint_host_services';
+
+/**
+ * Test Role/User loader for cypress. Checks to see if running in serverless and handles it as appropriate
+ */
+class TestRoleAndUserLoader extends EndpointSecurityTestRolesLoader {
+  constructor(
+    protected readonly kbnClient: KbnClient,
+    protected readonly logger: ToolingLog,
+    private readonly isServerless: boolean
+  ) {
+    super(kbnClient, logger);
+  }
+
+  async load(
+    name: EndpointSecurityRoleNames | KibanaKnownUserAccounts
+  ): Promise<LoadedRoleAndUser> {
+    // If its a known system account, then just exit here and use the default `changeme` password
+    if (KIBANA_KNOWN_DEFAULT_ACCOUNTS[name as KibanaKnownUserAccounts]) {
+      return {
+        role: name,
+        username: name,
+        password: 'changeme',
+      };
+    }
+
+    if (this.isServerless) {
+      // If the username is not one that we support in serverless, then throw an error.
+      if (!SECURITY_SERVERLESS_ROLE_NAMES[name as keyof typeof SECURITY_SERVERLESS_ROLE_NAMES]) {
+        throw new Error(
+          `username [${name}] is not valid when running in serverless. Valid values are: ${Object.keys(
+            SECURITY_SERVERLESS_ROLE_NAMES
+          ).join(', ')}`
+        );
+      }
+
+      // Roles/users for serverless will be already present in the env, so just return the defaults creds
+      return {
+        role: name,
+        username: name,
+        password: 'changeme',
+      };
+    }
+
+    return super.load(name as EndpointSecurityRoleNames);
+  }
+}
 
 /**
  * Cypress plugin for adding data loading related `task`s
@@ -68,7 +126,8 @@ export const dataLoaders = (
   on: Cypress.PluginEvents,
   config: Cypress.PluginConfigOptions
 ): void => {
-  // FIXME: investigate if we can create a `ToolingLog` that writes output to cypress and pass that to the stack services
+  // Env. variable is set by `cypress_serverless.config.ts`
+  const isServerless = config.env.IS_SERVERLESS;
 
   const stackServicesPromise = createRuntimeServices({
     kibanaUrl: config.env.KIBANA_URL,
@@ -81,6 +140,12 @@ export const dataLoaders = (
     asSuperuser: true,
   });
 
+  const roleAndUserLoaderPromise: Promise<TestRoleAndUserLoader> = stackServicesPromise.then(
+    ({ kbnClient, log }) => {
+      return new TestRoleAndUserLoader(kbnClient, log, isServerless);
+    }
+  );
+
   on('task', {
     indexFleetEndpointPolicy: async ({
       policyName,
@@ -201,6 +266,23 @@ export const dataLoaders = (
       const { esClient } = await stackServicesPromise;
       return deleteAllEndpointData(esClient, endpointAgentIds);
     },
+
+    /**
+     * Loads a user/role into Kibana. Used from `login()` task.
+     * @param name
+     */
+    loadUserAndRole: async ({ name }: LoadUserAndRoleCyTaskOptions): Promise<LoadedRoleAndUser> => {
+      return (await roleAndUserLoaderPromise).load(name);
+    },
+
+    /**
+     * Creates a new Role/User
+     */
+    createUserAndRole: async ({
+      role,
+    }: CreateUserAndRoleCyTaskOptions): Promise<LoadedRoleAndUser> => {
+      return (await roleAndUserLoaderPromise).create(role);
+    },
   });
 };
 
diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts
index 58eba62a72f82..8ac78b508d084 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts
@@ -5,242 +5,107 @@
  * 2.0.
  */
 
-import * as yaml from 'js-yaml';
-import type { Role } from '@kbn/security-plugin/common';
 import type { LoginState } from '@kbn/security-plugin/common/login_state';
-import { getWithResponseActionsRole } from '../../../../scripts/endpoint/common/roles_users/with_response_actions_role';
-import { getNoResponseActionsRole } from '../../../../scripts/endpoint/common/roles_users/without_response_actions_role';
-import { request } from './common';
-import { getT1Analyst } from '../../../../scripts/endpoint/common/roles_users/t1_analyst';
-import { getT2Analyst } from '../../../../scripts/endpoint/common/roles_users/t2_analyst';
-import { getHunter } from '../../../../scripts/endpoint/common/roles_users/hunter';
-import { getThreatIntelligenceAnalyst } from '../../../../scripts/endpoint/common/roles_users/threat_intelligence_analyst';
-import { getSocManager } from '../../../../scripts/endpoint/common/roles_users/soc_manager';
-import { getPlatformEngineer } from '../../../../scripts/endpoint/common/roles_users/platform_engineer';
-import { getEndpointOperationsAnalyst } from '../../../../scripts/endpoint/common/roles_users/endpoint_operations_analyst';
-import {
-  getEndpointSecurityPolicyManagementReadRole,
-  getEndpointSecurityPolicyManager,
-} from '../../../../scripts/endpoint/common/roles_users/endpoint_security_policy_manager';
-import { getDetectionsEngineer } from '../../../../scripts/endpoint/common/roles_users/detections_engineer';
-
-export enum ROLE {
-  t1_analyst = 't1Analyst',
-  t2_analyst = 't2Analyst',
-  analyst_hunter = 'hunter',
-  threat_intelligence_analyst = 'threatIntelligenceAnalyst',
-  detections_engineer = 'detectionsEngineer',
-  soc_manager = 'socManager',
-  platform_engineer = 'platformEngineer',
-  endpoint_operations_analyst = 'endpointOperationsAnalyst',
-  endpoint_security_policy_manager = 'endpointSecurityPolicyManager',
-  endpoint_response_actions_access = 'endpointResponseActionsAccess',
-  endpoint_response_actions_no_access = 'endpointResponseActionsNoAccess',
-  endpoint_security_policy_management_read = 'endpointSecurityPolicyManagementRead',
+import type { Role } from '@kbn/security-plugin/common';
+import { ENDPOINT_SECURITY_ROLE_NAMES } from '../../../../scripts/endpoint/common/roles_users';
+import type { SecurityTestUser } from '../common/constants';
+import { COMMON_API_HEADERS, request } from './common';
+
+export const ROLE = Object.freeze<Record<SecurityTestUser, SecurityTestUser>>({
+  ...ENDPOINT_SECURITY_ROLE_NAMES,
+  elastic: 'elastic',
+  elastic_serverless: 'elastic_serverless',
+  system_indices_superuser: 'system_indices_superuser',
+});
+
+interface CyLoginTask {
+  (user?: SecurityTestUser): ReturnType<typeof sendApiLoginRequest>;
+
+  /**
+   * Login using any username/password
+   * @param username
+   * @param password
+   */
+  with(username: string, password: string): ReturnType<typeof sendApiLoginRequest>;
+
+  /**
+   * Creates the provided role in kibana/ES along with a respective user (same name as role)
+   * and then login with this new user
+   * @param role
+   */
+  withCustomRole(role: Role): ReturnType<typeof sendApiLoginRequest>;
 }
 
-export const rolesMapping: { [key in ROLE]: Omit<Role, 'name'> } = {
-  t1Analyst: getT1Analyst(),
-  t2Analyst: getT2Analyst(),
-  hunter: getHunter(),
-  threatIntelligenceAnalyst: getThreatIntelligenceAnalyst(),
-  socManager: getSocManager(),
-  platformEngineer: getPlatformEngineer(),
-  endpointOperationsAnalyst: getEndpointOperationsAnalyst(),
-  endpointSecurityPolicyManager: getEndpointSecurityPolicyManager(),
-  detectionsEngineer: getDetectionsEngineer(),
-  endpointResponseActionsAccess: getWithResponseActionsRole(),
-  endpointResponseActionsNoAccess: getNoResponseActionsRole(),
-  endpointSecurityPolicyManagementRead: getEndpointSecurityPolicyManagementReadRole(),
-};
-/**
- * Credentials in the `kibana.dev.yml` config file will be used to authenticate
- * with Kibana when credentials are not provided via environment variables
- */
-const KIBANA_DEV_YML_PATH = '../../../config/kibana.dev.yml';
-
-/**
- * The configuration path in `kibana.dev.yml` to the username to be used when
- * authenticating with Kibana.
- */
-const ELASTICSEARCH_USERNAME_CONFIG_PATH = 'config.elasticsearch.username';
-
-/**
- * The configuration path in `kibana.dev.yml` to the password to be used when
- * authenticating with Kibana.
- */
-const ELASTICSEARCH_PASSWORD_CONFIG_PATH = 'config.elasticsearch.password';
-
-/**
- * The `CYPRESS_ELASTICSEARCH_USERNAME` environment variable specifies the
- * username to be used when authenticating with Kibana
- */
-const ELASTICSEARCH_USERNAME = 'ELASTICSEARCH_USERNAME';
-
 /**
- * The `CYPRESS_ELASTICSEARCH_PASSWORD` environment variable specifies the
- * username to be used when authenticating with Kibana
+ * Login to Kibana using API (not login page).
+ * By default, user will be logged in using `KIBANA_USERNAME` and `KIBANA_PASSWORD` retrieved from
+ * the cypress `env`
+ *
+ * @param user
  */
-const ELASTICSEARCH_PASSWORD = 'ELASTICSEARCH_PASSWORD';
-
-const KIBANA_USERNAME = 'KIBANA_USERNAME';
-const KIBANA_PASSWORD = 'KIBANA_PASSWORD';
+export const login: CyLoginTask = (
+  // FIXME:PT default user to `soc_manager`
+  user?: SecurityTestUser
+): ReturnType<typeof sendApiLoginRequest> => {
+  let username = Cypress.env('KIBANA_USERNAME');
+  let password = Cypress.env('KIBANA_PASSWORD');
+
+  if (user) {
+    return cy.task('loadUserAndRole', { name: user }).then((loadedUser) => {
+      username = loadedUser.username;
+      password = loadedUser.password;
+
+      return sendApiLoginRequest(username, password);
+    });
+  } else {
+    return sendApiLoginRequest(username, password);
+  }
+};
 
-export const createCustomRoleAndUser = (role: string, rolePrivileges: Omit<Role, 'name'>) => {
-  // post the role
-  request({
-    method: 'PUT',
-    url: `/api/security/role/${role}`,
-    body: rolePrivileges,
-  });
+login.with = (username: string, password: string): ReturnType<typeof sendApiLoginRequest> => {
+  return sendApiLoginRequest(username, password);
+};
 
-  // post the user associated with the role to elasticsearch
-  request({
-    method: 'POST',
-    url: `/internal/security/users/${role}`,
-    body: {
-      username: role,
-      password: Cypress.env(ELASTICSEARCH_PASSWORD),
-      roles: [role],
-    },
+login.withCustomRole = (role: Role): ReturnType<typeof sendApiLoginRequest> => {
+  return cy.task('createUserAndRole', { role }).then(({ username, password }) => {
+    return sendApiLoginRequest(username, password);
   });
 };
 
-const loginWithUsernameAndPassword = (username: string, password: string) => {
+/**
+ * Send login via API
+ * @param username
+ * @param password
+ *
+ * @private
+ */
+const sendApiLoginRequest = (
+  username: string,
+  password: string
+): Cypress.Chainable<{ username: string; password: string }> => {
   const baseUrl = Cypress.config().baseUrl;
-  if (!baseUrl) {
-    throw Error(`Cypress config baseUrl not set!`);
-  }
+  const loginUrl = `${baseUrl}/internal/security/login`;
+  const headers = { ...COMMON_API_HEADERS };
+
+  cy.log(`Authenticating [${username}] via ${loginUrl}`);
 
-  // Programmatically authenticate without interacting with the Kibana login page.
-  const headers = { 'kbn-xsrf': 'cypress-creds' };
-  request<LoginState>({ headers, url: `${baseUrl}/internal/security/login_state` }).then(
-    (loginState) => {
+  return request<LoginState>({ headers, url: `${baseUrl}/internal/security/login_state` })
+    .then((loginState) => {
       const basicProvider = loginState.body.selector.providers.find(
         (provider) => provider.type === 'basic'
       );
+
       return request({
-        url: `${baseUrl}/internal/security/login`,
+        url: loginUrl,
         method: 'POST',
         headers,
         body: {
-          providerType: basicProvider.type,
-          providerName: basicProvider.name,
+          providerType: basicProvider?.type,
+          providerName: basicProvider?.name,
           currentURL: '/',
           params: { username, password },
         },
       });
-    }
-  );
-};
-
-export const loginWithRole = (role: ROLE) => {
-  loginWithCustomRole(role, rolesMapping[role]);
-};
-
-export const loginWithCustomRole = (role: string, rolePrivileges: Omit<Role, 'name'>) => {
-  createCustomRoleAndUser(role, rolePrivileges);
-
-  cy.log(`origin: ${Cypress.config().baseUrl}`);
-
-  loginWithUsernameAndPassword(role, Cypress.env(ELASTICSEARCH_PASSWORD));
-};
-
-/**
- * Authenticates with Kibana using, if specified, credentials specified by
- * environment variables. The credentials in `kibana.dev.yml` will be used
- * for authentication when the environment variables are unset.
- *
- * To speed the execution of tests, prefer this non-interactive authentication,
- * which is faster than authentication via Kibana's interactive login page.
- */
-export const login = (role?: ROLE) => {
-  if (role != null) {
-    loginWithRole(role);
-  } else if (credentialsProvidedByEnvironment()) {
-    loginViaEnvironmentCredentials();
-  } else {
-    loginViaConfig();
-  }
-};
-
-/**
- * Returns `true` if the credentials used to login to Kibana are provided
- * via environment variables
- */
-const credentialsProvidedByEnvironment = (): boolean =>
-  (Cypress.env(KIBANA_USERNAME) != null && Cypress.env(KIBANA_PASSWORD) != null) ||
-  (Cypress.env(ELASTICSEARCH_USERNAME) != null && Cypress.env(ELASTICSEARCH_PASSWORD) != null);
-
-/**
- * Authenticates with Kibana by reading credentials from the
- * `CYPRESS_ELASTICSEARCH_USERNAME` and `CYPRESS_ELASTICSEARCH_PASSWORD`
- * environment variables, and POSTing the username and password directly to
- * Kibana's `/internal/security/login` endpoint, bypassing the login page (for speed).
- */
-const loginViaEnvironmentCredentials = () => {
-  let username: string;
-  let password: string;
-  let usernameEnvVar: string;
-  let passwordEnvVar: string;
-
-  if (Cypress.env(KIBANA_USERNAME) && Cypress.env(KIBANA_PASSWORD)) {
-    username = Cypress.env(KIBANA_USERNAME);
-    password = Cypress.env(KIBANA_PASSWORD);
-    usernameEnvVar = KIBANA_USERNAME;
-    passwordEnvVar = KIBANA_PASSWORD;
-  } else {
-    username = Cypress.env(ELASTICSEARCH_USERNAME);
-    password = Cypress.env(ELASTICSEARCH_PASSWORD);
-    usernameEnvVar = ELASTICSEARCH_USERNAME;
-    passwordEnvVar = ELASTICSEARCH_PASSWORD;
-  }
-
-  cy.log(
-    `Authenticating user [${username}] retrieved via environment credentials from the \`CYPRESS_${usernameEnvVar}\` and \`CYPRESS_${passwordEnvVar}\` environment variables`
-  );
-
-  loginWithUsernameAndPassword(username, password);
-};
-
-/**
- * Authenticates with Kibana by reading credentials from the
- * `kibana.dev.yml` file and POSTing the username and password directly to
- * Kibana's `/internal/security/login` endpoint, bypassing the login page (for speed).
- */
-const loginViaConfig = () => {
-  cy.log(
-    `Authenticating via config credentials \`${ELASTICSEARCH_USERNAME_CONFIG_PATH}\` and \`${ELASTICSEARCH_PASSWORD_CONFIG_PATH}\` from \`${KIBANA_DEV_YML_PATH}\``
-  );
-
-  // read the login details from `kibana.dev.yaml`
-  cy.readFile(KIBANA_DEV_YML_PATH).then((kibanaDevYml) => {
-    const config = yaml.safeLoad(kibanaDevYml);
-    loginWithUsernameAndPassword(
-      Cypress.env(ELASTICSEARCH_USERNAME),
-      config.elasticsearch.password
-    );
-  });
-};
-
-export const getRoleWithArtifactReadPrivilege = (privilegePrefix: string) => {
-  const endpointSecurityPolicyManagerRole = getEndpointSecurityPolicyManager();
-
-  return {
-    ...endpointSecurityPolicyManagerRole,
-    kibana: [
-      {
-        ...endpointSecurityPolicyManagerRole.kibana[0],
-        feature: {
-          ...endpointSecurityPolicyManagerRole.kibana[0].feature,
-          siem: [
-            ...endpointSecurityPolicyManagerRole.kibana[0].feature.siem.filter(
-              (privilege) => privilege !== `${privilegePrefix}all`
-            ),
-            `${privilegePrefix}read`,
-          ],
-        },
-      },
-    ],
-  };
+    })
+    .then(() => ({ username, password }));
 };
diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts
deleted file mode 100644
index 533a17663e16b..0000000000000
--- a/x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { LoginState } from '@kbn/security-plugin/common/login_state';
-import { COMMON_API_HEADERS, request } from './common';
-
-export enum ServerlessUser {
-  T1_ANALYST = 't1_analyst',
-  T2_ANALYST = 't2_analyst',
-  T3_ANALYST = 't3_analyst',
-  THREAT_INTELLIGENCE_ANALYST = 'threat_intelligence_analyst',
-  RULE_AUTHOR = 'rule_author',
-  SOC_MANAGER = 'soc_manager',
-  DETECTIONS_ADMIN = 'detections_admin',
-  PLATFORM_ENGINEER = 'platform_engineer',
-  ENDPOINT_OPERATIONS_ANALYST = 'endpoint_operations_analyst',
-  ENDPOINT_POLICY_MANAGER = 'endpoint_policy_manager',
-}
-
-/**
- * Send login via API
- * @param username
- * @param password
- *
- * @private
- */
-const sendApiLoginRequest = (
-  username: string,
-  password: string
-): Cypress.Chainable<{ username: string; password: string }> => {
-  const baseUrl = Cypress.config().baseUrl;
-  const headers = { ...COMMON_API_HEADERS };
-
-  cy.log(`Authenticating [${username}] via ${baseUrl}`);
-
-  return request<LoginState>({ headers, url: `${baseUrl}/internal/security/login_state` })
-    .then((loginState) => {
-      const basicProvider = loginState.body.selector.providers.find(
-        (provider) => provider.type === 'basic'
-      );
-
-      return request({
-        url: `${baseUrl}/internal/security/login`,
-        method: 'POST',
-        headers,
-        body: {
-          providerType: basicProvider?.type,
-          providerName: basicProvider?.name,
-          currentURL: '/',
-          params: { username, password },
-        },
-      });
-    })
-    .then(() => ({ username, password }));
-};
-
-interface CyLoginTask {
-  (user?: ServerlessUser | 'elastic'): ReturnType<typeof sendApiLoginRequest>;
-
-  /**
-   * Login using any username/password
-   * @param username
-   * @param password
-   */
-  with(username: string, password: string): ReturnType<typeof sendApiLoginRequest>;
-}
-
-/**
- * Login to Kibana using API (not login page). By default, user will be logged in using
- * the username and password defined via `KIBANA_USERNAME` and `KIBANA_PASSWORD` cypress env
- * variables.
- * @param user Defaults to `soc_manager`
- */
-export const loginServerless: CyLoginTask = (
-  user: ServerlessUser | 'elastic' = ServerlessUser.SOC_MANAGER
-): ReturnType<typeof sendApiLoginRequest> => {
-  const username = Cypress.env('KIBANA_USERNAME');
-  const password = Cypress.env('KIBANA_PASSWORD');
-
-  if (user && user !== 'elastic') {
-    throw new Error('Serverless usernames not yet implemented');
-
-    // return cy.task('loadUserAndRole', { name: user }).then((loadedUser) => {
-    //   username = loadedUser.username;
-    //   password = loadedUser.password;
-    //
-    //   return sendApiLoginRequest(username, password);
-    // });
-  } else {
-    return sendApiLoginRequest(username, password);
-  }
-};
-
-loginServerless.with = (
-  username: string,
-  password: string
-): ReturnType<typeof sendApiLoginRequest> => {
-  return sendApiLoginRequest(username, password);
-};
diff --git a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json
index 4d7fe47ec2d23..94d50ffe7f62a 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json
+++ b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json
@@ -32,5 +32,6 @@
     "@kbn/test",
     "@kbn/repo-info",
     "@kbn/data-views-plugin",
+    "@kbn/tooling-log",
   ]
 }
diff --git a/x-pack/plugins/security_solution/public/management/cypress/types.ts b/x-pack/plugins/security_solution/public/management/cypress/types.ts
index fecaa33a6a70a..aee97723c7d51 100644
--- a/x-pack/plugins/security_solution/public/management/cypress/types.ts
+++ b/x-pack/plugins/security_solution/public/management/cypress/types.ts
@@ -7,8 +7,10 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
+import type { Role } from '@kbn/security-plugin/common';
 import type { ActionDetails } from '../../../common/endpoint/types';
 import type { CyLoadEndpointDataOptions } from './support/plugin_handlers/endpoint_data_loader';
+import type { SecurityTestUser } from './common/constants';
 
 type PossibleChainable =
   | Cypress.Chainable<any>
@@ -56,3 +58,11 @@ export interface HostActionResponse {
     state: { state?: 'success' | 'failure' };
   };
 }
+
+export interface LoadUserAndRoleCyTaskOptions {
+  name: SecurityTestUser;
+}
+
+export interface CreateUserAndRoleCyTaskOptions {
+  role: Role;
+}
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts
index 1d2b3d5f47784..4a8b2daca2af4 100644
--- a/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts
+++ b/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts
@@ -10,3 +10,8 @@ export const HORIZONTAL_LINE = '-'.repeat(80);
 export const ENDPOINT_EVENTS_INDEX = 'logs-endpoint.events.process-default';
 
 export const ENDPOINT_ALERTS_INDEX = 'logs-endpoint.alerts-default';
+
+export const COMMON_API_HEADERS = Object.freeze({
+  'kbn-xsrf': 'security-solution',
+  'x-elastic-internal-origin': 'security-solution',
+});
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts
new file mode 100644
index 0000000000000..f8c51d5255018
--- /dev/null
+++ b/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable max-classes-per-file */
+
+import type { KbnClient } from '@kbn/test';
+import type { Role } from '@kbn/security-plugin/common';
+import type { ToolingLog } from '@kbn/tooling-log';
+import { inspect } from 'util';
+import type { AxiosError } from 'axios';
+import type { EndpointSecurityRoleDefinitions } from './roles_users';
+import { getAllEndpointSecurityRoles } from './roles_users';
+import { catchAxiosErrorFormatAndThrow } from './format_axios_error';
+import { COMMON_API_HEADERS } from './constants';
+
+const ignoreHttp409Error = (error: AxiosError) => {
+  if (error?.response?.status === 409) {
+    return;
+  }
+
+  throw error;
+};
+
+export interface LoadedRoleAndUser {
+  role: string;
+  username: string;
+  password: string;
+}
+
+export interface RoleAndUserLoaderInterface<R extends Record<string, Role> = Record<string, Role>> {
+  /**
+   * Loads the requested Role into kibana and then creates a user by the same role name that is
+   * assigned to the given role
+   * @param name
+   */
+  load(name: keyof R): Promise<LoadedRoleAndUser>;
+
+  /**
+   * Loads all roles/users
+   */
+  loadAll(): Promise<Record<keyof R, LoadedRoleAndUser>>;
+
+  /**
+   * Creates a new Role in kibana along with a user (by the same name as the Role name)
+   * that is assigned to the given role
+   * @param role
+   */
+  create(role: Role): Promise<LoadedRoleAndUser>;
+}
+
+/**
+ * A generic class for loading roles and creating associated user into kibana
+ */
+export class RoleAndUserLoader<R extends Record<string, Role> = Record<string, Role>>
+  implements RoleAndUserLoaderInterface<R>
+{
+  protected readonly logPromiseError: (error: Error) => never;
+
+  constructor(
+    protected readonly kbnClient: KbnClient,
+    protected readonly logger: ToolingLog,
+    protected readonly roles: R
+  ) {
+    this.logPromiseError = (error) => {
+      this.logger.error(inspect(error, { depth: 5 }));
+      throw error;
+    };
+  }
+
+  async load(name: keyof R): Promise<LoadedRoleAndUser> {
+    const role = this.roles[name];
+
+    if (!role) {
+      throw new Error(
+        `Unknown role/user: [${String(name)}]. Valid values are: [${Object.keys(this.roles).join(
+          ', '
+        )}]`
+      );
+    }
+
+    return this.create(role);
+  }
+
+  async loadAll(): Promise<Record<keyof R, LoadedRoleAndUser>> {
+    const response = {} as Record<keyof R, LoadedRoleAndUser>;
+
+    for (const [name, role] of Object.entries(this.roles)) {
+      response[name as keyof R] = await this.create(role);
+    }
+
+    return response;
+  }
+
+  public async create(role: Role): Promise<LoadedRoleAndUser> {
+    const roleName = role.name;
+
+    await this.createRole(role);
+    await this.createUser(roleName, 'changeme', [roleName]);
+
+    return {
+      role: roleName,
+      username: roleName,
+      password: 'changeme',
+    };
+  }
+
+  protected async createRole(role: Role): Promise<void> {
+    const { name: roleName, ...roleDefinition } = role;
+
+    this.logger.debug(`creating role:`, roleDefinition);
+
+    await this.kbnClient
+      .request({
+        method: 'PUT',
+        path: `/api/security/role/${roleName}`,
+        headers: {
+          ...COMMON_API_HEADERS,
+        },
+        body: roleDefinition,
+      })
+      .then((response) => {
+        this.logger.debug(`Role [${roleName}] created/updated`, response?.data);
+        return response;
+      })
+      .catch(ignoreHttp409Error)
+      .catch(catchAxiosErrorFormatAndThrow)
+      .catch(this.logPromiseError);
+  }
+
+  protected async createUser(
+    username: string,
+    password: string,
+    roles: string[] = []
+  ): Promise<void> {
+    const user = {
+      username,
+      password,
+      roles,
+      full_name: username,
+      email: '',
+    };
+
+    this.logger.debug(`creating user:`, user);
+
+    await this.kbnClient
+      .request({
+        method: 'POST',
+        path: `/internal/security/users/${username}`,
+        headers: {
+          ...COMMON_API_HEADERS,
+        },
+        body: user,
+      })
+      .then((response) => {
+        this.logger.debug(`User [${username}] created/updated`, response?.data);
+        return response;
+      })
+      .catch(ignoreHttp409Error)
+      .catch(catchAxiosErrorFormatAndThrow)
+      .catch(this.logPromiseError);
+  }
+}
+
+/**
+ * Role and user loader for Endpoint security dev/testing
+ */
+export class EndpointSecurityTestRolesLoader extends RoleAndUserLoader<EndpointSecurityRoleDefinitions> {
+  constructor(protected readonly kbnClient: KbnClient, protected readonly logger: ToolingLog) {
+    super(kbnClient, logger, getAllEndpointSecurityRoles());
+  }
+}
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts
new file mode 100644
index 0000000000000..b035f55bf1589
--- /dev/null
+++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Role } from '@kbn/security-plugin/common';
+import { getRuleAuthor } from './rule_author';
+import { getT3Analyst } from './t3_analyst';
+import { getT1Analyst } from './t1_analyst';
+import { getT2Analyst } from './t2_analyst';
+import { getHunter } from './hunter';
+import { getThreatIntelligenceAnalyst } from './threat_intelligence_analyst';
+import { getSocManager } from './soc_manager';
+import { getPlatformEngineer } from './platform_engineer';
+import { getEndpointOperationsAnalyst } from './endpoint_operations_analyst';
+import {
+  getEndpointSecurityPolicyManagementReadRole,
+  getEndpointSecurityPolicyManager,
+} from './endpoint_security_policy_manager';
+import { getDetectionsEngineer } from './detections_engineer';
+import { getWithResponseActionsRole } from './with_response_actions_role';
+import { getNoResponseActionsRole } from './without_response_actions_role';
+
+export * from './with_response_actions_role';
+export * from './without_response_actions_role';
+export * from './t1_analyst';
+export * from './t2_analyst';
+export * from './t3_analyst';
+export * from './hunter';
+export * from './threat_intelligence_analyst';
+export * from './soc_manager';
+export * from './platform_engineer';
+export * from './endpoint_operations_analyst';
+export * from './endpoint_security_policy_manager';
+export * from './detections_engineer';
+
+export type EndpointSecurityRoleNames = keyof typeof ENDPOINT_SECURITY_ROLE_NAMES;
+
+export type EndpointSecurityRoleDefinitions = Record<EndpointSecurityRoleNames, Role>;
+
+/**
+ * Security Solution set of roles that are loaded and used in serverless deployments.
+ * The source of these role definitions is under `project-controller` at:
+ *
+ * @see https://github.com/elastic/project-controller/blob/main/internal/project/security/config/roles.yml
+ *
+ * The role definition spreadsheet can be found here:
+ *
+ * @see https://docs.google.com/spreadsheets/d/16aGow187AunLCBFZLlbVyS81iQNuMpNxd96LOerWj4c/edit#gid=1936689222
+ */
+export const SECURITY_SERVERLESS_ROLE_NAMES = Object.freeze({
+  t1_analyst: 't1_analyst',
+  t2_analyst: 't2_analyst',
+  t3_analyst: 't3_analyst',
+  threat_intelligence_analyst: 'threat_intelligence_analyst',
+  rule_author: 'rule_author',
+  soc_manager: 'soc_manager',
+  detections_admin: 'detections_admin',
+  platform_engineer: 'platform_engineer',
+  endpoint_operations_analyst: 'endpoint_operations_analyst',
+  endpoint_policy_manager: 'endpoint_policy_manager',
+});
+
+export const ENDPOINT_SECURITY_ROLE_NAMES = Object.freeze({
+  // --------------------------------------
+  // Set of roles used in serverless
+  ...SECURITY_SERVERLESS_ROLE_NAMES,
+
+  // --------------------------------------
+  // Other roles used for testing
+  hunter: 'hunter',
+  endpoint_response_actions_access: 'endpoint_response_actions_access',
+  endpoint_response_actions_no_access: 'endpoint_response_actions_no_access',
+  endpoint_security_policy_management_read: 'endpoint_security_policy_management_read',
+});
+
+export const getAllEndpointSecurityRoles = (): EndpointSecurityRoleDefinitions => {
+  return {
+    t1_analyst: {
+      ...getT1Analyst(),
+      name: 't1_analyst',
+    },
+    t2_analyst: {
+      ...getT2Analyst(),
+      name: 't2_analyst',
+    },
+    t3_analyst: {
+      ...getT3Analyst(),
+      name: 't3_analyst',
+    },
+    threat_intelligence_analyst: {
+      ...getThreatIntelligenceAnalyst(),
+      name: 'threat_intelligence_analyst',
+    },
+    rule_author: {
+      ...getRuleAuthor(),
+      name: 'rule_author',
+    },
+    soc_manager: {
+      ...getSocManager(),
+      name: 'soc_manager',
+    },
+    detections_admin: {
+      ...getDetectionsEngineer(),
+      name: 'detections_admin',
+    },
+    platform_engineer: {
+      ...getPlatformEngineer(),
+      name: 'platform_engineer',
+    },
+    endpoint_operations_analyst: {
+      ...getEndpointOperationsAnalyst(),
+      name: 'endpoint_operations_analyst',
+    },
+    endpoint_policy_manager: {
+      ...getEndpointSecurityPolicyManager(),
+      name: 'endpoint_policy_manager',
+    },
+
+    hunter: {
+      ...getHunter(),
+      name: 'hunter',
+    },
+    endpoint_response_actions_access: {
+      ...getWithResponseActionsRole(),
+      name: 'endpoint_response_actions_access',
+    },
+    endpoint_response_actions_no_access: {
+      ...getNoResponseActionsRole(),
+      name: 'endpoint_response_actions_no_access',
+    },
+    endpoint_security_policy_management_read: {
+      ...getEndpointSecurityPolicyManagementReadRole(),
+      name: 'endpoint_security_policy_management_read',
+    },
+  };
+};
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts
new file mode 100644
index 0000000000000..f957fe8947c5d
--- /dev/null
+++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Role } from '@kbn/security-plugin/common';
+import { getNoResponseActionsRole } from './without_response_actions_role';
+
+export const getRuleAuthor: () => Omit<Role, 'name'> = () => {
+  const noResponseActionsRole = getNoResponseActionsRole();
+  return {
+    ...noResponseActionsRole,
+    kibana: [
+      {
+        ...noResponseActionsRole.kibana[0],
+        feature: {
+          ...noResponseActionsRole.kibana[0].feature,
+          siem: [
+            'all',
+            'read_alerts',
+            'crud_alerts',
+            'policy_management_all',
+            'endpoint_list_all',
+            'trusted_applications_all',
+            'event_filters_all',
+            'host_isolation_exceptions_read',
+            'blocklist_all',
+            'actions_log_management_read',
+          ],
+        },
+      },
+    ],
+  };
+};
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts
new file mode 100644
index 0000000000000..304c4e6d744ee
--- /dev/null
+++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Role } from '@kbn/security-plugin/common';
+import { getNoResponseActionsRole } from './without_response_actions_role';
+
+export const getT3Analyst: () => Omit<Role, 'name'> = () => {
+  const noResponseActionsRole = getNoResponseActionsRole();
+  return {
+    ...noResponseActionsRole,
+    kibana: [
+      {
+        ...noResponseActionsRole.kibana[0],
+        feature: {
+          ...noResponseActionsRole.kibana[0].feature,
+          siem: [
+            'all',
+            'read_alerts',
+            'crud_alerts',
+            'endpoint_list_all',
+            'trusted_applications_all',
+            'event_filters_all',
+            'host_isolation_exceptions_all',
+            'blocklist_all',
+            'policy_management_read',
+            'host_isolation_all',
+            'process_operations_all',
+            'actions_log_management_all',
+            'file_operations_all',
+          ],
+        },
+      },
+    ],
+  };
+};
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts
index c1c38dcf8b30a..330bcf4589a74 100644
--- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts
+++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts
@@ -14,36 +14,13 @@ import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { ToolingLog } from '@kbn/tooling-log';
 import type { KbnClientOptions } from '@kbn/test';
 import { KbnClient } from '@kbn/test';
-import type { Role } from '@kbn/security-plugin/common';
+import { EndpointSecurityTestRolesLoader } from './common/role_and_user_loader';
 import { METADATA_DATASTREAM } from '../../common/endpoint/constants';
 import { EndpointMetadataGenerator } from '../../common/endpoint/data_generators/endpoint_metadata_generator';
 import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
 import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data';
 import { fetchStackVersion, isServerlessKibanaFlavor } from './common/stack_services';
 import { ENDPOINT_ALERTS_INDEX, ENDPOINT_EVENTS_INDEX } from './common/constants';
-import { getWithResponseActionsRole } from './common/roles_users/with_response_actions_role';
-import { getNoResponseActionsRole } from './common/roles_users/without_response_actions_role';
-import { getT1Analyst } from './common/roles_users/t1_analyst';
-import { getT2Analyst } from './common/roles_users/t2_analyst';
-import { getEndpointOperationsAnalyst } from './common/roles_users/endpoint_operations_analyst';
-import { getEndpointSecurityPolicyManager } from './common/roles_users/endpoint_security_policy_manager';
-import { getHunter } from './common/roles_users/hunter';
-import { getPlatformEngineer } from './common/roles_users/platform_engineer';
-import { getSocManager } from './common/roles_users/soc_manager';
-import { getThreatIntelligenceAnalyst } from './common/roles_users/threat_intelligence_analyst';
-
-const rolesMapping: { [id: string]: Omit<Role, 'name'> } = {
-  t1Analyst: getT1Analyst(),
-  t2Analyst: getT2Analyst(),
-  hunter: getHunter(),
-  threatIntelligenceAnalyst: getThreatIntelligenceAnalyst(),
-  socManager: getSocManager(),
-  platformEngineer: getPlatformEngineer(),
-  endpointOperationsAnalyst: getEndpointOperationsAnalyst(),
-  endpointSecurityPolicyManager: getEndpointSecurityPolicyManager(),
-  withResponseActionsRole: getWithResponseActionsRole(),
-  noResponseActionsRole: getNoResponseActionsRole(),
-};
 
 main();
 
@@ -67,31 +44,6 @@ async function deleteIndices(indices: string[], client: Client) {
   }
 }
 
-async function addRole(kbnClient: KbnClient, role: Role): Promise<string | undefined> {
-  if (!role) {
-    console.log('No role data given');
-    return;
-  }
-
-  const { name, ...permissions } = role;
-  const path = `/api/security/role/${name}?createOnly=true`;
-
-  // add role if doesn't exist already
-  try {
-    console.log(`Adding ${name} role`);
-    await kbnClient.request({
-      method: 'PUT',
-      path,
-      body: permissions,
-    });
-
-    return name;
-  } catch (error) {
-    console.log(error);
-    handleErr(error);
-  }
-}
-
 interface UserInfo {
   username: string;
   password: string;
@@ -422,19 +374,7 @@ async function main() {
       throw new Error(`Can not use '--rbacUser' option against serverless deployment`);
     }
 
-    // Add roles and users with response actions kibana privileges
-    for (const role of Object.keys(rolesMapping)) {
-      const addedRole = await addRole(kbnClient, {
-        name: role,
-        ...rolesMapping[role],
-      });
-      if (addedRole) {
-        logger.info(`Successfully added ${role} role`);
-        await addUser(client, { username: role, password: 'changeme', roles: [role] });
-      } else {
-        logger.warning(`Failed to add role, ${role}`);
-      }
-    }
+    await loadRbacTestUsers(kbnClient, logger);
   }
 
   const seed = argv.seed || Math.random().toString();
@@ -499,3 +439,12 @@ async function main() {
 
   logger.info(`Creating and indexing documents took: ${new Date().getTime() - startTime}ms`);
 }
+
+const loadRbacTestUsers = async (kbnClient: KbnClient, logger: ToolingLog): Promise<void> => {
+  const loadedRoles = await new EndpointSecurityTestRolesLoader(kbnClient, logger).loadAll();
+
+  logger.info(`Roles and associated users loaded. Login accounts:
+  ${Object.values(loadedRoles)
+    .map(({ username, password }) => `${username} / ${password}`)
+    .join('\n  ')}`);
+};

From 077be69de1a99335ab72d710496c67835d51d75e Mon Sep 17 00:00:00 2001
From: Andrew Macri <andrew.macri@elastic.co>
Date: Mon, 25 Sep 2023 11:55:19 -0600
Subject: [PATCH 23/27] [Security Solution] [Elastic AI Assistant] LangChain
 Agents and Tools integration for ES|QL query generation via ELSER (#167097)

## [Security Solution] [Elastic AI Assistant] LangChain Agents and Tools integration for ES|QL query generation via ELSER

This PR integrates [LangChain](https://www.langchain.com/) [Agents](https://js.langchain.com/docs/modules/agents/) and [Tools](https://js.langchain.com/docs/modules/agents/tools/) with the [Elastic AI Assistant](https://www.elastic.co/blog/introducing-elastic-ai-assistant).

These abstractions enable the LLM to dynamically choose whether or not to query, via [ELSER](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-elser.html), an [ES|QL](https://www.elastic.co/blog/elasticsearch-query-language-esql) knowledge base. Context from the knowledge base is used to generate `ES|QL` queries, or answer questions about `ES|QL`.

Registration of the tool occurs in `x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts`:

```typescript
  const tools: Tool[] = [
    new ChainTool({
      name: 'esql-language-knowledge-base',
      description:
        'Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.',
      chain,
    }),
  ];
```

The `tools` array above may be updated in future PRs to include, for example, an `ES|QL` query validator endpoint.

### Details

The `callAgentExecutor` function in `x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts`:

1. Creates a `RetrievalQAChain` from an `ELSER` backed `ElasticsearchStore`, which serves as a knowledge base for `ES|QL`:

```typescript
  // ELSER backed ElasticsearchStore for Knowledge Base
  const esStore = new ElasticsearchStore(esClient, KNOWLEDGE_BASE_INDEX_PATTERN, logger);
  const chain = RetrievalQAChain.fromLLM(llm, esStore.asRetriever());
```

2. Registers the chain as a tool, which may be invoked by the LLM based on its description:

```typescript
  const tools: Tool[] = [
    new ChainTool({
      name: 'esql-language-knowledge-base',
      description:
        'Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.',
      chain,
    }),
  ];
```

3. Creates an Agent executor that combines the `tools` above, the `ActionsClientLlm` (an abstraction that calls `actionsClient.execute`), and memory of the previous messages in the conversation:

```typescript
  const executor = await initializeAgentExecutorWithOptions(tools, llm, {
    agentType: 'chat-conversational-react-description',
    memory,
    verbose: false,
  });
```

Note: Set `verbose` above to `true` to for detailed debugging output from LangChain.

4. Calls the `executor`, kicking it off with `latestMessage`:

```typescript
    await executor.call({ input: latestMessage[0].content });
```

### Changes to `x-pack/packages/kbn-elastic-assistant`

A client side change was required to the assistant, because the response returned from the agent executor is JSON. This response is parsed on the client in `x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx`:

```typescript
  return assistantLangChain ? getFormattedMessageContent(result) : result;
```

Client-side parsing of the response only happens when then `assistantLangChain` feature flag is `true`.

## Desk testing

Set

```typescript
assistantLangChain={true}
```

in `x-pack/plugins/security_solution/public/assistant/provider.tsx` to enable this experimental feature in development environments.

Also (optionally) set `verbose` to `true` in the following code in ``x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts``:

```typescript
  const executor = await initializeAgentExecutorWithOptions(tools, llm, {
    agentType: 'chat-conversational-react-description',
    memory,
    verbose: true,
  });
```

After setting the feature flag and optionally enabling verbose debugging output, you may ask the assistant to generate an `ES|QL` query, per the example in the next section.

### Example output

When the Elastic AI Assistant is asked:

```
From employees, I want to see the 5 earliest employees (hire_date), I want to display only the month and the year that they were hired in and their employee number (emp_no). Format the date as e.g. "September 2019". Only show the query
```

it replies:

```
Here is the query to get the employee number and the formatted hire date for the 5 earliest employees by hire_date:

FROM employees
| KEEP emp_no, hire_date
| EVAL month_year = DATE_FORMAT(hire_date, "MMMM YYYY")
| SORT hire_date
| LIMIT 5
```

Per the screenshot below:

![ESQL_query_via_langchain_agents_and_tools](https://github.com/elastic/kibana/assets/4459398/c5cc75da-f7aa-4a12-9078-ed531f3463e7)

The `verbose: true` output from LangChain logged to the console reveals that the prompt sent to the LLM includes text like the following:

```
Assistant can ask the user to use tools to look up information that may be helpful in answering the users original question. The tools the human can use are:\\n\\nesql-language-knowledge-base: Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.
```

along with instructions for "calling" the tool like a function.

The debugging output also reveals the agent selecting the tool, and returning results from ESLR:

```
[agent/action] [1:chain:AgentExecutor] Agent selected action: {
  "tool": "esql-language-knowledge-base",
  "toolInput": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.",
  "log": "```json\n{\n    \"action\": \"esql-language-knowledge-base\",\n    \"action_input\": \"Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\"\n}\n```"
}
[tool/start] [1:chain:AgentExecutor > 4:tool:ChainTool] Entering Tool run with input: "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
[chain/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain] Entering Chain run with input: {
  "query": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
}
[retriever/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 6:retriever:VectorStoreRetriever] Entering Retriever run with input: {
  "query": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
}
[retriever/end] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 6:retriever:VectorStoreRetriever] [115ms] Exiting Retriever run with output: {
  "documents": [
    {
      "pageContent": "[[esql-date_format]]\n=== `DATE_FORMAT`\nReturns a string representation of a date in the provided format. If no format\nis specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.\n\n[source,esql]\n----\nFROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(hire_date, \"YYYY-MM-dd\")\n----\n",
```

The documents containing `ES|QL` examples, retrieved from ELSER, are sent back to the LLM to answer the original question, per the abridged output below:

```
[llm/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain > 8:chain:LLMChain > 9:llm:ActionsClientLlm] Entering LLM run with input: {
  "prompts": [
    "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n\n[[esql-date_format]]\n=== `DATE_FORMAT`\nReturns a string representation of a date in the provided format. If no format\nis specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.\n\n[source,esql]\n----\nFROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(hire_date, \"YYYY-MM-dd\")\n----\n\n\n[[esql-date_trunc]]\n=== `DATE_TRUNC`\nRounds down a date to the closest interval. Intervals can be expressed using the\n<<esql-timespan-literals,timespan literal syntax>>.\n\n[source,esql]\n----\nFROM employees\n| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\n| STATS count(emp_no) BY year_hired\n| SORT year_hired\n----\n\n\n[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index,
```

### Complete (verbose) LangChain output from the example

The following `verbose: true` output from LangChain below was produced via the example in the previous section:

```
[chain/start] [1:chain:AgentExecutor] Entering Chain run with input: {
  "input": "\n\n\n\nFrom employees, I want to see the 5 earliest employees (hire_date), I want to display only the month and the year that they were hired in and their employee number (emp_no). Format the date as e.g. \"September 2019\". Only show the query",
  "chat_history": []
}
[chain/start] [1:chain:AgentExecutor > 2:chain:LLMChain] Entering Chain run with input: {
  "input": "\n\n\n\nFrom employees, I want to see the 5 earliest employees (hire_date), I want to display only the month and the year that they were hired in and their employee number (emp_no). Format the date as e.g. \"September 2019\". Only show the query",
  "chat_history": [],
  "agent_scratchpad": [],
  "stop": [
    "Observation:"
  ]
}
[llm/start] [1:chain:AgentExecutor > 2:chain:LLMChain > 3:llm:ActionsClientLlm] Entering LLM run with input: {
  "prompts": [
    "[{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"SystemMessage\"],\"kwargs\":{\"content\":\"Assistant is a large language model trained by OpenAI.\\n\\nAssistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\\n\\nAssistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\\n\\nOverall, Assistant is a powerful system that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist. However, above all else, all responses must adhere to the format of RESPONSE FORMAT INSTRUCTIONS.\",\"additional_kwargs\":{}}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"HumanMessage\"],\"kwargs\":{\"content\":\"TOOLS\\n------\\nAssistant can ask the user to use tools to look up information that may be helpful in answering the users original question. The tools the human can use are:\\n\\nesql-language-knowledge-base: Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.\\n\\nRESPONSE FORMAT INSTRUCTIONS\\n----------------------------\\n\\nOutput a JSON markdown code snippet containing a valid JSON object in one of two formats:\\n\\n**Option 1:**\\nUse this if you want the human to use a tool.\\nMarkdown code snippet formatted in the following schema:\\n\\n```json\\n{\\n    \\\"action\\\": string, // The action to take. Must be one of [esql-language-knowledge-base]\\n    \\\"action_input\\\": string // The input to the action. May be a stringified object.\\n}\\n```\\n\\n**Option #2:**\\nUse this if you want to respond directly and conversationally to the human. Markdown code snippet formatted in the following schema:\\n\\n```json\\n{\\n    \\\"action\\\": \\\"Final Answer\\\",\\n    \\\"action_input\\\": string // You should put what you want to return to use here and make sure to use valid json newline characters.\\n}\\n```\\n\\nFor both options, remember to always include the surrounding markdown code snippet delimiters (begin with \\\"```json\\\" and end with \\\"```\\\")!\\n\\n\\nUSER'S INPUT\\n--------------------\\nHere is the user's input (remember to respond with a markdown code snippet of a json blob with a single action, and NOTHING else):\\n\\n\\n\\n\\n\\nFrom employees, I want to see the 5 earliest employees (hire_date), I want to display only the month and the year that they were hired in and their employee number (emp_no). Format the date as e.g. \\\"September 2019\\\". Only show the query\",\"additional_kwargs\":{}}}]"
  ]
}
[llm/end] [1:chain:AgentExecutor > 2:chain:LLMChain > 3:llm:ActionsClientLlm] [3.08s] Exiting LLM run with output: {
  "generations": [
    [
      {
        "text": "```json\n{\n    \"action\": \"esql-language-knowledge-base\",\n    \"action_input\": \"Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\"\n}\n```"
      }
    ]
  ]
}
[chain/end] [1:chain:AgentExecutor > 2:chain:LLMChain] [3.09s] Exiting Chain run with output: {
  "text": "```json\n{\n    \"action\": \"esql-language-knowledge-base\",\n    \"action_input\": \"Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\"\n}\n```"
}
[agent/action] [1:chain:AgentExecutor] Agent selected action: {
  "tool": "esql-language-knowledge-base",
  "toolInput": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.",
  "log": "```json\n{\n    \"action\": \"esql-language-knowledge-base\",\n    \"action_input\": \"Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\"\n}\n```"
}
[tool/start] [1:chain:AgentExecutor > 4:tool:ChainTool] Entering Tool run with input: "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
[chain/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain] Entering Chain run with input: {
  "query": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
}
[retriever/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 6:retriever:VectorStoreRetriever] Entering Retriever run with input: {
  "query": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
}
[retriever/end] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 6:retriever:VectorStoreRetriever] [115ms] Exiting Retriever run with output: {
  "documents": [
    {
      "pageContent": "[[esql-date_format]]\n=== `DATE_FORMAT`\nReturns a string representation of a date in the provided format. If no format\nis specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.\n\n[source,esql]\n----\nFROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(hire_date, \"YYYY-MM-dd\")\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/functions/date_format.asciidoc"
      }
    },
    {
      "pageContent": "[[esql-date_trunc]]\n=== `DATE_TRUNC`\nRounds down a date to the closest interval. Intervals can be expressed using the\n<<esql-timespan-literals,timespan literal syntax>>.\n\n[source,esql]\n----\nFROM employees\n| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\n| STATS count(emp_no) BY year_hired\n| SORT year_hired\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/functions/date_trunc.asciidoc"
      }
    },
    {
      "pageContent": "[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index, or alias. Each row in the resulting table represents a\ndocument. Each column corresponds to a field, and can be accessed by the name\nof that field.\n\n[source,esql]\n----\nFROM employees\n----\n\nYou can use <<api-date-math-index-names,date math>> to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM <logs-{now/d}>\n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/source_commands/from.asciidoc"
      }
    },
    {
      "pageContent": "[[esql-where]]\n=== `WHERE`\n\nUse `WHERE` to produce a table that contains all the rows from the input table\nfor which the provided condition evaluates to `true`:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=where]\n----\n\nWhich, if `still_hired` is a boolean field, can be simplified to:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereBoolean]\n----\n\n[discrete]\n==== Operators\n\nRefer to <<esql-operators>> for an overview of the supported operators.\n\n[discrete]\n==== Functions\n`WHERE` supports various functions for calculating values. Refer to\n<<esql-functions,Functions>> for more information.\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereFunction]\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/processing_commands/where.asciidoc"
      }
    }
  ]
}
[chain/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain] Entering Chain run with input: {
  "question": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.",
  "input_documents": [
    {
      "pageContent": "[[esql-date_format]]\n=== `DATE_FORMAT`\nReturns a string representation of a date in the provided format. If no format\nis specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.\n\n[source,esql]\n----\nFROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(hire_date, \"YYYY-MM-dd\")\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/functions/date_format.asciidoc"
      }
    },
    {
      "pageContent": "[[esql-date_trunc]]\n=== `DATE_TRUNC`\nRounds down a date to the closest interval. Intervals can be expressed using the\n<<esql-timespan-literals,timespan literal syntax>>.\n\n[source,esql]\n----\nFROM employees\n| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\n| STATS count(emp_no) BY year_hired\n| SORT year_hired\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/functions/date_trunc.asciidoc"
      }
    },
    {
      "pageContent": "[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index, or alias. Each row in the resulting table represents a\ndocument. Each column corresponds to a field, and can be accessed by the name\nof that field.\n\n[source,esql]\n----\nFROM employees\n----\n\nYou can use <<api-date-math-index-names,date math>> to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM <logs-{now/d}>\n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/source_commands/from.asciidoc"
      }
    },
    {
      "pageContent": "[[esql-where]]\n=== `WHERE`\n\nUse `WHERE` to produce a table that contains all the rows from the input table\nfor which the provided condition evaluates to `true`:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=where]\n----\n\nWhich, if `still_hired` is a boolean field, can be simplified to:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereBoolean]\n----\n\n[discrete]\n==== Operators\n\nRefer to <<esql-operators>> for an overview of the supported operators.\n\n[discrete]\n==== Functions\n`WHERE` supports various functions for calculating values. Refer to\n<<esql-functions,Functions>> for more information.\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereFunction]\n----\n",
      "metadata": {
        "source": "/Users/andrew.goldstein/Projects/forks/spong/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/docs/processing_commands/where.asciidoc"
      }
    }
  ],
  "query": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'."
}
[chain/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain > 8:chain:LLMChain] Entering Chain run with input: {
  "question": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.",
  "query": "Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.",
  "context": "[[esql-date_format]]\n=== `DATE_FORMAT`\nReturns a string representation of a date in the provided format. If no format\nis specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.\n\n[source,esql]\n----\nFROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(hire_date, \"YYYY-MM-dd\")\n----\n\n\n[[esql-date_trunc]]\n=== `DATE_TRUNC`\nRounds down a date to the closest interval. Intervals can be expressed using the\n<<esql-timespan-literals,timespan literal syntax>>.\n\n[source,esql]\n----\nFROM employees\n| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\n| STATS count(emp_no) BY year_hired\n| SORT year_hired\n----\n\n\n[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index, or alias. Each row in the resulting table represents a\ndocument. Each column corresponds to a field, and can be accessed by the name\nof that field.\n\n[source,esql]\n----\nFROM employees\n----\n\nYou can use <<api-date-math-index-names,date math>> to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM <logs-{now/d}>\n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n\n\n[[esql-where]]\n=== `WHERE`\n\nUse `WHERE` to produce a table that contains all the rows from the input table\nfor which the provided condition evaluates to `true`:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=where]\n----\n\nWhich, if `still_hired` is a boolean field, can be simplified to:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereBoolean]\n----\n\n[discrete]\n==== Operators\n\nRefer to <<esql-operators>> for an overview of the supported operators.\n\n[discrete]\n==== Functions\n`WHERE` supports various functions for calculating values. Refer to\n<<esql-functions,Functions>> for more information.\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereFunction]\n----\n"
}
[llm/start] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain > 8:chain:LLMChain > 9:llm:ActionsClientLlm] Entering LLM run with input: {
  "prompts": [
    "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n\n[[esql-date_format]]\n=== `DATE_FORMAT`\nReturns a string representation of a date in the provided format. If no format\nis specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.\n\n[source,esql]\n----\nFROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(hire_date, \"YYYY-MM-dd\")\n----\n\n\n[[esql-date_trunc]]\n=== `DATE_TRUNC`\nRounds down a date to the closest interval. Intervals can be expressed using the\n<<esql-timespan-literals,timespan literal syntax>>.\n\n[source,esql]\n----\nFROM employees\n| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\n| STATS count(emp_no) BY year_hired\n| SORT year_hired\n----\n\n\n[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index, or alias. Each row in the resulting table represents a\ndocument. Each column corresponds to a field, and can be accessed by the name\nof that field.\n\n[source,esql]\n----\nFROM employees\n----\n\nYou can use <<api-date-math-index-names,date math>> to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM <logs-{now/d}>\n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n\n\n[[esql-where]]\n=== `WHERE`\n\nUse `WHERE` to produce a table that contains all the rows from the input table\nfor which the provided condition evaluates to `true`:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=where]\n----\n\nWhich, if `still_hired` is a boolean field, can be simplified to:\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereBoolean]\n----\n\n[discrete]\n==== Operators\n\nRefer to <<esql-operators>> for an overview of the supported operators.\n\n[discrete]\n==== Functions\n`WHERE` supports various functions for calculating values. Refer to\n<<esql-functions,Functions>> for more information.\n\n[source,esql]\n----\ninclude::{esql-specs}/docs.csv-spec[tag=whereFunction]\n----\n\n\nQuestion: Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\nHelpful Answer:"
  ]
}
[llm/end] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain > 8:chain:LLMChain > 9:llm:ActionsClientLlm] [2.23s] Exiting LLM run with output: {
  "generations": [
    [
      {
        "text": "FROM employees\n| KEEP emp_no, hire_date\n| EVAL month_year = DATE_FORMAT(hire_date, \"MMMM YYYY\")\n| SORT hire_date\n| LIMIT 5"
      }
    ]
  ]
}
[chain/end] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain > 8:chain:LLMChain] [2.23s] Exiting Chain run with output: {
  "text": "FROM employees\n| KEEP emp_no, hire_date\n| EVAL month_year = DATE_FORMAT(hire_date, \"MMMM YYYY\")\n| SORT hire_date\n| LIMIT 5"
}
[chain/end] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain > 7:chain:StuffDocumentsChain] [2.23s] Exiting Chain run with output: {
  "text": "FROM employees\n| KEEP emp_no, hire_date\n| EVAL month_year = DATE_FORMAT(hire_date, \"MMMM YYYY\")\n| SORT hire_date\n| LIMIT 5"
}
[chain/end] [1:chain:AgentExecutor > 4:tool:ChainTool > 5:chain:RetrievalQAChain] [2.35s] Exiting Chain run with output: {
  "text": "FROM employees\n| KEEP emp_no, hire_date\n| EVAL month_year = DATE_FORMAT(hire_date, \"MMMM YYYY\")\n| SORT hire_date\n| LIMIT 5"
}
[tool/end] [1:chain:AgentExecutor > 4:tool:ChainTool] [2.35s] Exiting Tool run with output: "FROM employees
| KEEP emp_no, hire_date
| EVAL month_year = DATE_FORMAT(hire_date, "MMMM YYYY")
| SORT hire_date
| LIMIT 5"
[chain/start] [1:chain:AgentExecutor > 10:chain:LLMChain] Entering Chain run with input: {
  "input": "\n\n\n\nFrom employees, I want to see the 5 earliest employees (hire_date), I want to display only the month and the year that they were hired in and their employee number (emp_no). Format the date as e.g. \"September 2019\". Only show the query",
  "chat_history": [],
  "agent_scratchpad": [
    {
      "lc": 1,
      "type": "constructor",
      "id": [
        "langchain",
        "schema",
        "AIMessage"
      ],
      "kwargs": {
        "content": "```json\n{\n    \"action\": \"esql-language-knowledge-base\",\n    \"action_input\": \"Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\"\n}\n```",
        "additional_kwargs": {}
      }
    },
    {
      "lc": 1,
      "type": "constructor",
      "id": [
        "langchain",
        "schema",
        "HumanMessage"
      ],
      "kwargs": {
        "content": "TOOL RESPONSE:\n---------------------\nFROM employees\n| KEEP emp_no, hire_date\n| EVAL month_year = DATE_FORMAT(hire_date, \"MMMM YYYY\")\n| SORT hire_date\n| LIMIT 5\n\nUSER'S INPUT\n--------------------\n\nOkay, so what is the response to my last comment? If using information obtained from the tools you must mention it explicitly without mentioning the tool names - I have forgotten all TOOL RESPONSES! Remember to respond with a markdown code snippet of a json blob with a single action, and NOTHING else.",
        "additional_kwargs": {}
      }
    }
  ],
  "stop": [
    "Observation:"
  ]
}
[llm/start] [1:chain:AgentExecutor > 10:chain:LLMChain > 11:llm:ActionsClientLlm] Entering LLM run with input: {
  "prompts": [
    "[{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"SystemMessage\"],\"kwargs\":{\"content\":\"Assistant is a large language model trained by OpenAI.\\n\\nAssistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\\n\\nAssistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\\n\\nOverall, Assistant is a powerful system that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist. However, above all else, all responses must adhere to the format of RESPONSE FORMAT INSTRUCTIONS.\",\"additional_kwargs\":{}}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"HumanMessage\"],\"kwargs\":{\"content\":\"TOOLS\\n------\\nAssistant can ask the user to use tools to look up information that may be helpful in answering the users original question. The tools the human can use are:\\n\\nesql-language-knowledge-base: Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.\\n\\nRESPONSE FORMAT INSTRUCTIONS\\n----------------------------\\n\\nOutput a JSON markdown code snippet containing a valid JSON object in one of two formats:\\n\\n**Option 1:**\\nUse this if you want the human to use a tool.\\nMarkdown code snippet formatted in the following schema:\\n\\n```json\\n{\\n    \\\"action\\\": string, // The action to take. Must be one of [esql-language-knowledge-base]\\n    \\\"action_input\\\": string // The input to the action. May be a stringified object.\\n}\\n```\\n\\n**Option #2:**\\nUse this if you want to respond directly and conversationally to the human. Markdown code snippet formatted in the following schema:\\n\\n```json\\n{\\n    \\\"action\\\": \\\"Final Answer\\\",\\n    \\\"action_input\\\": string // You should put what you want to return to use here and make sure to use valid json newline characters.\\n}\\n```\\n\\nFor both options, remember to always include the surrounding markdown code snippet delimiters (begin with \\\"```json\\\" and end with \\\"```\\\")!\\n\\n\\nUSER'S INPUT\\n--------------------\\nHere is the user's input (remember to respond with a markdown code snippet of a json blob with a single action, and NOTHING else):\\n\\n\\n\\n\\n\\nFrom employees, I want to see the 5 earliest employees (hire_date), I want to display only the month and the year that they were hired in and their employee number (emp_no). Format the date as e.g. \\\"September 2019\\\". Only show the query\",\"additional_kwargs\":{}}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"AIMessage\"],\"kwargs\":{\"content\":\"```json\\n{\\n    \\\"action\\\": \\\"esql-language-knowledge-base\\\",\\n    \\\"action_input\\\": \\\"Display the 'emp_no', month and year of the 5 earliest employees by 'hire_date'. Format the date as 'Month Year'.\\\"\\n}\\n```\",\"additional_kwargs\":{}}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"HumanMessage\"],\"kwargs\":{\"content\":\"TOOL RESPONSE:\\n---------------------\\nFROM employees\\n| KEEP emp_no, hire_date\\n| EVAL month_year = DATE_FORMAT(hire_date, \\\"MMMM YYYY\\\")\\n| SORT hire_date\\n| LIMIT 5\\n\\nUSER'S INPUT\\n--------------------\\n\\nOkay, so what is the response to my last comment? If using information obtained from the tools you must mention it explicitly without mentioning the tool names - I have forgotten all TOOL RESPONSES! Remember to respond with a markdown code snippet of a json blob with a single action, and NOTHING else.\",\"additional_kwargs\":{}}}]"
  ]
}
[llm/end] [1:chain:AgentExecutor > 10:chain:LLMChain > 11:llm:ActionsClientLlm] [6.47s] Exiting LLM run with output: {
  "generations": [
    [
      {
        "text": "```json\n{\n    \"action\": \"Final Answer\",\n    \"action_input\": \"Here is the query to get the employee number and the formatted hire date for the 5 earliest employees by hire_date:\\n\\nFROM employees\\n| KEEP emp_no, hire_date\\n| EVAL month_year = DATE_FORMAT(hire_date, \\\"MMMM YYYY\\\")\\n| SORT hire_date\\n| LIMIT 5\"\n}\n```"
      }
    ]
  ]
}
[chain/end] [1:chain:AgentExecutor > 10:chain:LLMChain] [6.47s] Exiting Chain run with output: {
  "text": "```json\n{\n    \"action\": \"Final Answer\",\n    \"action_input\": \"Here is the query to get the employee number and the formatted hire date for the 5 earliest employees by hire_date:\\n\\nFROM employees\\n| KEEP emp_no, hire_date\\n| EVAL month_year = DATE_FORMAT(hire_date, \\\"MMMM YYYY\\\")\\n| SORT hire_date\\n| LIMIT 5\"\n}\n```"
}
[chain/end] [1:chain:AgentExecutor] [11.91s] Exiting Chain run with output: {
  "output": "Here is the query to get the employee number and the formatted hire date for the 5 earliest employees by hire_date:\n\nFROM employees\n| KEEP emp_no, hire_date\n| EVAL month_year = DATE_FORMAT(hire_date, \"MMMM YYYY\")\n| SORT hire_date\n| LIMIT 5"
}
```
---
 .../impl/assistant/api.test.tsx               | 84 +++++++++++++++++++
 .../impl/assistant/api.tsx                    |  4 +-
 .../impl/assistant/helpers.test.ts            | 43 +++++++++-
 .../impl/assistant/helpers.ts                 | 21 +++++
 .../execute_custom_llm_chain/index.test.ts    | 29 ++++---
 .../execute_custom_llm_chain/index.ts         | 39 +++++----
 .../post_actions_connector_execute.test.ts    |  2 +-
 .../routes/post_actions_connector_execute.ts  |  4 +-
 8 files changed, 195 insertions(+), 31 deletions(-)

diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx
index 65b8183b60a0b..2f46e99d12b07 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx
@@ -126,4 +126,88 @@ describe('fetchConnectorExecuteAction', () => {
 
     expect(result).toBe('Test response');
   });
+
+  it('returns the value of the action_input property when assistantLangChain is true, and `content` has properly prefixed and suffixed JSON with the action_input property', async () => {
+    const content = '```json\n{"action_input": "value from action_input"}\n```';
+
+    (mockHttp.fetch as jest.Mock).mockResolvedValue({
+      status: 'ok',
+      data: {
+        choices: [
+          {
+            message: {
+              content,
+            },
+          },
+        ],
+      },
+    });
+
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: true, // <-- requires response parsing
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    const result = await fetchConnectorExecuteAction(testProps);
+
+    expect(result).toBe('value from action_input');
+  });
+
+  it('returns the original content when assistantLangChain is true, and `content` has properly formatted JSON WITHOUT the action_input property', async () => {
+    const content = '```json\n{"some_key": "some value"}\n```';
+
+    (mockHttp.fetch as jest.Mock).mockResolvedValue({
+      status: 'ok',
+      data: {
+        choices: [
+          {
+            message: {
+              content,
+            },
+          },
+        ],
+      },
+    });
+
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: true, // <-- requires response parsing
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    const result = await fetchConnectorExecuteAction(testProps);
+
+    expect(result).toBe(content);
+  });
+
+  it('returns the original when assistantLangChain is true, and `content` is not JSON', async () => {
+    const content = 'plain text content';
+
+    (mockHttp.fetch as jest.Mock).mockResolvedValue({
+      status: 'ok',
+      data: {
+        choices: [
+          {
+            message: {
+              content,
+            },
+          },
+        ],
+      },
+    });
+
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: true, // <-- requires response parsing
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    const result = await fetchConnectorExecuteAction(testProps);
+
+    expect(result).toBe(content);
+  });
 });
diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx
index 511b5aa585af0..6d3452b6f7880 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx
@@ -12,6 +12,7 @@ import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser';
 import type { Conversation, Message } from '../assistant_context/types';
 import { API_ERROR } from './translations';
 import { MODEL_GPT_3_5_TURBO } from '../connectorland/models/model_selector/model_selector';
+import { getFormattedMessageContent } from './helpers';
 
 export interface FetchConnectorExecuteAction {
   assistantLangChain: boolean;
@@ -78,7 +79,8 @@ export const fetchConnectorExecuteAction = async ({
 
     if (data.choices && data.choices.length > 0 && data.choices[0].message.content) {
       const result = data.choices[0].message.content.trim();
-      return result;
+
+      return assistantLangChain ? getFormattedMessageContent(result) : result;
     } else {
       return API_ERROR;
     }
diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts
index 69bed887e730e..f2b89a07c319e 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts
@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { getDefaultConnector, getBlockBotConversation } from './helpers';
+import {
+  getBlockBotConversation,
+  getDefaultConnector,
+  getFormattedMessageContent,
+} from './helpers';
 import { enterpriseMessaging } from './use_conversation/sample_conversations';
 import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public';
 
@@ -190,4 +194,41 @@ describe('getBlockBotConversation', () => {
       expect(result).toBeUndefined();
     });
   });
+
+  describe('getFormattedMessageContent', () => {
+    it('returns the value of the action_input property when `content` has properly prefixed and suffixed JSON with the action_input property', () => {
+      const content = '```json\n{"action_input": "value from action_input"}\n```';
+
+      expect(getFormattedMessageContent(content)).toBe('value from action_input');
+    });
+
+    it('returns the original content when `content` has properly formatted JSON WITHOUT the action_input property', () => {
+      const content = '```json\n{"some_key": "some value"}\n```';
+      expect(getFormattedMessageContent(content)).toBe(content);
+    });
+
+    it('returns the original content when `content` has improperly formatted JSON', () => {
+      const content = '```json\n{"action_input": "value from action_input",}\n```'; // <-- the trailing comma makes it invalid
+
+      expect(getFormattedMessageContent(content)).toBe(content);
+    });
+
+    it('returns the original content when `content` is missing the prefix', () => {
+      const content = '{"action_input": "value from action_input"}\n```'; // <-- missing prefix
+
+      expect(getFormattedMessageContent(content)).toBe(content);
+    });
+
+    it('returns the original content when `content` is missing the suffix', () => {
+      const content = '```json\n{"action_input": "value from action_input"}'; // <-- missing suffix
+
+      expect(getFormattedMessageContent(content)).toBe(content);
+    });
+
+    it('returns the original content when `content` does NOT contain a JSON string', () => {
+      const content = 'plain text content';
+
+      expect(getFormattedMessageContent(content)).toBe(content);
+    });
+  });
 });
diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts
index b01c9001e8319..2b2c5b76851f7 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts
@@ -59,3 +59,24 @@ export const getDefaultConnector = (
   connectors: Array<ActionConnector<Record<string, unknown>, Record<string, unknown>>> | undefined
 ): ActionConnector<Record<string, unknown>, Record<string, unknown>> | undefined =>
   connectors?.length === 1 ? connectors[0] : undefined;
+
+/**
+ * When `content` is a JSON string, prefixed with "```json\n"
+ * and suffixed with "\n```", this function will attempt to parse it and return
+ * the `action_input` property if it exists.
+ */
+export const getFormattedMessageContent = (content: string): string => {
+  const formattedContentMatch = content.match(/```json\n([\s\S]+)\n```/);
+
+  if (formattedContentMatch) {
+    try {
+      const parsedContent = JSON.parse(formattedContentMatch[1]);
+
+      return parsedContent.action_input ?? content;
+    } catch {
+      // we don't want to throw an error here, so we'll fall back to the original content
+    }
+  }
+
+  return content;
+};
diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts
index be1adbc2e1ce4..67fb3859b9943 100644
--- a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts
+++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts
@@ -12,7 +12,7 @@ import { ResponseBody } from '../helpers';
 import { ActionsClientLlm } from '../llm/actions_client_llm';
 import { mockActionResultData } from '../../../__mocks__/action_result_data';
 import { langChainMessages } from '../../../__mocks__/lang_chain_messages';
-import { executeCustomLlmChain } from '.';
+import { callAgentExecutor } from '.';
 import { loggerMock } from '@kbn/logging-mocks';
 import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
 
@@ -23,11 +23,18 @@ const mockConversationChain = {
 };
 
 jest.mock('langchain/chains', () => ({
-  ConversationalRetrievalQAChain: {
+  RetrievalQAChain: {
     fromLLM: jest.fn().mockImplementation(() => mockConversationChain),
   },
 }));
 
+const mockCall = jest.fn();
+jest.mock('langchain/agents', () => ({
+  initializeAgentExecutorWithOptions: jest.fn().mockImplementation(() => ({
+    call: mockCall,
+  })),
+}));
+
 const mockConnectorId = 'mock-connector-id';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -42,7 +49,7 @@ const mockActions: ActionsPluginStart = {} as ActionsPluginStart;
 const mockLogger = loggerMock.create();
 const esClientMock = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
 
-describe('executeCustomLlmChain', () => {
+describe('callAgentExecutor', () => {
   beforeEach(() => {
     jest.clearAllMocks();
 
@@ -52,7 +59,7 @@ describe('executeCustomLlmChain', () => {
   });
 
   it('creates an instance of ActionsClientLlm with the expected context from the request', async () => {
-    await executeCustomLlmChain({
+    await callAgentExecutor({
       actions: mockActions,
       connectorId: mockConnectorId,
       esClient: esClientMock,
@@ -70,7 +77,7 @@ describe('executeCustomLlmChain', () => {
   });
 
   it('kicks off the chain with (only) the last message', async () => {
-    await executeCustomLlmChain({
+    await callAgentExecutor({
       actions: mockActions,
       connectorId: mockConnectorId,
       esClient: esClientMock,
@@ -79,15 +86,15 @@ describe('executeCustomLlmChain', () => {
       request: mockRequest,
     });
 
-    expect(mockConversationChain.call).toHaveBeenCalledWith({
-      question: '\n\nDo you know my name?',
+    expect(mockCall).toHaveBeenCalledWith({
+      input: '\n\nDo you know my name?',
     });
   });
 
   it('kicks off the chain with the expected message when langChainMessages has only one entry', async () => {
     const onlyOneMessage = [langChainMessages[0]];
 
-    await executeCustomLlmChain({
+    await callAgentExecutor({
       actions: mockActions,
       connectorId: mockConnectorId,
       esClient: esClientMock,
@@ -96,13 +103,13 @@ describe('executeCustomLlmChain', () => {
       request: mockRequest,
     });
 
-    expect(mockConversationChain.call).toHaveBeenCalledWith({
-      question: 'What is my name?',
+    expect(mockCall).toHaveBeenCalledWith({
+      input: 'What is my name?',
     });
   });
 
   it('returns the expected response body', async () => {
-    const result: ResponseBody = await executeCustomLlmChain({
+    const result: ResponseBody = await callAgentExecutor({
       actions: mockActions,
       connectorId: mockConnectorId,
       esClient: esClientMock,
diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts
index 5a65b1589b21e..b6a768ad69598 100644
--- a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts
+++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts
@@ -7,16 +7,18 @@
 
 import { ElasticsearchClient, KibanaRequest, Logger } from '@kbn/core/server';
 import type { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server';
+import { initializeAgentExecutorWithOptions } from 'langchain/agents';
+import { RetrievalQAChain } from 'langchain/chains';
 import { BufferMemory, ChatMessageHistory } from 'langchain/memory';
 import { BaseMessage } from 'langchain/schema';
+import { ChainTool, Tool } from 'langchain/tools';
 
-import { ConversationalRetrievalQAChain } from 'langchain/chains';
+import { ElasticsearchStore } from '../elasticsearch_store/elasticsearch_store';
 import { ResponseBody } from '../helpers';
 import { ActionsClientLlm } from '../llm/actions_client_llm';
-import { ElasticsearchStore } from '../elasticsearch_store/elasticsearch_store';
 import { KNOWLEDGE_BASE_INDEX_PATTERN } from '../../../routes/knowledge_base/constants';
 
-export const executeCustomLlmChain = async ({
+export const callAgentExecutor = async ({
   actions,
   connectorId,
   esClient,
@@ -34,31 +36,38 @@ export const executeCustomLlmChain = async ({
 }): Promise<ResponseBody> => {
   const llm = new ActionsClientLlm({ actions, connectorId, request, logger });
 
-  // Chat History Memory: in-memory memory, from client local storage, first message is the system prompt
   const pastMessages = langChainMessages.slice(0, -1); // all but the last message
   const latestMessage = langChainMessages.slice(-1); // the last message
+
   const memory = new BufferMemory({
     chatHistory: new ChatMessageHistory(pastMessages),
-    memoryKey: 'chat_history',
+    memoryKey: 'chat_history', // this is the key expected by https://github.com/langchain-ai/langchainjs/blob/a13a8969345b0f149c1ca4a120d63508b06c52a5/langchain/src/agents/initialize.ts#L166
+    inputKey: 'input',
+    outputKey: 'output',
+    returnMessages: true,
   });
 
   // ELSER backed ElasticsearchStore for Knowledge Base
   const esStore = new ElasticsearchStore(esClient, KNOWLEDGE_BASE_INDEX_PATTERN, logger);
+  const chain = RetrievalQAChain.fromLLM(llm, esStore.asRetriever());
+
+  const tools: Tool[] = [
+    new ChainTool({
+      name: 'esql-language-knowledge-base',
+      description:
+        'Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.',
+      chain,
+    }),
+  ];
 
-  // Chain w/ chat history memory and knowledge base retriever
-  const chain = ConversationalRetrievalQAChain.fromLLM(llm, esStore.asRetriever(), {
+  const executor = await initializeAgentExecutorWithOptions(tools, llm, {
+    agentType: 'chat-conversational-react-description',
     memory,
-    // See `qaChainOptions` from https://js.langchain.com/docs/modules/chains/popular/chat_vector_db
-    qaChainOptions: { type: 'stuff' },
+    verbose: false,
   });
-  await chain.call({ question: latestMessage[0].content });
 
-  // Chain w/ just knowledge base retriever
-  // const chain = RetrievalQAChain.fromLLM(llm, esStore.asRetriever());
-  // await chain.call({ query: latestMessage[0].content });
+  await executor.call({ input: latestMessage[0].content });
 
-  // The assistant (on the client side) expects the same response returned
-  // from the actions framework, so we need to return the same shape of data:
   return {
     connector_id: connectorId,
     data: llm.getActionResultData(), // the response from the actions framework
diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts
index 2e6709a6e33c2..57f2b25f5a65f 100644
--- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts
@@ -20,7 +20,7 @@ jest.mock('../lib/build_response', () => ({
 }));
 
 jest.mock('../lib/langchain/execute_custom_llm_chain', () => ({
-  executeCustomLlmChain: jest.fn().mockImplementation(
+  callAgentExecutor: jest.fn().mockImplementation(
     async ({
       connectorId,
     }: {
diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts
index 1043f68f0f9c1..bbb1c76e3e579 100644
--- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts
@@ -20,7 +20,7 @@ import {
   PostActionsConnectorExecutePathParams,
 } from '../schemas/post_actions_connector_execute';
 import { ElasticAssistantRequestHandlerContext } from '../types';
-import { executeCustomLlmChain } from '../lib/langchain/execute_custom_llm_chain';
+import { callAgentExecutor } from '../lib/langchain/execute_custom_llm_chain';
 
 export const postActionsConnectorExecuteRoute = (
   router: IRouter<ElasticAssistantRequestHandlerContext>
@@ -53,7 +53,7 @@ export const postActionsConnectorExecuteRoute = (
         // convert the assistant messages to LangChain messages:
         const langChainMessages = getLangChainMessages(assistantMessages);
 
-        const langChainResponseBody = await executeCustomLlmChain({
+        const langChainResponseBody = await callAgentExecutor({
           actions,
           connectorId,
           esClient,

From d941c4a5658769f823fa08fb801303bde8292ff9 Mon Sep 17 00:00:00 2001
From: Devon Thomson <devon.thomson@elastic.co>
Date: Mon, 25 Sep 2023 13:58:12 -0400
Subject: [PATCH 24/27] unskip serverless dashboard import tests (#167161)

Unskips the serverless Dashboard import test.
---
 .../test_suites/search/dashboards/import_dashboard.ts          | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts b/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts
index c935ab3f15f83..30d99d112e640 100644
--- a/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts
+++ b/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts
@@ -27,8 +27,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     'svlCommonPage',
   ]);
 
-  // Failing: See https://github.com/elastic/kibana/issues/166573
-  describe.skip('Importing an existing dashboard', () => {
+  describe('Importing an existing dashboard', () => {
     before(async () => {
       await PageObjects.svlCommonPage.login();
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional');

From f23f2f49dbba97350d7159c6d96d0fbb1ef25d84 Mon Sep 17 00:00:00 2001
From: Devon Thomson <devon.thomson@elastic.co>
Date: Mon, 25 Sep 2023 14:04:31 -0400
Subject: [PATCH 25/27] [Canvas] Remove Kui style sheet import (#167054)

removes import of the kui_light stylesheet from Canvas shareable runtime.
---
 x-pack/plugins/canvas/shareable_runtime/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/plugins/canvas/shareable_runtime/index.ts b/x-pack/plugins/canvas/shareable_runtime/index.ts
index aee57c3780503..475989494c574 100644
--- a/x-pack/plugins/canvas/shareable_runtime/index.ts
+++ b/x-pack/plugins/canvas/shareable_runtime/index.ts
@@ -9,4 +9,3 @@ export * from './api';
 import '@kbn/core-apps-server-internal/assets/legacy_light_theme.css';
 import '../public/style/index.scss';
 import '@elastic/eui/dist/eui_theme_light.css';
-import '@kbn/ui-framework/dist/kui_light.css';

From c20d177a036be73d7b1180dc17e644afa260994f Mon Sep 17 00:00:00 2001
From: Kyle Pollich <kyle.pollich@elastic.co>
Date: Mon, 25 Sep 2023 14:05:03 -0400
Subject: [PATCH 26/27] [Fleet] Increase package install max timeout + add
 concurrency control to rollovers (#166775)

Fixes https://github.com/elastic/kibana/issues/166761
Ref https://github.com/elastic/kibana/issues/162772

## Summary

- Increase overall timeout for waiting to retry "stuck" installations
from 1 minute to 30 minutes
- Add `pMap` concurrency control limiting concurrent `putMapping` +
`rollover` requests to mitigate ES load

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 x-pack/plugins/fleet/common/constants/epm.ts  |   2 +-
 .../epm/elasticsearch/template/template.ts    |  25 ++-
 .../epm/packages/_install_package.test.ts     | 144 +++++++++++++++++-
 .../services/epm/packages/_install_package.ts |  32 ++--
 4 files changed, 179 insertions(+), 24 deletions(-)

diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts
index 52cb24271afa5..3548fee93fbf2 100644
--- a/x-pack/plugins/fleet/common/constants/epm.ts
+++ b/x-pack/plugins/fleet/common/constants/epm.ts
@@ -9,7 +9,7 @@ import { ElasticsearchAssetType, KibanaAssetType } from '../types/models';
 
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
-export const MAX_TIME_COMPLETE_INSTALL = 60000;
+export const MAX_TIME_COMPLETE_INSTALL = 30 * 60 * 1000; // 30 minutes
 
 export const FLEET_SYSTEM_PACKAGE = 'system';
 export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent';
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
index 3e9363fa9828f..5682749d7e381 100644
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -11,6 +11,8 @@ import type {
   MappingTypeMapping,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import pMap from 'p-map';
+
 import type { Field, Fields } from '../../fields/field';
 import type {
   RegistryDataStream,
@@ -729,15 +731,22 @@ const updateAllDataStreams = async (
   esClient: ElasticsearchClient,
   logger: Logger
 ): Promise<void> => {
-  const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => {
-    return updateExistingDataStream({
-      esClient,
-      logger,
-      dataStreamName: templateEntry.dataStreamName,
-    });
-  });
-  await Promise.all(updatedataStreamPromises);
+  await pMap(
+    indexNameWithTemplates,
+    (templateEntry) => {
+      return updateExistingDataStream({
+        esClient,
+        logger,
+        dataStreamName: templateEntry.dataStreamName,
+      });
+    },
+    {
+      // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
+      concurrency: 20,
+    }
+  );
 };
+
 const updateExistingDataStream = async ({
   dataStreamName,
   esClient,
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts
index 3c308b8e85b0a..b7fe0d95310ef 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts
@@ -5,11 +5,21 @@
  * 2.0.
  */
 
-import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server';
+import type {
+  SavedObjectsClientContract,
+  ElasticsearchClient,
+  SavedObject,
+} from '@kbn/core/server';
 import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
 import { loggerMock } from '@kbn/logging-mocks';
 import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
 
+import { ConcurrentInstallOperationError } from '../../../errors';
+
+import type { Installation } from '../../../../common';
+
+import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common';
+
 import { appContextService } from '../../app_context';
 import { createAppContextStartContractMock } from '../../../mocks';
 import { saveArchiveEntries } from '../archive/storage';
@@ -29,7 +39,9 @@ jest.mock('../elasticsearch/datastream_ilm/install');
 import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
 import { installKibanaAssetsAndReferences } from '../kibana/assets/install';
 
-import { installIndexTemplatesAndPipelines } from './install';
+import { MAX_TIME_COMPLETE_INSTALL } from '../../../../common/constants';
+
+import { installIndexTemplatesAndPipelines, restartInstallation } from './install';
 
 import { _installPackage } from './_install_package';
 
@@ -69,9 +81,7 @@ describe('_installPackage', () => {
     jest.mocked(saveArchiveEntries).mockResolvedValue({
       saved_objects: [],
     });
-  });
-  afterEach(async () => {
-    appContextService.stop();
+    jest.mocked(restartInstallation).mockReset();
   });
   it('handles errors from  installKibanaAssets', async () => {
     // force errors from this function
@@ -226,4 +236,128 @@ describe('_installPackage', () => {
     expect(installILMPolicy).toBeCalled();
     expect(installIlmForDataStream).toBeCalled();
   });
+
+  describe('when package is stuck in `installing`', () => {
+    afterEach(() => {});
+    const mockInstalledPackageSo: SavedObject<Installation> = {
+      id: 'mocked-package',
+      attributes: {
+        name: 'test-package',
+        version: '1.0.0',
+        install_status: 'installing',
+        install_version: '1.0.0',
+        install_started_at: new Date().toISOString(),
+        install_source: 'registry',
+        verification_status: 'verified',
+        installed_kibana: [] as any,
+        installed_es: [] as any,
+        es_index_patterns: {},
+      },
+      type: PACKAGES_SAVED_OBJECT_TYPE,
+      references: [],
+    };
+
+    beforeEach(() => {
+      appContextService.start(
+        createAppContextStartContractMock({
+          internal: {
+            disableILMPolicies: true,
+            disableProxies: false,
+            fleetServerStandalone: false,
+            onlyAllowAgentUpgradeToKnownVersions: false,
+            registry: {
+              kibanaVersionCheckEnabled: true,
+              capabilities: [],
+            },
+          },
+        })
+      );
+    });
+
+    describe('timeout reached', () => {
+      it('restarts installation', async () => {
+        await _installPackage({
+          savedObjectsClient: soClient,
+          // @ts-ignore
+          savedObjectsImporter: jest.fn(),
+          esClient,
+          logger: loggerMock.create(),
+          paths: [],
+          packageInfo: {
+            name: mockInstalledPackageSo.attributes.name,
+            version: mockInstalledPackageSo.attributes.version,
+            title: mockInstalledPackageSo.attributes.name,
+          } as any,
+          installedPkg: {
+            ...mockInstalledPackageSo,
+            attributes: {
+              ...mockInstalledPackageSo.attributes,
+              install_started_at: new Date(
+                Date.now() - MAX_TIME_COMPLETE_INSTALL * 2
+              ).toISOString(),
+            },
+          },
+        });
+
+        expect(restartInstallation).toBeCalled();
+      });
+    });
+
+    describe('timeout not reached', () => {
+      describe('force flag not provided', () => {
+        it('throws concurrent installation error if force flag is not provided', async () => {
+          expect(
+            _installPackage({
+              savedObjectsClient: soClient,
+              // @ts-ignore
+              savedObjectsImporter: jest.fn(),
+              esClient,
+              logger: loggerMock.create(),
+              paths: [],
+              packageInfo: {
+                name: mockInstalledPackageSo.attributes.name,
+                version: mockInstalledPackageSo.attributes.version,
+                title: mockInstalledPackageSo.attributes.name,
+              } as any,
+              installedPkg: {
+                ...mockInstalledPackageSo,
+                attributes: {
+                  ...mockInstalledPackageSo.attributes,
+                  install_started_at: new Date(Date.now() - 1000).toISOString(),
+                },
+              },
+            })
+          ).rejects.toThrowError(ConcurrentInstallOperationError);
+        });
+      });
+
+      describe('force flag provided', () => {
+        it('restarts installation', async () => {
+          await _installPackage({
+            savedObjectsClient: soClient,
+            // @ts-ignore
+            savedObjectsImporter: jest.fn(),
+            esClient,
+            logger: loggerMock.create(),
+            paths: [],
+            packageInfo: {
+              name: mockInstalledPackageSo.attributes.name,
+              version: mockInstalledPackageSo.attributes.version,
+              title: mockInstalledPackageSo.attributes.name,
+            } as any,
+            installedPkg: {
+              ...mockInstalledPackageSo,
+              attributes: {
+                ...mockInstalledPackageSo.attributes,
+                install_started_at: new Date(Date.now() - 1000).toISOString(),
+              },
+            },
+            force: true,
+          });
+
+          expect(restartInstallation).toBeCalled();
+        });
+      });
+    });
+  });
 });
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
index 337cf59bbd613..3bfc74bf68968 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
@@ -99,18 +99,30 @@ export async function _installPackage({
   try {
     // if some installation already exists
     if (installedPkg) {
+      const isStatusInstalling = installedPkg.attributes.install_status === 'installing';
+      const hasExceededTimeout =
+        Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
+        MAX_TIME_COMPLETE_INSTALL;
+
       // if the installation is currently running, don't try to install
       // instead, only return already installed assets
-      if (
-        installedPkg.attributes.install_status === 'installing' &&
-        Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
-          MAX_TIME_COMPLETE_INSTALL
-      ) {
-        throw new ConcurrentInstallOperationError(
-          `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${
-            pkgVersion || 'unknown'
-          } detected, aborting.`
-        );
+      if (isStatusInstalling && hasExceededTimeout) {
+        // If this is a forced installation, ignore the timeout and restart the installation anyway
+        if (force) {
+          await restartInstallation({
+            savedObjectsClient,
+            pkgName,
+            pkgVersion,
+            installSource,
+            verificationResult,
+          });
+        } else {
+          throw new ConcurrentInstallOperationError(
+            `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${
+              pkgVersion || 'unknown'
+            } detected, aborting.`
+          );
+        }
       } else {
         // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL
         // (it might be stuck) update the saved object and proceed

From da2695fef32513e43e9ef20e3faf4fdcf6ca463b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= <contact@patrykkopycinski.com>
Date: Mon, 25 Sep 2023 18:34:47 +0000
Subject: [PATCH 27/27] [security_solution] Skip search_filter.cy.ts (#167177)

## Summary

<img width="984" alt="image"
src="https://github.com/elastic/kibana/assets/5188868/14775f02-5370-4e8f-90c0-267049c6ece4">
---
 .../e2e/investigations/timelines/discover/search_filter.cy.ts  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts
index afedab5dbb094..c48fa80baad46 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts
@@ -38,7 +38,8 @@ const INITIAL_START_DATE = 'Jan 18, 2021 @ 20:33:29.186';
 const INITIAL_END_DATE = 'Jan 19, 2024 @ 20:33:29.186';
 const NEW_START_DATE = 'Jan 18, 2023 @ 20:33:29.186';
 
-describe(
+// Failing: See https://github.com/elastic/kibana/issues/167186
+describe.skip(
   'Basic discover search and filter operations',
   {
     env: { ftrConfig: { enableExperimental: ['discoverInTimeline'] } },