From cbc7c067a2388b51238b8174f64d9bbdd50ea1b3 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Mon, 11 Nov 2024 11:27:20 +0100
Subject: [PATCH 01/37] initial commit

---
 .../kbn-entities-schema/src/schema/entity.ts  |   7 +
 x-pack/plugins/entity_manager/kibana.jsonc    |  26 +-
 .../entity_manager/public/application.tsx     | 134 ++++++++
 .../public/context/plugin_context.ts          |  21 ++
 .../entity_manager/public/hooks/use_kibana.ts |  22 ++
 .../public/hooks/use_plugin_context.ts        |  19 ++
 .../public/pages/overview/index.tsx           | 320 ++++++++++++++++++
 .../plugins/entity_manager/public/plugin.ts   |  66 +++-
 .../plugins/entity_manager/public/routes.tsx  |  27 ++
 x-pack/plugins/entity_manager/public/types.ts |  26 ++
 .../server/lib/entity_client.ts               |  47 ++-
 .../server/lib/queries/index.ts               |  39 +++
 .../server/lib/queries/utils.ts               |  48 +++
 .../server/routes/entities/index.ts           |   3 +
 .../server/routes/entities/search.ts          |  85 +++++
 .../entity_definition_template.ts             |  38 +++
 16 files changed, 919 insertions(+), 9 deletions(-)
 create mode 100644 x-pack/plugins/entity_manager/public/application.tsx
 create mode 100644 x-pack/plugins/entity_manager/public/context/plugin_context.ts
 create mode 100644 x-pack/plugins/entity_manager/public/hooks/use_kibana.ts
 create mode 100644 x-pack/plugins/entity_manager/public/hooks/use_plugin_context.ts
 create mode 100644 x-pack/plugins/entity_manager/public/pages/overview/index.tsx
 create mode 100644 x-pack/plugins/entity_manager/public/routes.tsx
 create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/index.ts
 create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/utils.ts
 create mode 100644 x-pack/plugins/entity_manager/server/routes/entities/search.ts
 create mode 100644 x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
index 9ab02e0931d9c..bb1b45634a4d3 100644
--- a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
+++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
@@ -23,6 +23,13 @@ export interface MetadataRecord {
   [key: string]: string[] | MetadataRecord | string;
 }
 
+export interface Entity {
+  'entity.id': string;
+  'entity.last_seen_timestamp': string;
+  'entity.type': string;
+  [metadata: string]: any;
+}
+
 const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
 type Literal = z.infer<typeof literalSchema>;
 type Metadata = Literal | { [key: string]: Metadata } | Metadata[];
diff --git a/x-pack/plugins/entity_manager/kibana.jsonc b/x-pack/plugins/entity_manager/kibana.jsonc
index efd6d3a445b3f..d9e4b82439ab2 100644
--- a/x-pack/plugins/entity_manager/kibana.jsonc
+++ b/x-pack/plugins/entity_manager/kibana.jsonc
@@ -6,9 +6,31 @@
   "plugin": {
     "id": "entityManager",
     "configPath": ["xpack", "entityManager"],
-    "requiredPlugins": ["security", "encryptedSavedObjects", "licensing"],
     "browser": true,
     "server": true,
-    "requiredBundles": []
+    "requiredPlugins": [
+      "data",
+      "dataViews",
+      "dataViewEditor",
+      "dataViewFieldEditor",
+      "security",
+      "encryptedSavedObjects",
+      "lens",
+      "observability",
+      "observabilityShared",
+      "embeddable",
+      "presentationUtil",
+      "usageCollection",
+      "licensing",
+      "transform"
+    ],
+    "optionalPlugins": [
+      "cloud",
+      "serverless"
+    ],
+    "requiredBundles": [
+      "kibanaReact",
+      "kibanaUtils"
+    ]
   }
 }
diff --git a/x-pack/plugins/entity_manager/public/application.tsx b/x-pack/plugins/entity_manager/public/application.tsx
new file mode 100644
index 0000000000000..3680aed1dee59
--- /dev/null
+++ b/x-pack/plugins/entity_manager/public/application.tsx
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { AppMountParameters, APP_WRAPPER_CLASS, CoreStart } from '@kbn/core/public';
+import { PerformanceContextProvider } from '@kbn/ebt-tools';
+import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common';
+import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
+import { Storage } from '@kbn/kibana-utils-plugin/public';
+import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public';
+import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
+import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme';
+import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import React from 'react';
+import ReactDOM from 'react-dom';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
+import { Route, Router, Routes } from '@kbn/shared-ux-router';
+import { PluginContext } from './context/plugin_context';
+import { EntityManagerPluginStart } from './types';
+import { getRoutes } from './routes';
+import { EntityClient } from './lib/entity_client';
+import { EntityManagerOverviewPage } from './pages/overview';
+
+function App() {
+  const routes = getRoutes();
+  return (
+    <Routes>
+      {Object.keys(routes).map((path) => {
+        const { handler, exact } = routes[path];
+        const Wrapper = () => handler();
+        return <Route key={path} path={path} exact={exact} component={Wrapper} />;
+      })}
+    </Routes>
+  );
+}
+
+export function renderApp({
+  core,
+  plugins,
+  appMountParameters,
+  ObservabilityPageTemplate,
+  usageCollection,
+  isDev,
+  kibanaVersion,
+  isServerless,
+  entityClient,
+}: {
+  core: CoreStart;
+  plugins: EntityManagerPluginStart;
+  appMountParameters: AppMountParameters;
+  ObservabilityPageTemplate: React.ComponentType<LazyObservabilityPageTemplateProps>;
+  usageCollection: UsageCollectionSetup;
+  isDev?: boolean;
+  kibanaVersion: string;
+  isServerless?: boolean;
+  entityClient: EntityClient;
+}) {
+  const { element, history, theme$ } = appMountParameters;
+  const isDarkMode = core.theme.getTheme().darkMode;
+
+  // ensure all divs are .kbnAppWrappers
+  element.classList.add(APP_WRAPPER_CLASS);
+
+  const queryClient = new QueryClient();
+
+  const ApplicationUsageTrackingProvider =
+    usageCollection?.components.ApplicationUsageTrackingProvider ?? React.Fragment;
+
+  const CloudProvider = plugins.cloud?.CloudContextProvider ?? React.Fragment;
+
+  const PresentationContextProvider = React.Fragment;
+
+  ReactDOM.render(
+    <KibanaRenderContextProvider {...core}>
+      <PresentationContextProvider>
+        <ApplicationUsageTrackingProvider>
+          <KibanaThemeProvider {...{ theme: { theme$ } }}>
+            <CloudProvider>
+              <KibanaContextProvider
+                services={{
+                  ...core,
+                  ...plugins,
+                  storage: new Storage(localStorage),
+                  entityClient: new EntityClient(core),
+                  isDev,
+                  kibanaVersion,
+                  isServerless,
+                }}
+              >
+                <PluginContext.Provider
+                  value={{
+                    isDev,
+                    isServerless,
+                    appMountParameters,
+                    ObservabilityPageTemplate,
+                    entityClient,
+                  }}
+                >
+                  <Router history={history}>
+                    <EuiThemeProvider darkMode={isDarkMode}>
+                      <RedirectAppLinks
+                        coreStart={core}
+                        data-test-subj="observabilityMainContainer"
+                      >
+                        <PerformanceContextProvider>
+                          <QueryClientProvider client={queryClient}>
+                            <EntityManagerOverviewPage />
+                          </QueryClientProvider>
+                        </PerformanceContextProvider>
+                      </RedirectAppLinks>
+                    </EuiThemeProvider>
+                  </Router>
+                </PluginContext.Provider>
+              </KibanaContextProvider>
+            </CloudProvider>
+          </KibanaThemeProvider>
+        </ApplicationUsageTrackingProvider>
+      </PresentationContextProvider>
+    </KibanaRenderContextProvider>,
+    element
+  );
+
+  return () => {
+    // This needs to be present to fix https://github.com/elastic/kibana/issues/155704
+    // as the Overview page renders the UX Section component. That component renders a Lens embeddable
+    // via the ExploratoryView app, which uses search sessions. Therefore on unmounting we need to clear
+    // these sessions.
+    plugins.data.search.session.clear();
+    ReactDOM.unmountComponentAtNode(element);
+  };
+}
diff --git a/x-pack/plugins/entity_manager/public/context/plugin_context.ts b/x-pack/plugins/entity_manager/public/context/plugin_context.ts
new file mode 100644
index 0000000000000..83c5bb2735b7c
--- /dev/null
+++ b/x-pack/plugins/entity_manager/public/context/plugin_context.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createContext } from 'react';
+import type { AppMountParameters } from '@kbn/core/public';
+import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public';
+import { EntityClient } from '../lib/entity_client';
+
+export interface PluginContextValue {
+  isDev?: boolean;
+  isServerless?: boolean;
+  appMountParameters?: AppMountParameters;
+  ObservabilityPageTemplate: React.ComponentType<LazyObservabilityPageTemplateProps>;
+  entityClient: EntityClient;
+}
+
+export const PluginContext = createContext<PluginContextValue | null>(null);
diff --git a/x-pack/plugins/entity_manager/public/hooks/use_kibana.ts b/x-pack/plugins/entity_manager/public/hooks/use_kibana.ts
new file mode 100644
index 0000000000000..6d6b9e49b4dac
--- /dev/null
+++ b/x-pack/plugins/entity_manager/public/hooks/use_kibana.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart } from '@kbn/core/public';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
+import { Storage } from '@kbn/kibana-utils-plugin/public';
+import { EntityClient } from '../lib/entity_client';
+
+export type StartServices<AdditionalServices extends object = {}> = CoreStart &
+  AdditionalServices & {
+    storage: Storage;
+    kibanaVersion: string;
+    entityClient: EntityClient;
+  };
+const useTypedKibana = <AdditionalServices extends object = {}>() =>
+  useKibana<StartServices<AdditionalServices>>();
+
+export { useTypedKibana as useKibana };
diff --git a/x-pack/plugins/entity_manager/public/hooks/use_plugin_context.ts b/x-pack/plugins/entity_manager/public/hooks/use_plugin_context.ts
new file mode 100644
index 0000000000000..d0640deb575b2
--- /dev/null
+++ b/x-pack/plugins/entity_manager/public/hooks/use_plugin_context.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useContext } from 'react';
+import { PluginContext } from '../context/plugin_context';
+import type { PluginContextValue } from '../context/plugin_context';
+
+export function usePluginContext(): PluginContextValue {
+  const context = useContext(PluginContext);
+  if (!context) {
+    throw new Error('Plugin context value is missing!');
+  }
+
+  return context;
+}
diff --git a/x-pack/plugins/entity_manager/public/pages/overview/index.tsx b/x-pack/plugins/entity_manager/public/pages/overview/index.tsx
new file mode 100644
index 0000000000000..8f68312064a7a
--- /dev/null
+++ b/x-pack/plugins/entity_manager/public/pages/overview/index.tsx
@@ -0,0 +1,320 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useState } from 'react';
+import { v4 as uuid } from 'uuid';
+import {
+  EuiBasicTable,
+  EuiButton,
+  EuiButtonIcon,
+  EuiCallOut,
+  EuiFieldText,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiForm,
+  EuiFormRow,
+  EuiHorizontalRule,
+  EuiSpacer,
+  EuiTitle,
+} from '@elastic/eui';
+import { usePluginContext } from '../../hooks/use_plugin_context';
+
+function EntitySourceForm({
+  source,
+  index,
+  onFieldChange,
+}: {
+  source: any;
+  index: number;
+  onFieldChange: Function;
+}) {
+  const onArrayFieldChange =
+    (field: Exclude<keyof EntitySource, 'id'>) => (e: React.ChangeEvent<HTMLInputElement>) => {
+      const value = e.target.value.trim();
+      if (!value) {
+        onFieldChange(index, field, []);
+      } else {
+        onFieldChange(index, field, e.target.value.trim().split(','));
+      }
+    };
+
+  return (
+    <EuiFlexGroup>
+      <EuiFlexItem>
+        <EuiFormRow label="Index patterns (comma-separated)">
+          <EuiFieldText
+            name="index_patterns"
+            defaultValue={source.index_patterns.join(',')}
+            isInvalid={source.index_patterns.length === 0}
+            onChange={onArrayFieldChange('index_patterns')}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
+
+      <EuiFlexItem>
+        <EuiFormRow label="Identify fields (comma-separated field names)">
+          <EuiFieldText
+            name="identity_fields"
+            defaultValue={source.identity_fields.join(',')}
+            isInvalid={source.identity_fields.length === 0}
+            onChange={onArrayFieldChange('identity_fields')}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
+
+      <EuiFlexItem>
+        <EuiFormRow label="Filters (comma-separated ESQL filters)">
+          <EuiFieldText
+            name="filters"
+            defaultValue={source.filters.join(',')}
+            onChange={onArrayFieldChange('filters')}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
+
+      <EuiFlexItem>
+        <EuiFormRow label="Metadata (comma-separated field names)">
+          <EuiFieldText
+            name="metadata"
+            defaultValue={source.metadata_fields.join(',')}
+            onChange={onArrayFieldChange('metadata_fields')}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  );
+}
+
+interface EntitySource {
+  id: string;
+  index_patterns?: string[];
+  identity_fields?: string[];
+  metadata_fields?: string[];
+  filters?: string[];
+}
+
+const newEntitySource = ({
+  indexPatterns = [],
+  identityFields = [],
+  metadataFields = [],
+  filters = [],
+}: {
+  indexPatterns?: string[];
+  identityFields?: string[];
+  metadataFields?: string[];
+  filters?: string[];
+}) => ({
+  id: uuid(),
+  index_patterns: indexPatterns,
+  identity_fields: identityFields,
+  metadata_fields: metadataFields,
+  filters,
+});
+
+export function EntityManagerOverviewPage() {
+  const { ObservabilityPageTemplate, entityClient } = usePluginContext();
+  const [previewEntities, setPreviewEntities] = useState([]);
+  const [isSearchingEntities, setIsSearchingEntities] = useState(false);
+  const [previewError, setPreviewError] = useState(null);
+  const [formErrors, setFormErrors] = useState<string[]>([]);
+  const [entityType, setEntityType] = useState('service');
+  const [entitySources, setEntitySources] = useState([
+    newEntitySource({
+      indexPatterns: ['remote_cluster:logs-*'],
+      identityFields: ['service.name'],
+    }),
+  ]);
+
+  const searchEntities = async () => {
+    if (
+      !entitySources.some(
+        (source) => source.identity_fields.length > 0 && source.index_patterns.length > 0
+      )
+    ) {
+      setFormErrors(['No valid source found']);
+      return;
+    }
+
+    setIsSearchingEntities(true);
+    setFormErrors([]);
+    setPreviewError(null);
+
+    try {
+      const { entities } = await entityClient.repositoryClient(
+        'POST /internal/entities/_search/preview',
+        {
+          params: {
+            body: {
+              sources: entitySources
+                .filter(
+                  (source) => source.index_patterns.length > 0 && source.identity_fields.length > 0
+                )
+                .map((source) => ({ ...source, type: entityType })),
+            },
+          },
+        }
+      );
+
+      setPreviewEntities(entities);
+    } catch (err) {
+      setPreviewError(err.body?.message);
+    } finally {
+      setIsSearchingEntities(false);
+    }
+  };
+
+  return (
+    <ObservabilityPageTemplate
+      data-test-subj="entitiesPage"
+      pageHeader={{
+        bottomBorder: true,
+        pageTitle: 'Entity Manager',
+      }}
+    >
+      <EuiForm component="form" isInvalid={formErrors.length > 0} error={formErrors}>
+        <EuiFlexGroup direction="column" gutterSize="s">
+          <EuiFlexItem>
+            <EuiTitle size="s">
+              <h2>Entity type</h2>
+            </EuiTitle>
+          </EuiFlexItem>
+
+          <EuiFlexItem>
+            <EuiFormRow>
+              <EuiFieldText
+                name="type"
+                defaultValue={entityType}
+                placeholder="host, service, user..."
+                onChange={(e) => {
+                  setEntityType(e.target.value.trim());
+                }}
+              />
+            </EuiFormRow>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+
+        <EuiSpacer size="m" />
+
+        <EuiFlexGroup alignItems="center">
+          <EuiFlexItem grow={false}>
+            <EuiTitle size="s">
+              <h2>Entity sources</h2>
+            </EuiTitle>
+          </EuiFlexItem>
+
+          <EuiFlexItem grow={false}>
+            <EuiButtonIcon
+              iconType="plusInCircle"
+              onClick={() => setEntitySources([...entitySources, newEntitySource({})])}
+            />
+          </EuiFlexItem>
+        </EuiFlexGroup>
+
+        <EuiSpacer size="s" />
+
+        {entitySources.map((source, i) => (
+          <div key={source.id}>
+            <EuiFlexGroup alignItems="center">
+              <EuiFlexItem grow={false}>
+                <EuiTitle size="xxs">
+                  <h4>Source {i + 1}</h4>
+                </EuiTitle>
+              </EuiFlexItem>
+              {entitySources.length > 1 ? (
+                <EuiFlexItem grow={false}>
+                  <EuiButtonIcon
+                    color={'danger'}
+                    iconType={'minusInCircle'}
+                    onClick={() => {
+                      entitySources.splice(i, 1);
+                      setEntitySources(entitySources.map((_source) => ({ ..._source })));
+                    }}
+                  />
+                </EuiFlexItem>
+              ) : null}
+            </EuiFlexGroup>
+
+            <EuiSpacer size="s" />
+
+            <EntitySourceForm
+              source={source}
+              index={i}
+              onFieldChange={(
+                index: number,
+                field: Exclude<keyof EntitySource, 'id'>,
+                value: string[]
+              ) => {
+                entitySources[index][field] = value;
+                setEntitySources([...entitySources]);
+              }}
+            />
+            {i === entitySources.length - 1 ? (
+              <EuiSpacer size="m" />
+            ) : (
+              <EuiHorizontalRule margin="m" />
+            )}
+          </div>
+        ))}
+
+        <EuiFormRow>
+          <EuiFlexGroup>
+            <EuiFlexItem>
+              <EuiButton isDisabled={isSearchingEntities} onClick={searchEntities}>
+                Preview
+              </EuiButton>
+            </EuiFlexItem>
+            <EuiFlexItem>
+              <EuiButton isDisabled={true} color="primary">
+                Create
+              </EuiButton>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFormRow>
+      </EuiForm>
+
+      <EuiSpacer size="s" />
+
+      {previewError ? (
+        <EuiCallOut title="Error previewing entity definition" color="danger" iconType="error">
+          <p>{previewError}</p>
+        </EuiCallOut>
+      ) : null}
+
+      <EuiBasicTable
+        loading={isSearchingEntities}
+        tableCaption={'Preview entities'}
+        items={previewEntities}
+        columns={[
+          {
+            field: 'entity.id',
+            name: 'entity.id',
+          },
+          {
+            field: 'entity.type',
+            name: 'entity.type',
+          },
+          {
+            field: 'entity.last_seen_timestamp',
+            name: 'entity.last_seen_timestamp',
+          },
+          ...Array.from(new Set(entitySources.flatMap((source) => source.identity_fields))).map(
+            (field) => ({
+              field,
+              name: field,
+            })
+          ),
+          ...Array.from(new Set(entitySources.flatMap((source) => source.metadata_fields))).map(
+            (field) => ({
+              field: `metadata.${field}`,
+              name: `metadata.${field}`,
+            })
+          ),
+        ]}
+      />
+    </ObservabilityPageTemplate>
+  );
+}
diff --git a/x-pack/plugins/entity_manager/public/plugin.ts b/x-pack/plugins/entity_manager/public/plugin.ts
index 6d6d56a95b757..cec49a45f4109 100644
--- a/x-pack/plugins/entity_manager/public/plugin.ts
+++ b/x-pack/plugins/entity_manager/public/plugin.ts
@@ -5,26 +5,80 @@
  * 2.0.
  */
 
-import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public';
+import { BehaviorSubject } from 'rxjs';
+import {
+  App,
+  AppMountParameters,
+  AppStatus,
+  AppUpdater,
+  CoreSetup,
+  CoreStart,
+  DEFAULT_APP_CATEGORIES,
+  PluginInitializerContext,
+} from '@kbn/core/public';
 import { Logger } from '@kbn/logging';
 
-import { EntityManagerPluginClass } from './types';
+import {
+  EntityManagerPluginClass,
+  EntityManagerPluginSetup,
+  EntityManagerPluginStart,
+  EntityManagerPublicPluginStart,
+} from './types';
 import type { EntityManagerPublicConfig } from '../common/config';
 import { EntityClient } from './lib/entity_client';
 
 export class Plugin implements EntityManagerPluginClass {
   public config: EntityManagerPublicConfig;
   public logger: Logger;
+  private readonly appUpdater$ = new BehaviorSubject<AppUpdater>(() => ({}));
 
-  constructor(context: PluginInitializerContext<{}>) {
+  constructor(private readonly context: PluginInitializerContext<{}>) {
     this.config = context.config.get();
     this.logger = context.logger.get();
   }
 
-  setup(core: CoreSetup) {
-    const entityClient = new EntityClient(core);
+  setup(
+    core: CoreSetup<EntityManagerPluginStart, EntityManagerPublicPluginStart>,
+    pluginSetup: EntityManagerPluginSetup
+  ) {
+    const kibanaVersion = this.context.env.packageInfo.version;
+
+    const mount = async (params: AppMountParameters<unknown>) => {
+      const { renderApp } = await import('./application');
+      const [coreStart, pluginsStart] = await core.getStartServices();
+
+      return renderApp({
+        appMountParameters: params,
+        core: coreStart,
+        isDev: this.context.env.mode.dev,
+        kibanaVersion,
+        usageCollection: pluginSetup.usageCollection,
+        ObservabilityPageTemplate: pluginsStart.observabilityShared.navigation.PageTemplate,
+        plugins: pluginsStart,
+        isServerless: !!pluginsStart.serverless,
+        entityClient: new EntityClient(core),
+      });
+    };
+
+    const appUpdater$ = this.appUpdater$;
+    const app: App = {
+      id: 'entity_manager',
+      title: 'Entity Manager',
+      order: 8002,
+      updater$: appUpdater$,
+      euiIconType: 'logoObservability',
+      appRoute: '/app/entity_manager',
+      category: DEFAULT_APP_CATEGORIES.observability,
+      mount,
+      visibleIn: [],
+      keywords: ['observability', 'monitor', 'entities'],
+      status: AppStatus.inaccessible,
+    };
+
+    core.application.register(app);
+
     return {
-      entityClient,
+      entityClient: new EntityClient(core),
     };
   }
 
diff --git a/x-pack/plugins/entity_manager/public/routes.tsx b/x-pack/plugins/entity_manager/public/routes.tsx
new file mode 100644
index 0000000000000..80baa45422bfd
--- /dev/null
+++ b/x-pack/plugins/entity_manager/public/routes.tsx
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EntityManagerOverviewPage } from './pages/overview';
+
+interface RouteDef {
+  [key: string]: {
+    handler: () => React.ReactElement;
+    params: Record<string, string>;
+    exact: boolean;
+  };
+}
+
+export function getRoutes(): RouteDef {
+  return {
+    '/app/entity_manager': {
+      handler: () => <EntityManagerOverviewPage />,
+      params: {},
+      exact: true,
+    },
+  };
+}
diff --git a/x-pack/plugins/entity_manager/public/types.ts b/x-pack/plugins/entity_manager/public/types.ts
index 66499479299dc..a797f9b7b0ebb 100644
--- a/x-pack/plugins/entity_manager/public/types.ts
+++ b/x-pack/plugins/entity_manager/public/types.ts
@@ -5,8 +5,34 @@
  * 2.0.
  */
 import type { Plugin as PluginClass } from '@kbn/core/public';
+import { PresentationUtilPluginStart } from '@kbn/presentation-util-plugin/public';
+import { CloudStart } from '@kbn/cloud-plugin/public';
+import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public';
+import { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public';
+import {
+  ObservabilitySharedPluginSetup,
+  ObservabilitySharedPluginStart,
+} from '@kbn/observability-shared-plugin/public';
+import { ObservabilityAIAssistantPublicSetup } from '@kbn/observability-ai-assistant-plugin/public';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import type { EntityClient } from './lib/entity_client';
 
+export interface EntityManagerPluginSetup {
+  data: DataPublicPluginSetup;
+  observability: ObservabilityAIAssistantPublicSetup;
+  observabilityShared: ObservabilitySharedPluginSetup;
+  serverless?: ServerlessPluginSetup;
+  usageCollection: UsageCollectionSetup;
+}
+
+export interface EntityManagerPluginStart {
+  data: DataPublicPluginStart;
+  presentationUtil: PresentationUtilPluginStart;
+  cloud?: CloudStart;
+  serverless?: ServerlessPluginStart;
+  observabilityShared: ObservabilitySharedPluginStart;
+}
+
 export interface EntityManagerPublicPluginSetup {
   entityClient: EntityClient;
 }
diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 4e1dd263f9ca3..95df0863e1713 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -5,9 +5,11 @@
  * 2.0.
  */
 
-import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
+import { Entity, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
+import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects';
+import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client';
 import { Logger } from '@kbn/logging';
 import {
   installEntityDefinition,
@@ -23,6 +25,8 @@ import { stopTransforms } from './entities/stop_transforms';
 import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
+import { EntitySource, getEntityInstancesQuery } from './queries';
+import { mergeEntitiesList } from './queries/utils';
 
 export class EntityClient {
   constructor(
@@ -187,4 +191,45 @@ export class EntityClient {
       this.options.logger
     );
   }
+
+  async searchEntities({ sources, limit = 10 }: { sources: EntitySource[]; limit?: number }) {
+    const entities = await Promise.all(
+      sources.map(async (source) => {
+        const esClient = createObservabilityEsClient({
+          client: this.options.clusterClient.asCurrentUser,
+          logger: this.options.logger,
+          plugin: `@kbn/entityManager-plugin`,
+        });
+
+        const requiredFields = [...source.identity_fields, ...source.metadata_fields];
+        const { fields } = await esClient.client.fieldCaps({
+          index: source.index_patterns,
+          fields: requiredFields,
+        });
+
+        const sourceHasIdentityFields = source.identity_fields.every((field) => !!fields[field]);
+        if (!sourceHasIdentityFields) {
+          return [];
+        }
+
+        const availableMetadataFields = source.metadata_fields.filter((field) => fields[field]);
+
+        const query = getEntityInstancesQuery(
+          { ...source, metadata_fields: availableMetadataFields },
+          limit
+        );
+        this.options.logger.info(`Entity query: ${query}`);
+
+        return await esClient.esql('search_entities', { query }).then((result) =>
+          esqlResultToPlainObjects(result).map((entity) => {
+            entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
+            entity['entity.type'] = source.type;
+            return entity as Entity;
+          })
+        );
+      })
+    ).then((results) => results.flat());
+
+    return mergeEntitiesList(entities);
+  }
 }
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
new file mode 100644
index 0000000000000..b670ff5c16948
--- /dev/null
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface EntitySource {
+  type: string;
+  index_patterns: string[];
+  identity_fields: string[];
+  metadata_fields: string[];
+  filters: string[];
+}
+
+export function getEntityInstancesQuery(source: EntitySource, limit: number): string {
+  let query = `FROM ${source.index_patterns} |`;
+
+  source.identity_fields.forEach((field) => {
+    query += `WHERE ${field} IS NOT NULL |`;
+  });
+
+  source.filters.forEach((filter) => {
+    query += `WHERE ${filter} |`;
+  });
+
+  const aggs = [
+    // default 'last_seen' attribute
+    'entity.last_seen_timestamp=MAX(@timestamp)',
+    ...source.metadata_fields
+      .filter((field) => !source.identity_fields.some((idField) => idField === field))
+      .map((field) => `metadata.${field}=VALUES(${field})`),
+  ];
+
+  query += `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(',')} |`;
+  query += `LIMIT ${limit}`;
+
+  return query;
+}
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
new file mode 100644
index 0000000000000..0fbf186524da7
--- /dev/null
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Entity } from '@kbn/entities-schema';
+
+function mergeEntities(entity1: Entity, entity2: Entity): Entity {
+  const merged: Entity = {
+    ...entity1,
+    'entity.last_seen_timestamp': new Date(
+      Math.max(
+        Date.parse(entity1['entity.last_seen_timestamp']),
+        Date.parse(entity2['entity.last_seen_timestamp'])
+      )
+    ).toISOString(),
+  };
+
+  for (const [key, value] of Object.entries(entity2).filter(([_key]) =>
+    _key.startsWith('metadata.')
+  )) {
+    if (merged[key]) {
+      merged[key] = Array.isArray(merged[key]) ? [...merged[key], value] : [merged[key], value];
+    } else {
+      merged[key] = value;
+    }
+  }
+  return merged;
+}
+
+export function mergeEntitiesList(entities: Entity[]): Entity[] {
+  const instances: { [key: string]: Entity } = {};
+
+  for (let i = 0; i < entities.length; i++) {
+    const entity = entities[i];
+    const id = entity['entity.id'];
+
+    if (instances[id]) {
+      instances[id] = mergeEntities(instances[id], entity);
+    } else {
+      instances[id] = entity;
+    }
+  }
+
+  return Array.from(Object.values(instances));
+}
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/index.ts b/x-pack/plugins/entity_manager/server/routes/entities/index.ts
index 539423c6a5e17..5ed21b5112f6f 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/index.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/index.ts
@@ -10,6 +10,7 @@ import { deleteEntityDefinitionRoute } from './delete';
 import { getEntityDefinitionRoute } from './get';
 import { resetEntityDefinitionRoute } from './reset';
 import { updateEntityDefinitionRoute } from './update';
+import { searchEntitiesRoute, searchEntitiesPreviewRoute } from './search';
 
 export const entitiesRoutes = {
   ...createEntityDefinitionRoute,
@@ -17,4 +18,6 @@ export const entitiesRoutes = {
   ...getEntityDefinitionRoute,
   ...resetEntityDefinitionRoute,
   ...updateEntityDefinitionRoute,
+  ...searchEntitiesRoute,
+  ...searchEntitiesPreviewRoute,
 };
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
new file mode 100644
index 0000000000000..8891ba96052c7
--- /dev/null
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import { z } from '@kbn/zod';
+import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
+import { EntitySource } from '../../lib/queries';
+
+export const searchEntitiesRoute = createEntityManagerServerRoute({
+  endpoint: 'POST /internal/entities/_search',
+  params: z.object({
+    body: z.object({
+      type: z.string(),
+      limit: z.optional(z.number()).default(5),
+    }),
+  }),
+  handler: async ({ request, response, params, logger, getScopedClient }) => {
+    try {
+      const { type, limit = 5 } = params.body;
+
+      const client = await getScopedClient({ request });
+      const sources = [
+        {
+          type: 'service',
+          index_patterns: ['remote_cluster:logs-*'],
+          identity_fields: ['service.name', 'service.environment'],
+          metadata_fields: [],
+          filters: [],
+        },
+      ] as EntitySource[]; // get sources for the type
+
+      const entities = await client.searchEntities({ sources, limit });
+
+      return response.ok({ body: { entities } });
+    } catch (e) {
+      logger.error(e);
+      return response.customError({ body: e, statusCode: 500 });
+    }
+  },
+});
+
+export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
+  endpoint: 'POST /internal/entities/_search/preview',
+  params: z.object({
+    body: z.object({
+      filters: z.optional(z.array(z.string())).default([]),
+      metadata_fields: z.optional(z.array(z.string())).default([]),
+      limit: z.optional(z.number()).default(5),
+      sources: z.array(
+        z.object({
+          type: z.string(),
+          index_patterns: z.array(z.string()),
+          identity_fields: z.array(z.string()),
+          metadata_fields: z.array(z.string()),
+          filters: z
+            .array(z.string())
+            .transform((filters) => [
+              ...filters,
+              `@timestamp >= "${moment().subtract(1, 'hour').toISOString()}"`,
+            ]),
+        })
+      ),
+    }),
+  }),
+  handler: async ({ request, response, params, logger, getScopedClient }) => {
+    try {
+      const { sources, limit } = params.body;
+
+      const client = await getScopedClient({ request });
+      const entities = await client.searchEntities({
+        sources,
+        limit,
+      });
+
+      return response.ok({ body: { entities } });
+    } catch (e) {
+      logger.error(e);
+      return response.customError({ body: e, statusCode: 500 });
+    }
+  },
+});
diff --git a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts
new file mode 100644
index 0000000000000..991e2facf1328
--- /dev/null
+++ b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObject, SavedObjectsType } from '@kbn/core/server';
+import { EntityDefinitionTemplate } from '@kbn/entities-schema';
+
+export const SO_ENTITY_DEFINITION_TEMPLATE_TYPE = 'entity-definition-template';
+
+export const entityDefinitionTemplate: SavedObjectsType = {
+  name: SO_ENTITY_DEFINITION_TEMPLATE_TYPE,
+  hidden: false,
+  namespaceType: 'multiple-isolated',
+  mappings: {
+    dynamic: false,
+    properties: {
+      id: { type: 'keyword' },
+      version: { type: 'keyword' },
+      type: { type: 'keyword' },
+    },
+  },
+  management: {
+    displayName: 'Entity Definition Template',
+    getInAppUrl: (savedObject) => {
+      return {
+        path: `/app/entities#/create?template=${savedObject.id}`,
+        uiCapabilitiesPath: '',
+      };
+    },
+    importableAndExportable: true,
+    getTitle(savedObject: SavedObject<EntityDefinitionTemplate>) {
+      return `EntityDefinitionTemplate: [${savedObject.attributes.name}]`;
+    },
+  },
+};

From 6f10eb4c1c3e9261aa3a59d174801ffab1c87afe Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Mon, 11 Nov 2024 11:45:08 +0100
Subject: [PATCH 02/37] unneeded array.from

---
 x-pack/plugins/entity_manager/server/lib/queries/utils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
index 0fbf186524da7..7036d33389f2c 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
@@ -44,5 +44,5 @@ export function mergeEntitiesList(entities: Entity[]): Entity[] {
     }
   }
 
-  return Array.from(Object.values(instances));
+  return Object.values(instances);
 }

From 2092eda3ad46c27c6e57b8b681f4a2dddc687de5 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Mon, 11 Nov 2024 11:47:19 +0100
Subject: [PATCH 03/37] should not be added

---
 .../entity_definition_template.ts             | 38 -------------------
 1 file changed, 38 deletions(-)
 delete mode 100644 x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts

diff --git a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts b/x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts
deleted file mode 100644
index 991e2facf1328..0000000000000
--- a/x-pack/plugins/entity_manager/server/saved_objects/entity_definition_template.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { SavedObject, SavedObjectsType } from '@kbn/core/server';
-import { EntityDefinitionTemplate } from '@kbn/entities-schema';
-
-export const SO_ENTITY_DEFINITION_TEMPLATE_TYPE = 'entity-definition-template';
-
-export const entityDefinitionTemplate: SavedObjectsType = {
-  name: SO_ENTITY_DEFINITION_TEMPLATE_TYPE,
-  hidden: false,
-  namespaceType: 'multiple-isolated',
-  mappings: {
-    dynamic: false,
-    properties: {
-      id: { type: 'keyword' },
-      version: { type: 'keyword' },
-      type: { type: 'keyword' },
-    },
-  },
-  management: {
-    displayName: 'Entity Definition Template',
-    getInAppUrl: (savedObject) => {
-      return {
-        path: `/app/entities#/create?template=${savedObject.id}`,
-        uiCapabilitiesPath: '',
-      };
-    },
-    importableAndExportable: true,
-    getTitle(savedObject: SavedObject<EntityDefinitionTemplate>) {
-      return `EntityDefinitionTemplate: [${savedObject.attributes.name}]`;
-    },
-  },
-};

From 87a3c5514f1b1b9b689f3a6d520e7d55a904fd2a Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 12 Nov 2024 13:15:49 +0100
Subject: [PATCH 04/37] tweaks

---
 x-pack/plugins/entity_manager/server/lib/entity_client.ts    | 3 +++
 x-pack/plugins/entity_manager/server/lib/queries/index.ts    | 1 +
 x-pack/plugins/entity_manager/server/lib/queries/utils.ts    | 5 ++++-
 .../plugins/entity_manager/server/routes/entities/search.ts  | 4 ++--
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 95df0863e1713..89bfba02f4663 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -209,9 +209,12 @@ export class EntityClient {
 
         const sourceHasIdentityFields = source.identity_fields.every((field) => !!fields[field]);
         if (!sourceHasIdentityFields) {
+          // we can't build entities without id fields so we ignore the source.
+          // filters should likely behave similarly.
           return [];
         }
 
+        // but metadata field not being available is fine
         const availableMetadataFields = source.metadata_fields.filter((field) => fields[field]);
 
         const query = getEntityInstancesQuery(
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index b670ff5c16948..9b9c0e3c422ed 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -33,6 +33,7 @@ export function getEntityInstancesQuery(source: EntitySource, limit: number): st
   ];
 
   query += `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(',')} |`;
+  query += `SORT entity.last_seen_timestamp DESC |`;
   query += `LIMIT ${limit}`;
 
   return query;
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
index 7036d33389f2c..9661fb3c19bfd 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
@@ -6,6 +6,7 @@
  */
 
 import { Entity } from '@kbn/entities-schema';
+import { uniq } from 'lodash';
 
 function mergeEntities(entity1: Entity, entity2: Entity): Entity {
   const merged: Entity = {
@@ -22,7 +23,9 @@ function mergeEntities(entity1: Entity, entity2: Entity): Entity {
     _key.startsWith('metadata.')
   )) {
     if (merged[key]) {
-      merged[key] = Array.isArray(merged[key]) ? [...merged[key], value] : [merged[key], value];
+      merged[key] = uniq(
+        Array.isArray(merged[key]) ? [...merged[key], value] : [merged[key], value]
+      );
     } else {
       merged[key] = value;
     }
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
index 8891ba96052c7..c75e792f45084 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -49,7 +49,7 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
     body: z.object({
       filters: z.optional(z.array(z.string())).default([]),
       metadata_fields: z.optional(z.array(z.string())).default([]),
-      limit: z.optional(z.number()).default(5),
+      limit: z.optional(z.number()).default(10),
       sources: z.array(
         z.object({
           type: z.string(),
@@ -60,7 +60,7 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
             .array(z.string())
             .transform((filters) => [
               ...filters,
-              `@timestamp >= "${moment().subtract(1, 'hour').toISOString()}"`,
+              `@timestamp >= "${moment().subtract(5, 'minutes').toISOString()}"`,
             ]),
         })
       ),

From 63ce56991e7996eba32799b33bb6155a3c28cc63 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 12 Nov 2024 13:40:30 +0100
Subject: [PATCH 05/37] include es metadata

---
 .../server/lib/queries/index.ts               | 41 +++++++++++++++----
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index 9b9c0e3c422ed..d388f60d08e06 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -13,17 +13,34 @@ export interface EntitySource {
   filters: string[];
 }
 
-export function getEntityInstancesQuery(source: EntitySource, limit: number): string {
-  let query = `FROM ${source.index_patterns} |`;
+const sourceCommand = (source: EntitySource) => {
+  let query = `FROM ${source.index_patterns}`;
+
+  const esMetadataFields = source.metadata_fields.filter((field) =>
+    ['_index', '_id'].includes(field)
+  );
+  if (esMetadataFields.length) {
+    query += ` METADATA ${esMetadataFields.join(',')}`;
+  }
+
+  return query;
+};
+
+const filterCommands = (source: EntitySource) => {
+  const commands: string[] = [];
 
   source.identity_fields.forEach((field) => {
-    query += `WHERE ${field} IS NOT NULL |`;
+    commands.push(`WHERE ${field} IS NOT NULL`);
   });
 
   source.filters.forEach((filter) => {
-    query += `WHERE ${filter} |`;
+    commands.push(`WHERE ${filter}`);
   });
 
+  return commands;
+};
+
+const statsCommand = (source: EntitySource) => {
   const aggs = [
     // default 'last_seen' attribute
     'entity.last_seen_timestamp=MAX(@timestamp)',
@@ -32,9 +49,17 @@ export function getEntityInstancesQuery(source: EntitySource, limit: number): st
       .map((field) => `metadata.${field}=VALUES(${field})`),
   ];
 
-  query += `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(',')} |`;
-  query += `SORT entity.last_seen_timestamp DESC |`;
-  query += `LIMIT ${limit}`;
+  return `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(',')}`;
+};
 
-  return query;
+export function getEntityInstancesQuery(source: EntitySource, limit: number): string {
+  const commands = [
+    sourceCommand(source),
+    ...filterCommands(source),
+    statsCommand(source),
+    `SORT entity.last_seen_timestamp DESC`,
+    `LIMIT ${limit}`,
+  ];
+
+  return commands.join('|');
 }

From 79338d846487f35934224ce58f3423bee0b89af8 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 12 Nov 2024 17:53:47 +0100
Subject: [PATCH 06/37] slice result set

---
 x-pack/plugins/entity_manager/server/lib/entity_client.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 89bfba02f4663..10aa5219a853b 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -233,6 +233,6 @@ export class EntityClient {
       })
     ).then((results) => results.flat());
 
-    return mergeEntitiesList(entities);
+    return mergeEntitiesList(entities).slice(0, limit);
   }
 }

From 4f9aca55ba1b80daddcd9ab910415407e11b4e72 Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Mon, 18 Nov 2024 15:10:01 +0000
Subject: [PATCH 07/37] [CI] Auto-commit changed files from 'node
 scripts/yarn_deduplicate'

---
 x-pack/plugins/entity_manager/tsconfig.json | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json
index 34c57a27dd829..a32bf5f6cdb77 100644
--- a/x-pack/plugins/entity_manager/tsconfig.json
+++ b/x-pack/plugins/entity_manager/tsconfig.json
@@ -35,5 +35,19 @@
     "@kbn/encrypted-saved-objects-plugin",
     "@kbn/licensing-plugin",
     "@kbn/core-saved-objects-server",
+    "@kbn/ebt-tools",
+    "@kbn/kibana-react-plugin",
+    "@kbn/kibana-utils-plugin",
+    "@kbn/observability-shared-plugin",
+    "@kbn/react-kibana-context-render",
+    "@kbn/react-kibana-context-theme",
+    "@kbn/shared-ux-link-redirect-app",
+    "@kbn/usage-collection-plugin",
+    "@kbn/shared-ux-router",
+    "@kbn/presentation-util-plugin",
+    "@kbn/cloud-plugin",
+    "@kbn/serverless",
+    "@kbn/data-plugin",
+    "@kbn/observability-ai-assistant-plugin",
   ]
 }

From 8926c245a64bfaca05142f78224dd96a11293be0 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Mon, 18 Nov 2024 17:19:49 +0100
Subject: [PATCH 08/37] fix imports and add @timestamp as required field

---
 .../server/lib/entity_client.ts               | 27 +++++++++--------
 .../server/lib/queries/index.ts               | 18 +++++++-----
 .../server/routes/entities/search.ts          | 29 +++++++------------
 3 files changed, 35 insertions(+), 39 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 45977074bb8d7..388ee2d16550d 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -7,8 +7,7 @@
 
 import { Entity, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
-import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects';
-import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client';
+import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
 import {
@@ -177,19 +176,19 @@ export class EntityClient {
     const entities = await Promise.all(
       sources.map(async (source) => {
         const esClient = createObservabilityEsClient({
-          client: this.options.clusterClient.asCurrentUser,
+          client: this.options.esClient,
           logger: this.options.logger,
           plugin: `@kbn/entityManager-plugin`,
         });
 
-        const requiredFields = [...source.identity_fields, ...source.metadata_fields];
+        const mandatoryFields = ['@timestamp', ...source.identity_fields];
         const { fields } = await esClient.client.fieldCaps({
           index: source.index_patterns,
-          fields: requiredFields,
+          fields: [...mandatoryFields, ...source.metadata_fields],
         });
 
-        const sourceHasIdentityFields = source.identity_fields.every((field) => !!fields[field]);
-        if (!sourceHasIdentityFields) {
+        const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
+        if (!sourceHasMandatoryFields) {
           // we can't build entities without id fields so we ignore the source.
           // filters should likely behave similarly.
           return [];
@@ -204,13 +203,13 @@ export class EntityClient {
         );
         this.options.logger.info(`Entity query: ${query}`);
 
-        return await esClient.esql('search_entities', { query }).then((result) =>
-          esqlResultToPlainObjects(result).map((entity) => {
-            entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
-            entity['entity.type'] = source.type;
-            return entity as Entity;
-          })
-        );
+        const rawEntities = await esClient.esql<Entity>('search_entities', { query });
+
+        return rawEntities.map((entity) => {
+          entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
+          entity['entity.type'] = source.type;
+          return entity;
+        });
       })
     ).then((results) => results.flat());
 
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index d388f60d08e06..bec5d5f32818d 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -5,13 +5,17 @@
  * 2.0.
  */
 
-export interface EntitySource {
-  type: string;
-  index_patterns: string[];
-  identity_fields: string[];
-  metadata_fields: string[];
-  filters: string[];
-}
+import z from 'zod';
+
+export const entitySourceSchema = z.object({
+  type: z.string(),
+  index_patterns: z.array(z.string()),
+  identity_fields: z.array(z.string()),
+  metadata_fields: z.array(z.string()),
+  filters: z.array(z.string()),
+});
+
+export type EntitySource = z.infer<typeof entitySourceSchema>;
 
 const sourceCommand = (source: EntitySource) => {
   let query = `FROM ${source.index_patterns}`;
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
index c75e792f45084..93ab07b59897b 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -8,19 +8,19 @@
 import moment from 'moment';
 import { z } from '@kbn/zod';
 import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
-import { EntitySource } from '../../lib/queries';
+import { EntitySource, entitySourceSchema } from '../../lib/queries';
 
 export const searchEntitiesRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/_search',
   params: z.object({
     body: z.object({
       type: z.string(),
-      limit: z.optional(z.number()).default(5),
+      limit: z.optional(z.number()).default(10),
     }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
     try {
-      const { type, limit = 5 } = params.body;
+      const { type, limit } = params.body;
 
       const client = await getScopedClient({ request });
       const sources = [
@@ -47,23 +47,16 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/_search/preview',
   params: z.object({
     body: z.object({
-      filters: z.optional(z.array(z.string())).default([]),
-      metadata_fields: z.optional(z.array(z.string())).default([]),
-      limit: z.optional(z.number()).default(10),
       sources: z.array(
-        z.object({
-          type: z.string(),
-          index_patterns: z.array(z.string()),
-          identity_fields: z.array(z.string()),
-          metadata_fields: z.array(z.string()),
-          filters: z
-            .array(z.string())
-            .transform((filters) => [
-              ...filters,
-              `@timestamp >= "${moment().subtract(5, 'minutes').toISOString()}"`,
-            ]),
-        })
+        entitySourceSchema.transform((source) => ({
+          ...source,
+          filters: [
+            ...source.filters,
+            `@timestamp >= "${moment().subtract(5, 'minutes').toISOString()}"`,
+          ],
+        }))
       ),
+      limit: z.optional(z.number()).default(10),
     }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {

From fcfc507edbd283b17270cb0f24e898380ced1ab6 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Mon, 18 Nov 2024 17:22:56 +0100
Subject: [PATCH 09/37] configurable timestamp

---
 x-pack/plugins/entity_manager/server/lib/queries/index.ts      | 3 ++-
 x-pack/plugins/entity_manager/server/routes/entities/search.ts | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index bec5d5f32818d..d693e1c91106b 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -9,6 +9,7 @@ import z from 'zod';
 
 export const entitySourceSchema = z.object({
   type: z.string(),
+  timestamp_field: z.optional(z.string()).default('@timestamp'),
   index_patterns: z.array(z.string()),
   identity_fields: z.array(z.string()),
   metadata_fields: z.array(z.string()),
@@ -47,7 +48,7 @@ const filterCommands = (source: EntitySource) => {
 const statsCommand = (source: EntitySource) => {
   const aggs = [
     // default 'last_seen' attribute
-    'entity.last_seen_timestamp=MAX(@timestamp)',
+    `entity.last_seen_timestamp=MAX(${source.timestamp_field})`,
     ...source.metadata_fields
       .filter((field) => !source.identity_fields.some((idField) => idField === field))
       .map((field) => `metadata.${field}=VALUES(${field})`),
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
index 93ab07b59897b..25a9e28e35e18 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -26,6 +26,7 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
       const sources = [
         {
           type: 'service',
+          timestamp_field: '@timestamp',
           index_patterns: ['remote_cluster:logs-*'],
           identity_fields: ['service.name', 'service.environment'],
           metadata_fields: [],
@@ -52,7 +53,7 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
           ...source,
           filters: [
             ...source.filters,
-            `@timestamp >= "${moment().subtract(5, 'minutes').toISOString()}"`,
+            `${source.timestamp_field} >= "${moment().subtract(5, 'minutes').toISOString()}"`,
           ],
         }))
       ),

From 1678d83fb1c5954cc14fe8806736b70aa5a84348 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Mon, 18 Nov 2024 17:24:18 +0100
Subject: [PATCH 10/37] mandatory timestamp field

---
 x-pack/plugins/entity_manager/server/lib/entity_client.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 388ee2d16550d..104992a1a044f 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -181,7 +181,7 @@ export class EntityClient {
           plugin: `@kbn/entityManager-plugin`,
         });
 
-        const mandatoryFields = ['@timestamp', ...source.identity_fields];
+        const mandatoryFields = [source.timestamp_field, ...source.identity_fields];
         const { fields } = await esClient.client.fieldCaps({
           index: source.index_patterns,
           fields: [...mandatoryFields, ...source.metadata_fields],

From 6d80b81c6b442059de49eecba5f6f50e0c532ab2 Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Mon, 18 Nov 2024 16:36:39 +0000
Subject: [PATCH 11/37] [CI] Auto-commit changed files from 'node
 scripts/yarn_deduplicate'

---
 x-pack/plugins/entity_manager/tsconfig.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json
index a32bf5f6cdb77..bbb4fcd5b6495 100644
--- a/x-pack/plugins/entity_manager/tsconfig.json
+++ b/x-pack/plugins/entity_manager/tsconfig.json
@@ -49,5 +49,6 @@
     "@kbn/serverless",
     "@kbn/data-plugin",
     "@kbn/observability-ai-assistant-plugin",
+    "@kbn/observability-utils-server",
   ]
 }

From 54498255caf74230d157bc1e2919461c80944cbe Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Mon, 18 Nov 2024 16:48:55 +0000
Subject: [PATCH 12/37] [CI] Auto-commit changed files from 'node
 scripts/eslint --no-cache --fix'

---
 x-pack/plugins/entity_manager/server/lib/queries/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index d693e1c91106b..babe570702d9b 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import z from 'zod';
+import z from '@kbn/zod';
 
 export const entitySourceSchema = z.object({
   type: z.string(),

From 7451b899cae91c5aaeed12fce2712ccf19089893 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 19 Nov 2024 18:06:40 +0100
Subject: [PATCH 13/37] fetch sources from kibana_entity_definitions

---
 .../server/lib/entity_client.ts               | 19 +++++++++-
 .../server/lib/queries/index.ts               |  2 +-
 .../server/lib/queries/utils.ts               | 38 +++++++++++++++++++
 .../server/routes/entities/search.ts          | 17 +++------
 4 files changed, 62 insertions(+), 14 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 104992a1a044f..9ed61916f56f6 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -25,7 +25,7 @@ import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
 import { EntitySource, getEntityInstancesQuery } from './queries';
-import { mergeEntitiesList } from './queries/utils';
+import { mergeEntitiesList, runESQLQuery } from './queries/utils';
 
 export class EntityClient {
   constructor(
@@ -172,6 +172,21 @@ export class EntityClient {
     return stopTransforms(this.options.esClient, definition, this.options.logger);
   }
 
+  async getEntitySources({ type }: { type: string }) {
+    const result = await this.options.esClient.search<EntitySource>({
+      index: 'kibana_entity_definitions',
+      query: {
+        bool: {
+          must: {
+            term: { entity_type: type },
+          },
+        },
+      },
+    });
+
+    return result.hits.hits.map((hit) => hit._source) as EntitySource[];
+  }
+
   async searchEntities({ sources, limit = 10 }: { sources: EntitySource[]; limit?: number }) {
     const entities = await Promise.all(
       sources.map(async (source) => {
@@ -203,7 +218,7 @@ export class EntityClient {
         );
         this.options.logger.info(`Entity query: ${query}`);
 
-        const rawEntities = await esClient.esql<Entity>('search_entities', { query });
+        const rawEntities = await runESQLQuery<Entity>({ query, esClient: this.options.esClient });
 
         return rawEntities.map((entity) => {
           entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index babe570702d9b..acf96da699337 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import z from '@kbn/zod';
+import { z } from '@kbn/zod';
 
 export const entitySourceSchema = z.object({
   type: z.string(),
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
index 9661fb3c19bfd..38cf525d7c9a3 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
@@ -5,7 +5,9 @@
  * 2.0.
  */
 
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Entity } from '@kbn/entities-schema';
+import { ESQLSearchResponse } from '@kbn/es-types';
 import { uniq } from 'lodash';
 
 function mergeEntities(entity1: Entity, entity2: Entity): Entity {
@@ -49,3 +51,39 @@ export function mergeEntitiesList(entities: Entity[]): Entity[] {
 
   return Object.values(instances);
 }
+
+export async function runESQLQuery<T>({
+  esClient,
+  query,
+}: {
+  esClient: ElasticsearchClient;
+  query: string;
+}): Promise<T[]> {
+  const esqlResponse = (await esClient.esql.query(
+    {
+      query,
+      format: 'json',
+    },
+    { querystring: { drop_null_columns: true } }
+  )) as unknown as ESQLSearchResponse;
+
+  const documents = esqlResponse.values.map((row) =>
+    row.reduce<Record<string, any>>((acc, value, index) => {
+      const column = esqlResponse.columns[index];
+
+      if (!column) {
+        return acc;
+      }
+
+      // Removes the type suffix from the column name
+      const name = column.name.replace(/\.(text|keyword)$/, '');
+      if (!acc[name]) {
+        acc[name] = value;
+      }
+
+      return acc;
+    }, {})
+  ) as T[];
+
+  return documents;
+}
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
index 25a9e28e35e18..44ca74c3067ef 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -8,7 +8,7 @@
 import moment from 'moment';
 import { z } from '@kbn/zod';
 import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
-import { EntitySource, entitySourceSchema } from '../../lib/queries';
+import { entitySourceSchema } from '../../lib/queries';
 
 export const searchEntitiesRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/_search',
@@ -23,16 +23,11 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
       const { type, limit } = params.body;
 
       const client = await getScopedClient({ request });
-      const sources = [
-        {
-          type: 'service',
-          timestamp_field: '@timestamp',
-          index_patterns: ['remote_cluster:logs-*'],
-          identity_fields: ['service.name', 'service.environment'],
-          metadata_fields: [],
-          filters: [],
-        },
-      ] as EntitySource[]; // get sources for the type
+
+      const sources = await client.getEntitySources({ type });
+      if (sources.length === 0) {
+        return response.notFound({ body: { message: `No sources found for type [${type}]` } });
+      }
 
       const entities = await client.searchEntities({ sources, limit });
 

From 83db43a255ed0b6fca1cfe52a462ccdf59975b2c Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Tue, 19 Nov 2024 18:11:40 +0000
Subject: [PATCH 14/37] [CI] Auto-commit changed files from 'node
 scripts/yarn_deduplicate'

---
 x-pack/plugins/entity_manager/tsconfig.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json
index bbb4fcd5b6495..d8a8f6654b997 100644
--- a/x-pack/plugins/entity_manager/tsconfig.json
+++ b/x-pack/plugins/entity_manager/tsconfig.json
@@ -50,5 +50,6 @@
     "@kbn/data-plugin",
     "@kbn/observability-ai-assistant-plugin",
     "@kbn/observability-utils-server",
+    "@kbn/es-types",
   ]
 }

From 6f07b1c4098203cf962eb78e36aa738acbf85840 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 19 Nov 2024 21:28:09 +0100
Subject: [PATCH 15/37] move ui to entity_manager_app

---
 package.json                                  |  1 +
 tsconfig.base.json                            |  2 +
 x-pack/plugins/entity_manager/kibana.jsonc    | 22 +----
 x-pack/plugins/entity_manager/public/index.ts |  2 +
 .../plugins/entity_manager/public/plugin.ts   | 66 ++-------------
 x-pack/plugins/entity_manager/public/types.ts | 27 -------
 .../entity_manager_app/README.md              |  3 +
 .../entity_manager_app/jest.config.js         | 18 +++++
 .../entity_manager_app/kibana.jsonc           | 29 +++++++
 .../public/application.tsx                    | 22 +----
 .../public/context/plugin_context.ts          |  2 +-
 .../public/hooks/use_kibana.ts                |  2 +-
 .../public/hooks/use_plugin_context.ts        |  0
 .../entity_manager_app/public/index.ts        | 13 +++
 .../public/pages/overview/index.tsx           | 15 +++-
 .../entity_manager_app/public/plugin.ts       | 80 +++++++++++++++++++
 .../entity_manager_app}/public/routes.tsx     |  0
 .../entity_manager_app/public/types.ts        | 32 ++++++++
 .../entity_manager_app/tsconfig.json          | 54 +++++++++++++
 yarn.lock                                     |  4 +
 20 files changed, 264 insertions(+), 130 deletions(-)
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/README.md
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/jest.config.js
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
 rename x-pack/plugins/{entity_manager => observability_solution/entity_manager_app}/public/application.tsx (88%)
 rename x-pack/plugins/{entity_manager => observability_solution/entity_manager_app}/public/context/plugin_context.ts (92%)
 rename x-pack/plugins/{entity_manager => observability_solution/entity_manager_app}/public/hooks/use_kibana.ts (92%)
 rename x-pack/plugins/{entity_manager => observability_solution/entity_manager_app}/public/hooks/use_plugin_context.ts (100%)
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/index.ts
 rename x-pack/plugins/{entity_manager => observability_solution/entity_manager_app}/public/pages/overview/index.tsx (92%)
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
 rename x-pack/plugins/{entity_manager => observability_solution/entity_manager_app}/public/routes.tsx (100%)
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
 create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json

diff --git a/package.json b/package.json
index eed5b3a9b61cb..c66479a87e90e 100644
--- a/package.json
+++ b/package.json
@@ -479,6 +479,7 @@
     "@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access",
     "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
     "@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin",
+    "@kbn/entityManager-app-plugin": "link:x-pack/plugins/observability_solution/entity_manager_app",
     "@kbn/entityManager-plugin": "link:x-pack/plugins/entity_manager",
     "@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
     "@kbn/es-errors": "link:packages/kbn-es-errors",
diff --git a/tsconfig.base.json b/tsconfig.base.json
index 3e1d80208f5b4..86be828412324 100644
--- a/tsconfig.base.json
+++ b/tsconfig.base.json
@@ -820,6 +820,8 @@
       "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
       "@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"],
       "@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"],
+      "@kbn/entityManager-app-plugin": ["x-pack/plugins/observability_solution/entity_manager_app"],
+      "@kbn/entityManager-app-plugin/*": ["x-pack/plugins/observability_solution/entity_manager_app/*"],
       "@kbn/entityManager-plugin": ["x-pack/plugins/entity_manager"],
       "@kbn/entityManager-plugin/*": ["x-pack/plugins/entity_manager/*"],
       "@kbn/error-boundary-example-plugin": ["examples/error_boundary"],
diff --git a/x-pack/plugins/entity_manager/kibana.jsonc b/x-pack/plugins/entity_manager/kibana.jsonc
index 6615a534bf30c..c18822d48ac0a 100644
--- a/x-pack/plugins/entity_manager/kibana.jsonc
+++ b/x-pack/plugins/entity_manager/kibana.jsonc
@@ -11,28 +11,10 @@
     "browser": true,
     "server": true,
     "requiredPlugins": [
-      "data",
-      "dataViews",
-      "dataViewEditor",
-      "dataViewFieldEditor",
       "security",
       "encryptedSavedObjects",
-      "lens",
-      "observability",
-      "observabilityShared",
-      "embeddable",
-      "presentationUtil",
-      "usageCollection",
-      "licensing",
-      "transform"
+      "licensing"
     ],
-    "optionalPlugins": [
-      "cloud",
-      "serverless"
-    ],
-    "requiredBundles": [
-      "kibanaReact",
-      "kibanaUtils"
-    ]
+    "requiredBundles": []
   }
 }
diff --git a/x-pack/plugins/entity_manager/public/index.ts b/x-pack/plugins/entity_manager/public/index.ts
index 85a9285b1692d..73d23ad45e9c1 100644
--- a/x-pack/plugins/entity_manager/public/index.ts
+++ b/x-pack/plugins/entity_manager/public/index.ts
@@ -16,6 +16,8 @@ export const plugin: PluginInitializer<
   return new Plugin(context);
 };
 
+export { EntityClient } from './lib/entity_client';
+
 export type { EntityManagerPublicPluginSetup, EntityManagerPublicPluginStart };
 export type EntityManagerAppId = 'entityManager';
 
diff --git a/x-pack/plugins/entity_manager/public/plugin.ts b/x-pack/plugins/entity_manager/public/plugin.ts
index cec49a45f4109..7ff6354c997eb 100644
--- a/x-pack/plugins/entity_manager/public/plugin.ts
+++ b/x-pack/plugins/entity_manager/public/plugin.ts
@@ -5,87 +5,31 @@
  * 2.0.
  */
 
-import { BehaviorSubject } from 'rxjs';
-import {
-  App,
-  AppMountParameters,
-  AppStatus,
-  AppUpdater,
-  CoreSetup,
-  CoreStart,
-  DEFAULT_APP_CATEGORIES,
-  PluginInitializerContext,
-} from '@kbn/core/public';
+import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public';
 import { Logger } from '@kbn/logging';
 
-import {
-  EntityManagerPluginClass,
-  EntityManagerPluginSetup,
-  EntityManagerPluginStart,
-  EntityManagerPublicPluginStart,
-} from './types';
+import { EntityManagerPluginClass } from './types';
 import type { EntityManagerPublicConfig } from '../common/config';
 import { EntityClient } from './lib/entity_client';
 
 export class Plugin implements EntityManagerPluginClass {
   public config: EntityManagerPublicConfig;
   public logger: Logger;
-  private readonly appUpdater$ = new BehaviorSubject<AppUpdater>(() => ({}));
 
-  constructor(private readonly context: PluginInitializerContext<{}>) {
+  constructor(context: PluginInitializerContext<{}>) {
     this.config = context.config.get();
     this.logger = context.logger.get();
   }
 
-  setup(
-    core: CoreSetup<EntityManagerPluginStart, EntityManagerPublicPluginStart>,
-    pluginSetup: EntityManagerPluginSetup
-  ) {
-    const kibanaVersion = this.context.env.packageInfo.version;
-
-    const mount = async (params: AppMountParameters<unknown>) => {
-      const { renderApp } = await import('./application');
-      const [coreStart, pluginsStart] = await core.getStartServices();
-
-      return renderApp({
-        appMountParameters: params,
-        core: coreStart,
-        isDev: this.context.env.mode.dev,
-        kibanaVersion,
-        usageCollection: pluginSetup.usageCollection,
-        ObservabilityPageTemplate: pluginsStart.observabilityShared.navigation.PageTemplate,
-        plugins: pluginsStart,
-        isServerless: !!pluginsStart.serverless,
-        entityClient: new EntityClient(core),
-      });
-    };
-
-    const appUpdater$ = this.appUpdater$;
-    const app: App = {
-      id: 'entity_manager',
-      title: 'Entity Manager',
-      order: 8002,
-      updater$: appUpdater$,
-      euiIconType: 'logoObservability',
-      appRoute: '/app/entity_manager',
-      category: DEFAULT_APP_CATEGORIES.observability,
-      mount,
-      visibleIn: [],
-      keywords: ['observability', 'monitor', 'entities'],
-      status: AppStatus.inaccessible,
-    };
-
-    core.application.register(app);
-
+  setup(core: CoreSetup) {
     return {
       entityClient: new EntityClient(core),
     };
   }
 
   start(core: CoreStart) {
-    const entityClient = new EntityClient(core);
     return {
-      entityClient,
+      entityClient: new EntityClient(core),
     };
   }
 
diff --git a/x-pack/plugins/entity_manager/public/types.ts b/x-pack/plugins/entity_manager/public/types.ts
index a797f9b7b0ebb..90d9026e8b9ba 100644
--- a/x-pack/plugins/entity_manager/public/types.ts
+++ b/x-pack/plugins/entity_manager/public/types.ts
@@ -5,38 +5,11 @@
  * 2.0.
  */
 import type { Plugin as PluginClass } from '@kbn/core/public';
-import { PresentationUtilPluginStart } from '@kbn/presentation-util-plugin/public';
-import { CloudStart } from '@kbn/cloud-plugin/public';
-import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public';
-import { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public';
-import {
-  ObservabilitySharedPluginSetup,
-  ObservabilitySharedPluginStart,
-} from '@kbn/observability-shared-plugin/public';
-import { ObservabilityAIAssistantPublicSetup } from '@kbn/observability-ai-assistant-plugin/public';
-import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import type { EntityClient } from './lib/entity_client';
 
-export interface EntityManagerPluginSetup {
-  data: DataPublicPluginSetup;
-  observability: ObservabilityAIAssistantPublicSetup;
-  observabilityShared: ObservabilitySharedPluginSetup;
-  serverless?: ServerlessPluginSetup;
-  usageCollection: UsageCollectionSetup;
-}
-
-export interface EntityManagerPluginStart {
-  data: DataPublicPluginStart;
-  presentationUtil: PresentationUtilPluginStart;
-  cloud?: CloudStart;
-  serverless?: ServerlessPluginStart;
-  observabilityShared: ObservabilitySharedPluginStart;
-}
-
 export interface EntityManagerPublicPluginSetup {
   entityClient: EntityClient;
 }
-
 export interface EntityManagerPublicPluginStart {
   entityClient: EntityClient;
 }
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/README.md b/x-pack/plugins/observability_solution/entity_manager_app/README.md
new file mode 100644
index 0000000000000..1fd230a046c54
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/README.md
@@ -0,0 +1,3 @@
+# Entity Manager App Plugin
+
+This plugin provides a user interface to interact with the Entity Manager.
\ No newline at end of file
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js b/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js
new file mode 100644
index 0000000000000..eef557bb41a68
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/observability_solution/entity_manager_app'],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/observability_solution/entity_manager_app',
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/observability_solution/entity_manager_app/{common,public,server}/**/*.{js,ts,tsx}',
+  ],
+};
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
new file mode 100644
index 0000000000000..93e6687f9b4c3
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
@@ -0,0 +1,29 @@
+{
+  "type": "plugin",
+  "id": "@kbn/entityManager-app-plugin",
+  "owner": "@elastic/obs-entities",
+  "group": "observability",
+  "visibility": "private",
+  "description": "Entity manager plugin for entity assets (inventory, topology, etc)",
+  "plugin": {
+    "id": "entityManagerApp",
+    "configPath": ["xpack", "entityManagerApp"],
+    "browser": true,
+    "server": false,
+    "requiredPlugins": [
+      "entityManager",
+      "observabilityShared",
+      "presentationUtil",
+      "usageCollection",
+      "licensing"
+    ],
+    "optionalPlugins": [
+      "cloud",
+      "serverless"
+    ],
+    "requiredBundles": [
+      "kibanaReact",
+      "kibanaUtils"
+    ]
+  }
+}
diff --git a/x-pack/plugins/entity_manager/public/application.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
similarity index 88%
rename from x-pack/plugins/entity_manager/public/application.tsx
rename to x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
index 3680aed1dee59..5a5bb9c29b154 100644
--- a/x-pack/plugins/entity_manager/public/application.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
@@ -14,29 +14,15 @@ import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
 import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme';
 import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { EntityClient } from '@kbn/entityManager-plugin/public';
 import React from 'react';
 import ReactDOM from 'react-dom';
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
-import { Route, Router, Routes } from '@kbn/shared-ux-router';
+import { Router } from '@kbn/shared-ux-router';
 import { PluginContext } from './context/plugin_context';
-import { EntityManagerPluginStart } from './types';
-import { getRoutes } from './routes';
-import { EntityClient } from './lib/entity_client';
+import { EntityManagerAppPluginStart } from './types';
 import { EntityManagerOverviewPage } from './pages/overview';
 
-function App() {
-  const routes = getRoutes();
-  return (
-    <Routes>
-      {Object.keys(routes).map((path) => {
-        const { handler, exact } = routes[path];
-        const Wrapper = () => handler();
-        return <Route key={path} path={path} exact={exact} component={Wrapper} />;
-      })}
-    </Routes>
-  );
-}
-
 export function renderApp({
   core,
   plugins,
@@ -49,7 +35,7 @@ export function renderApp({
   entityClient,
 }: {
   core: CoreStart;
-  plugins: EntityManagerPluginStart;
+  plugins: EntityManagerAppPluginStart;
   appMountParameters: AppMountParameters;
   ObservabilityPageTemplate: React.ComponentType<LazyObservabilityPageTemplateProps>;
   usageCollection: UsageCollectionSetup;
diff --git a/x-pack/plugins/entity_manager/public/context/plugin_context.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts
similarity index 92%
rename from x-pack/plugins/entity_manager/public/context/plugin_context.ts
rename to x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts
index 83c5bb2735b7c..7da2833be4395 100644
--- a/x-pack/plugins/entity_manager/public/context/plugin_context.ts
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts
@@ -8,7 +8,7 @@
 import { createContext } from 'react';
 import type { AppMountParameters } from '@kbn/core/public';
 import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public';
-import { EntityClient } from '../lib/entity_client';
+import { EntityClient } from '@kbn/entityManager-plugin/public';
 
 export interface PluginContextValue {
   isDev?: boolean;
diff --git a/x-pack/plugins/entity_manager/public/hooks/use_kibana.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts
similarity index 92%
rename from x-pack/plugins/entity_manager/public/hooks/use_kibana.ts
rename to x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts
index 6d6b9e49b4dac..a515b9b80b014 100644
--- a/x-pack/plugins/entity_manager/public/hooks/use_kibana.ts
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts
@@ -8,7 +8,7 @@
 import { CoreStart } from '@kbn/core/public';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { Storage } from '@kbn/kibana-utils-plugin/public';
-import { EntityClient } from '../lib/entity_client';
+import { EntityClient } from '@kbn/entityManager-plugin/public';
 
 export type StartServices<AdditionalServices extends object = {}> = CoreStart &
   AdditionalServices & {
diff --git a/x-pack/plugins/entity_manager/public/hooks/use_plugin_context.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_plugin_context.ts
similarity index 100%
rename from x-pack/plugins/entity_manager/public/hooks/use_plugin_context.ts
rename to x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_plugin_context.ts
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/index.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/index.ts
new file mode 100644
index 0000000000000..5b83ea1d297d3
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/index.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public';
+import { Plugin } from './plugin';
+
+export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => {
+  return new Plugin(context);
+};
diff --git a/x-pack/plugins/entity_manager/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
similarity index 92%
rename from x-pack/plugins/entity_manager/public/pages/overview/index.tsx
rename to x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
index 8f68312064a7a..222d760f480dd 100644
--- a/x-pack/plugins/entity_manager/public/pages/overview/index.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
@@ -47,6 +47,7 @@ function EntitySourceForm({
       <EuiFlexItem>
         <EuiFormRow label="Index patterns (comma-separated)">
           <EuiFieldText
+            data-test-subj="entityManagerFormIndexPatterns"
             name="index_patterns"
             defaultValue={source.index_patterns.join(',')}
             isInvalid={source.index_patterns.length === 0}
@@ -58,6 +59,7 @@ function EntitySourceForm({
       <EuiFlexItem>
         <EuiFormRow label="Identify fields (comma-separated field names)">
           <EuiFieldText
+            data-test-subj="entityManagerFormIdentityFields"
             name="identity_fields"
             defaultValue={source.identity_fields.join(',')}
             isInvalid={source.identity_fields.length === 0}
@@ -69,6 +71,7 @@ function EntitySourceForm({
       <EuiFlexItem>
         <EuiFormRow label="Filters (comma-separated ESQL filters)">
           <EuiFieldText
+            data-test-subj="entityManagerFormFilters"
             name="filters"
             defaultValue={source.filters.join(',')}
             onChange={onArrayFieldChange('filters')}
@@ -79,6 +82,7 @@ function EntitySourceForm({
       <EuiFlexItem>
         <EuiFormRow label="Metadata (comma-separated field names)">
           <EuiFieldText
+            data-test-subj="entityManagerFormMetadata"
             name="metadata"
             defaultValue={source.metadata_fields.join(',')}
             onChange={onArrayFieldChange('metadata_fields')}
@@ -186,6 +190,7 @@ export function EntityManagerOverviewPage() {
           <EuiFlexItem>
             <EuiFormRow>
               <EuiFieldText
+                data-test-subj="entityManagerFormType"
                 name="type"
                 defaultValue={entityType}
                 placeholder="host, service, user..."
@@ -208,6 +213,7 @@ export function EntityManagerOverviewPage() {
 
           <EuiFlexItem grow={false}>
             <EuiButtonIcon
+              data-test-subj="entityManagerFormAddSource"
               iconType="plusInCircle"
               onClick={() => setEntitySources([...entitySources, newEntitySource({})])}
             />
@@ -227,6 +233,7 @@ export function EntityManagerOverviewPage() {
               {entitySources.length > 1 ? (
                 <EuiFlexItem grow={false}>
                   <EuiButtonIcon
+                    data-test-subj="entityManagerFormRemoveSource"
                     color={'danger'}
                     iconType={'minusInCircle'}
                     onClick={() => {
@@ -263,12 +270,16 @@ export function EntityManagerOverviewPage() {
         <EuiFormRow>
           <EuiFlexGroup>
             <EuiFlexItem>
-              <EuiButton isDisabled={isSearchingEntities} onClick={searchEntities}>
+              <EuiButton
+                data-test-subj="entityManagerFormPreview"
+                isDisabled={isSearchingEntities}
+                onClick={searchEntities}
+              >
                 Preview
               </EuiButton>
             </EuiFlexItem>
             <EuiFlexItem>
-              <EuiButton isDisabled={true} color="primary">
+              <EuiButton data-test-subj="entityManagerFormCreate" isDisabled={true} color="primary">
                 Create
               </EuiButton>
             </EuiFlexItem>
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
new file mode 100644
index 0000000000000..0db381522b020
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { BehaviorSubject } from 'rxjs';
+import {
+  App,
+  AppMountParameters,
+  AppStatus,
+  AppUpdater,
+  CoreSetup,
+  DEFAULT_APP_CATEGORIES,
+  PluginInitializerContext,
+} from '@kbn/core/public';
+import { Logger } from '@kbn/logging';
+import { EntityClient } from '@kbn/entityManager-plugin/public';
+
+import {
+  EntityManagerAppPluginClass,
+  EntityManagerPluginStart,
+  EntityManagerPluginSetup,
+} from './types';
+
+export class Plugin implements EntityManagerAppPluginClass {
+  public logger: Logger;
+  private readonly appUpdater$ = new BehaviorSubject<AppUpdater>(() => ({}));
+
+  constructor(private readonly context: PluginInitializerContext<{}>) {
+    this.logger = context.logger.get();
+  }
+
+  setup(core: CoreSetup<EntityManagerPluginStart, {}>, pluginSetup: EntityManagerPluginSetup) {
+    const kibanaVersion = this.context.env.packageInfo.version;
+
+    const mount = async (params: AppMountParameters<unknown>) => {
+      const { renderApp } = await import('./application');
+      const [coreStart, pluginsStart] = await core.getStartServices();
+
+      return renderApp({
+        appMountParameters: params,
+        core: coreStart,
+        isDev: this.context.env.mode.dev,
+        kibanaVersion,
+        usageCollection: pluginSetup.usageCollection,
+        ObservabilityPageTemplate: pluginsStart.observabilityShared.navigation.PageTemplate,
+        plugins: pluginsStart,
+        isServerless: !!pluginsStart.serverless,
+        entityClient: new EntityClient(core),
+      });
+    };
+
+    const appUpdater$ = this.appUpdater$;
+    const app: App = {
+      id: 'entity_manager',
+      title: 'Entity Manager',
+      order: 8002,
+      updater$: appUpdater$,
+      euiIconType: 'logoObservability',
+      appRoute: '/app/entity_manager',
+      category: DEFAULT_APP_CATEGORIES.observability,
+      mount,
+      visibleIn: [],
+      keywords: ['observability', 'monitor', 'entities'],
+      status: AppStatus.inaccessible,
+    };
+
+    core.application.register(app);
+
+    return {};
+  }
+
+  start() {
+    return {};
+  }
+
+  stop() {}
+}
diff --git a/x-pack/plugins/entity_manager/public/routes.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/routes.tsx
similarity index 100%
rename from x-pack/plugins/entity_manager/public/routes.tsx
rename to x-pack/plugins/observability_solution/entity_manager_app/public/routes.tsx
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
new file mode 100644
index 0000000000000..b735771d79f80
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { Plugin as PluginClass } from '@kbn/core/public';
+import { PresentationUtilPluginStart } from '@kbn/presentation-util-plugin/public';
+import { CloudStart } from '@kbn/cloud-plugin/public';
+import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public';
+import {
+  ObservabilitySharedPluginSetup,
+  ObservabilitySharedPluginStart,
+} from '@kbn/observability-shared-plugin/public';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
+import { EntityManagerPublicPluginSetup } from '@kbn/entityManager-plugin/public/types';
+
+export interface EntityManagerPluginSetup {
+  observabilityShared: ObservabilitySharedPluginSetup;
+  serverless?: ServerlessPluginSetup;
+  usageCollection: UsageCollectionSetup;
+  entityManager: EntityManagerPublicPluginSetup;
+}
+
+export interface EntityManagerPluginStart {
+  presentationUtil: PresentationUtilPluginStart;
+  cloud?: CloudStart;
+  serverless?: ServerlessPluginStart;
+  observabilityShared: ObservabilitySharedPluginStart;
+}
+
+export type EntityManagerAppPluginClass = PluginClass<{}, {}>;
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
new file mode 100644
index 0000000000000..7f0162d31b585
--- /dev/null
+++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
@@ -0,0 +1,54 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": [
+    "../../../typings/**/*",
+    "common/**/*",
+    "public/**/*",
+    "types/**/*"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/config-schema",
+    "@kbn/entities-schema",
+    "@kbn/core",
+    "@kbn/core-plugins-server",
+    "@kbn/server-route-repository-client",
+    "@kbn/logging",
+    "@kbn/core-http-server",
+    "@kbn/security-plugin",
+    "@kbn/es-query",
+    "@kbn/core-elasticsearch-server",
+    "@kbn/core-saved-objects-api-server",
+    "@kbn/core-elasticsearch-client-server-mocks",
+    "@kbn/core-saved-objects-api-server-mocks",
+    "@kbn/logging-mocks",
+    "@kbn/datemath",
+    "@kbn/server-route-repository",
+    "@kbn/zod",
+    "@kbn/zod-helpers",
+    "@kbn/encrypted-saved-objects-plugin",
+    "@kbn/licensing-plugin",
+    "@kbn/core-saved-objects-server",
+    "@kbn/ebt-tools",
+    "@kbn/kibana-react-plugin",
+    "@kbn/kibana-utils-plugin",
+    "@kbn/observability-shared-plugin",
+    "@kbn/react-kibana-context-render",
+    "@kbn/react-kibana-context-theme",
+    "@kbn/shared-ux-link-redirect-app",
+    "@kbn/usage-collection-plugin",
+    "@kbn/shared-ux-router",
+    "@kbn/presentation-util-plugin",
+    "@kbn/cloud-plugin",
+    "@kbn/serverless",
+    "@kbn/data-plugin",
+    "@kbn/observability-ai-assistant-plugin",
+    "@kbn/observability-utils-server",
+    "@kbn/es-types",
+  ]
+}
diff --git a/yarn.lock b/yarn.lock
index 5b3b6f246ff42..942dda6c411dd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4908,6 +4908,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/entityManager-app-plugin@link:x-pack/plugins/observability_solution/entity_manager_app":
+  version "0.0.0"
+  uid ""
+
 "@kbn/entityManager-plugin@link:x-pack/plugins/entity_manager":
   version "0.0.0"
   uid ""

From 2872334d8843099340fb19176a755b649e39b42b Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 19 Nov 2024 21:46:34 +0100
Subject: [PATCH 16/37] add start/end to _search

---
 .../server/lib/entity_client.ts               | 22 +++++++---
 .../server/lib/queries/index.ts               | 40 +++++++++++++++----
 .../server/routes/entities/search.ts          | 16 +++++++-
 3 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 9ed61916f56f6..9224129fa27c9 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -187,7 +187,17 @@ export class EntityClient {
     return result.hits.hits.map((hit) => hit._source) as EntitySource[];
   }
 
-  async searchEntities({ sources, limit = 10 }: { sources: EntitySource[]; limit?: number }) {
+  async searchEntities({
+    sources,
+    start,
+    end,
+    limit = 10,
+  }: {
+    sources: EntitySource[];
+    start?: string;
+    end?: string;
+    limit?: number;
+  }) {
     const entities = await Promise.all(
       sources.map(async (source) => {
         const esClient = createObservabilityEsClient({
@@ -212,10 +222,12 @@ export class EntityClient {
         // but metadata field not being available is fine
         const availableMetadataFields = source.metadata_fields.filter((field) => fields[field]);
 
-        const query = getEntityInstancesQuery(
-          { ...source, metadata_fields: availableMetadataFields },
-          limit
-        );
+        const query = getEntityInstancesQuery({
+          source: { ...source, metadata_fields: availableMetadataFields },
+          start,
+          end,
+          limit,
+        });
         this.options.logger.info(`Entity query: ${query}`);
 
         const rawEntities = await runESQLQuery<Entity>({ query, esClient: this.options.esClient });
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index acf96da699337..c04856caa01e1 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -18,7 +18,7 @@ export const entitySourceSchema = z.object({
 
 export type EntitySource = z.infer<typeof entitySourceSchema>;
 
-const sourceCommand = (source: EntitySource) => {
+const sourceCommand = ({ source }: { source: EntitySource }) => {
   let query = `FROM ${source.index_patterns}`;
 
   const esMetadataFields = source.metadata_fields.filter((field) =>
@@ -31,7 +31,15 @@ const sourceCommand = (source: EntitySource) => {
   return query;
 };
 
-const filterCommands = (source: EntitySource) => {
+const filterCommands = ({
+  source,
+  start,
+  end,
+}: {
+  source: EntitySource;
+  start?: string;
+  end?: string;
+}) => {
   const commands: string[] = [];
 
   source.identity_fields.forEach((field) => {
@@ -42,10 +50,18 @@ const filterCommands = (source: EntitySource) => {
     commands.push(`WHERE ${filter}`);
   });
 
+  if (start) {
+    commands.push(`WHERE ${source.timestamp_field} >= "${start}"`);
+  }
+
+  if (end) {
+    commands.push(`WHERE ${source.timestamp_field} <= "${end}"`);
+  }
+
   return commands;
 };
 
-const statsCommand = (source: EntitySource) => {
+const statsCommand = ({ source }: { source: EntitySource }) => {
   const aggs = [
     // default 'last_seen' attribute
     `entity.last_seen_timestamp=MAX(${source.timestamp_field})`,
@@ -57,11 +73,21 @@ const statsCommand = (source: EntitySource) => {
   return `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(',')}`;
 };
 
-export function getEntityInstancesQuery(source: EntitySource, limit: number): string {
+export function getEntityInstancesQuery({
+  source,
+  limit,
+  start,
+  end,
+}: {
+  source: EntitySource;
+  limit: number;
+  start?: string;
+  end?: string;
+}): string {
   const commands = [
-    sourceCommand(source),
-    ...filterCommands(source),
-    statsCommand(source),
+    sourceCommand({ source }),
+    ...filterCommands({ source, start, end }),
+    statsCommand({ source }),
     `SORT entity.last_seen_timestamp DESC`,
     `LIMIT ${limit}`,
   ];
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
index 44ca74c3067ef..34d292eeaec46 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -15,12 +15,24 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
   params: z.object({
     body: z.object({
       type: z.string(),
+      start: z
+        .optional(z.string())
+        .default(() => moment().subtract(5, 'minutes').toISOString())
+        .refine((val) => moment(val).isValid(), {
+          message: 'start should be a date in ISO format',
+        }),
+      end: z
+        .optional(z.string())
+        .default(() => moment().toISOString())
+        .refine((val) => moment(val).isValid(), {
+          message: 'start should be a date in ISO format',
+        }),
       limit: z.optional(z.number()).default(10),
     }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
     try {
-      const { type, limit } = params.body;
+      const { type, start, end, limit } = params.body;
 
       const client = await getScopedClient({ request });
 
@@ -29,7 +41,7 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
         return response.notFound({ body: { message: `No sources found for type [${type}]` } });
       }
 
-      const entities = await client.searchEntities({ sources, limit });
+      const entities = await client.searchEntities({ sources, start, end, limit });
 
       return response.ok({ body: { entities } });
     } catch (e) {

From a64dccf202022406a7376b9455833484bafb95cd Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 19 Nov 2024 22:02:49 +0100
Subject: [PATCH 17/37] non optional start/end

---
 .../server/lib/entity_client.ts               | 13 +++-------
 .../server/lib/queries/index.ts               | 21 ++++++---------
 .../server/routes/entities/search.ts          | 26 ++++++++++++-------
 3 files changed, 27 insertions(+), 33 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 9224129fa27c9..e39b61ac74928 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -7,7 +7,6 @@
 
 import { Entity, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
-import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
 import {
@@ -194,20 +193,14 @@ export class EntityClient {
     limit = 10,
   }: {
     sources: EntitySource[];
-    start?: string;
-    end?: string;
+    start: string;
+    end: string;
     limit?: number;
   }) {
     const entities = await Promise.all(
       sources.map(async (source) => {
-        const esClient = createObservabilityEsClient({
-          client: this.options.esClient,
-          logger: this.options.logger,
-          plugin: `@kbn/entityManager-plugin`,
-        });
-
         const mandatoryFields = [source.timestamp_field, ...source.identity_fields];
-        const { fields } = await esClient.client.fieldCaps({
+        const { fields } = await this.options.esClient.fieldCaps({
           index: source.index_patterns,
           fields: [...mandatoryFields, ...source.metadata_fields],
         });
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index c04856caa01e1..f0d7ad913d2df 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -37,10 +37,13 @@ const filterCommands = ({
   end,
 }: {
   source: EntitySource;
-  start?: string;
-  end?: string;
+  start: string;
+  end: string;
 }) => {
-  const commands: string[] = [];
+  const commands = [
+    `WHERE ${source.timestamp_field} >= "${start}"`,
+    `WHERE ${source.timestamp_field} <= "${end}"`,
+  ];
 
   source.identity_fields.forEach((field) => {
     commands.push(`WHERE ${field} IS NOT NULL`);
@@ -50,14 +53,6 @@ const filterCommands = ({
     commands.push(`WHERE ${filter}`);
   });
 
-  if (start) {
-    commands.push(`WHERE ${source.timestamp_field} >= "${start}"`);
-  }
-
-  if (end) {
-    commands.push(`WHERE ${source.timestamp_field} <= "${end}"`);
-  }
-
   return commands;
 };
 
@@ -81,8 +76,8 @@ export function getEntityInstancesQuery({
 }: {
   source: EntitySource;
   limit: number;
-  start?: string;
-  end?: string;
+  start: string;
+  end: string;
 }): string {
   const commands = [
     sourceCommand({ source }),
diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
index 34d292eeaec46..8edfef17f5973 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/search.ts
@@ -55,25 +55,31 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/_search/preview',
   params: z.object({
     body: z.object({
-      sources: z.array(
-        entitySourceSchema.transform((source) => ({
-          ...source,
-          filters: [
-            ...source.filters,
-            `${source.timestamp_field} >= "${moment().subtract(5, 'minutes').toISOString()}"`,
-          ],
-        }))
-      ),
+      sources: z.array(entitySourceSchema),
+      start: z
+        .optional(z.string())
+        .default(() => moment().subtract(5, 'minutes').toISOString())
+        .refine((val) => moment(val).isValid(), {
+          message: 'start should be a date in ISO format',
+        }),
+      end: z
+        .optional(z.string())
+        .default(() => moment().toISOString())
+        .refine((val) => moment(val).isValid(), {
+          message: 'start should be a date in ISO format',
+        }),
       limit: z.optional(z.number()).default(10),
     }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
     try {
-      const { sources, limit } = params.body;
+      const { sources, start, end, limit } = params.body;
 
       const client = await getScopedClient({ request });
       const entities = await client.searchEntities({
         sources,
+        start,
+        end,
         limit,
       });
 

From 39f7758f2dd05a9610304146db78e831d77671a5 Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Tue, 19 Nov 2024 21:13:24 +0000
Subject: [PATCH 18/37] [CI] Auto-commit changed files from 'node
 scripts/build_plugin_list_docs'

---
 .github/CODEOWNERS                  | 1 +
 docs/developer/plugin-list.asciidoc | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 6898e69a44eeb..ea9a58d711d34 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -919,6 +919,7 @@ x-pack/plugins/observability_solution/apm_data_access @elastic/obs-knowledge-tea
 x-pack/plugins/observability_solution/apm/ftr_e2e @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/dataset_quality @elastic/obs-ux-logs-team
 x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
+x-pack/plugins/observability_solution/entity_manager_app @elastic/obs-entities
 x-pack/plugins/observability_solution/exploratory_view @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team
diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc
index ea31863576115..d835bb9139e1e 100644
--- a/docs/developer/plugin-list.asciidoc
+++ b/docs/developer/plugin-list.asciidoc
@@ -579,6 +579,10 @@ security and spaces filtering.
 |This plugin provides access to observed entity data, such as information about hosts, pods, containers, services, and more.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/entity_manager_app/README.md[entityManagerApp]
+|This plugin provides a user interface to interact with the Entity Manager.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/event_log/README.md[eventLog]
 |The event log plugin provides a persistent history of alerting and action
 activities.

From 535622eb55799a4c26c4ae26fb86fd5251cbc358 Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Tue, 19 Nov 2024 21:24:04 +0000
Subject: [PATCH 19/37] [CI] Auto-commit changed files from 'node
 scripts/notice'

---
 x-pack/plugins/entity_manager/tsconfig.json   | 15 ------------
 .../entity_manager_app/tsconfig.json          | 24 +------------------
 2 files changed, 1 insertion(+), 38 deletions(-)

diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json
index d8a8f6654b997..2ef8551f373fd 100644
--- a/x-pack/plugins/entity_manager/tsconfig.json
+++ b/x-pack/plugins/entity_manager/tsconfig.json
@@ -35,21 +35,6 @@
     "@kbn/encrypted-saved-objects-plugin",
     "@kbn/licensing-plugin",
     "@kbn/core-saved-objects-server",
-    "@kbn/ebt-tools",
-    "@kbn/kibana-react-plugin",
-    "@kbn/kibana-utils-plugin",
-    "@kbn/observability-shared-plugin",
-    "@kbn/react-kibana-context-render",
-    "@kbn/react-kibana-context-theme",
-    "@kbn/shared-ux-link-redirect-app",
-    "@kbn/usage-collection-plugin",
-    "@kbn/shared-ux-router",
-    "@kbn/presentation-util-plugin",
-    "@kbn/cloud-plugin",
-    "@kbn/serverless",
-    "@kbn/data-plugin",
-    "@kbn/observability-ai-assistant-plugin",
-    "@kbn/observability-utils-server",
     "@kbn/es-types",
   ]
 }
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
index 7f0162d31b585..5d43d93d7d1bd 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
+++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
@@ -13,27 +13,8 @@
     "target/**/*"
   ],
   "kbn_references": [
-    "@kbn/config-schema",
-    "@kbn/entities-schema",
     "@kbn/core",
-    "@kbn/core-plugins-server",
-    "@kbn/server-route-repository-client",
     "@kbn/logging",
-    "@kbn/core-http-server",
-    "@kbn/security-plugin",
-    "@kbn/es-query",
-    "@kbn/core-elasticsearch-server",
-    "@kbn/core-saved-objects-api-server",
-    "@kbn/core-elasticsearch-client-server-mocks",
-    "@kbn/core-saved-objects-api-server-mocks",
-    "@kbn/logging-mocks",
-    "@kbn/datemath",
-    "@kbn/server-route-repository",
-    "@kbn/zod",
-    "@kbn/zod-helpers",
-    "@kbn/encrypted-saved-objects-plugin",
-    "@kbn/licensing-plugin",
-    "@kbn/core-saved-objects-server",
     "@kbn/ebt-tools",
     "@kbn/kibana-react-plugin",
     "@kbn/kibana-utils-plugin",
@@ -46,9 +27,6 @@
     "@kbn/presentation-util-plugin",
     "@kbn/cloud-plugin",
     "@kbn/serverless",
-    "@kbn/data-plugin",
-    "@kbn/observability-ai-assistant-plugin",
-    "@kbn/observability-utils-server",
-    "@kbn/es-types",
+    "@kbn/entityManager-plugin",
   ]
 }

From 1a02de89d1a3234716f7eaa73ffc53d3a84a2e40 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 19 Nov 2024 22:26:24 +0100
Subject: [PATCH 20/37] add timestamp_field to form

---
 .../entity_manager/server/lib/entity_client.ts    |  5 ++++-
 .../public/pages/overview/index.tsx               | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index e39b61ac74928..67ab4a92f1b74 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -209,6 +209,9 @@ export class EntityClient {
         if (!sourceHasMandatoryFields) {
           // we can't build entities without id fields so we ignore the source.
           // filters should likely behave similarly.
+          this.options.logger.info(
+            `Ignoring source for type [${source.type}] with index_patterns [${source.index_patterns}] because some mandatory fields [${mandatoryFields}] are not mapped`
+          );
           return [];
         }
 
@@ -221,7 +224,7 @@ export class EntityClient {
           end,
           limit,
         });
-        this.options.logger.info(`Entity query: ${query}`);
+        this.options.logger.debug(`Entity query: ${query}`);
 
         const rawEntities = await runESQLQuery<Entity>({ query, esClient: this.options.esClient });
 
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
index 222d760f480dd..d6140317b3a0c 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
@@ -89,6 +89,17 @@ function EntitySourceForm({
           />
         </EuiFormRow>
       </EuiFlexItem>
+
+      <EuiFlexItem>
+        <EuiFormRow label="Timestamp field">
+          <EuiFieldText
+            data-test-subj="entityManagerFormTimestamp"
+            name="timestamp_field"
+            defaultValue={source.timestamp_field}
+            onChange={(e) => onFieldChange(index, 'timestamp_field', e.target.value)}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
     </EuiFlexGroup>
   );
 }
@@ -99,6 +110,7 @@ interface EntitySource {
   identity_fields?: string[];
   metadata_fields?: string[];
   filters?: string[];
+  timestamp_field?: string;
 }
 
 const newEntitySource = ({
@@ -106,16 +118,19 @@ const newEntitySource = ({
   identityFields = [],
   metadataFields = [],
   filters = [],
+  timestampField = '@timestamp',
 }: {
   indexPatterns?: string[];
   identityFields?: string[];
   metadataFields?: string[];
   filters?: string[];
+  timestampField?: string;
 }) => ({
   id: uuid(),
   index_patterns: indexPatterns,
   identity_fields: identityFields,
   metadata_fields: metadataFields,
+  timestamp_field: timestampField,
   filters,
 });
 

From 820f38136466d7ad5e1f0d3deb7d7c4129036195 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Tue, 19 Nov 2024 23:00:16 +0100
Subject: [PATCH 21/37] fix configs

---
 packages/kbn-optimizer/limits.yml                        | 1 +
 .../entity_manager_app/jest.config.js                    | 4 +++-
 .../entity_manager_app/public/application.tsx            | 9 ++-------
 .../entity_manager_app/public/pages/overview/index.tsx   | 2 +-
 .../entity_manager_app/tsconfig.json                     | 4 ++--
 5 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
index e1e9b6aa81898..607b3a1e993f8 100644
--- a/packages/kbn-optimizer/limits.yml
+++ b/packages/kbn-optimizer/limits.yml
@@ -42,6 +42,7 @@ pageLoadAssetSize:
   embeddableEnhanced: 22107
   enterpriseSearch: 66810
   entityManager: 17175
+  entityManagerApp: 20378
   esql: 37000
   esqlDataGrid: 24582
   esUiShared: 326654
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js b/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js
index eef557bb41a68..d8217a43063a2 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js
+++ b/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js
@@ -5,9 +5,11 @@
  * 2.0.
  */
 
+const path = require('path');
+
 module.exports = {
   preset: '@kbn/test',
-  rootDir: '../../..',
+  rootDir: path.resolve(__dirname, '../../../..'),
   roots: ['<rootDir>/x-pack/plugins/observability_solution/entity_manager_app'],
   coverageDirectory:
     '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/observability_solution/entity_manager_app',
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
index 5a5bb9c29b154..e6d2d94d21562 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
@@ -20,7 +20,7 @@ import ReactDOM from 'react-dom';
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import { Router } from '@kbn/shared-ux-router';
 import { PluginContext } from './context/plugin_context';
-import { EntityManagerAppPluginStart } from './types';
+import { EntityManagerPluginStart } from './types';
 import { EntityManagerOverviewPage } from './pages/overview';
 
 export function renderApp({
@@ -35,7 +35,7 @@ export function renderApp({
   entityClient,
 }: {
   core: CoreStart;
-  plugins: EntityManagerAppPluginStart;
+  plugins: EntityManagerPluginStart;
   appMountParameters: AppMountParameters;
   ObservabilityPageTemplate: React.ComponentType<LazyObservabilityPageTemplateProps>;
   usageCollection: UsageCollectionSetup;
@@ -110,11 +110,6 @@ export function renderApp({
   );
 
   return () => {
-    // This needs to be present to fix https://github.com/elastic/kibana/issues/155704
-    // as the Overview page renders the UX Section component. That component renders a Lens embeddable
-    // via the ExploratoryView app, which uses search sessions. Therefore on unmounting we need to clear
-    // these sessions.
-    plugins.data.search.session.clear();
     ReactDOM.unmountComponentAtNode(element);
   };
 }
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
index d6140317b3a0c..6e64f288d9c08 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
@@ -268,7 +268,7 @@ export function EntityManagerOverviewPage() {
               onFieldChange={(
                 index: number,
                 field: Exclude<keyof EntitySource, 'id'>,
-                value: string[]
+                value: any
               ) => {
                 entitySources[index][field] = value;
                 setEntitySources([...entitySources]);
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
index 5d43d93d7d1bd..a286a1ec7b2aa 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
+++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
@@ -1,10 +1,10 @@
 {
-  "extends": "../../../tsconfig.base.json",
+  "extends": "../../../../tsconfig.base.json",
   "compilerOptions": {
     "outDir": "target/types"
   },
   "include": [
-    "../../../typings/**/*",
+    "../../../../typings/**/*",
     "common/**/*",
     "public/**/*",
     "types/**/*"

From 832297e42d4c65d3e268f832cf66735d9a299fd2 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Wed, 20 Nov 2024 08:52:47 +0100
Subject: [PATCH 22/37] remove usagecollection

---
 .../entity_manager_app/kibana.jsonc           |  1 -
 .../entity_manager_app/public/application.tsx | 77 ++++++++-----------
 .../entity_manager_app/public/plugin.ts       |  1 -
 .../entity_manager_app/public/types.ts        |  2 -
 .../entity_manager_app/tsconfig.json          |  1 -
 5 files changed, 33 insertions(+), 49 deletions(-)

diff --git a/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
index 93e6687f9b4c3..006ce6f1c4c33 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
+++ b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
@@ -14,7 +14,6 @@
       "entityManager",
       "observabilityShared",
       "presentationUtil",
-      "usageCollection",
       "licensing"
     ],
     "optionalPlugins": [
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
index e6d2d94d21562..8779c461cb10f 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
@@ -17,7 +17,6 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import { EntityClient } from '@kbn/entityManager-plugin/public';
 import React from 'react';
 import ReactDOM from 'react-dom';
-import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import { Router } from '@kbn/shared-ux-router';
 import { PluginContext } from './context/plugin_context';
 import { EntityManagerPluginStart } from './types';
@@ -28,7 +27,6 @@ export function renderApp({
   plugins,
   appMountParameters,
   ObservabilityPageTemplate,
-  usageCollection,
   isDev,
   kibanaVersion,
   isServerless,
@@ -38,7 +36,6 @@ export function renderApp({
   plugins: EntityManagerPluginStart;
   appMountParameters: AppMountParameters;
   ObservabilityPageTemplate: React.ComponentType<LazyObservabilityPageTemplateProps>;
-  usageCollection: UsageCollectionSetup;
   isDev?: boolean;
   kibanaVersion: string;
   isServerless?: boolean;
@@ -52,9 +49,6 @@ export function renderApp({
 
   const queryClient = new QueryClient();
 
-  const ApplicationUsageTrackingProvider =
-    usageCollection?.components.ApplicationUsageTrackingProvider ?? React.Fragment;
-
   const CloudProvider = plugins.cloud?.CloudContextProvider ?? React.Fragment;
 
   const PresentationContextProvider = React.Fragment;
@@ -62,48 +56,43 @@ export function renderApp({
   ReactDOM.render(
     <KibanaRenderContextProvider {...core}>
       <PresentationContextProvider>
-        <ApplicationUsageTrackingProvider>
-          <KibanaThemeProvider {...{ theme: { theme$ } }}>
-            <CloudProvider>
-              <KibanaContextProvider
-                services={{
-                  ...core,
-                  ...plugins,
-                  storage: new Storage(localStorage),
-                  entityClient: new EntityClient(core),
+        <KibanaThemeProvider {...{ theme: { theme$ } }}>
+          <CloudProvider>
+            <KibanaContextProvider
+              services={{
+                ...core,
+                ...plugins,
+                storage: new Storage(localStorage),
+                entityClient: new EntityClient(core),
+                isDev,
+                kibanaVersion,
+                isServerless,
+              }}
+            >
+              <PluginContext.Provider
+                value={{
                   isDev,
-                  kibanaVersion,
                   isServerless,
+                  appMountParameters,
+                  ObservabilityPageTemplate,
+                  entityClient,
                 }}
               >
-                <PluginContext.Provider
-                  value={{
-                    isDev,
-                    isServerless,
-                    appMountParameters,
-                    ObservabilityPageTemplate,
-                    entityClient,
-                  }}
-                >
-                  <Router history={history}>
-                    <EuiThemeProvider darkMode={isDarkMode}>
-                      <RedirectAppLinks
-                        coreStart={core}
-                        data-test-subj="observabilityMainContainer"
-                      >
-                        <PerformanceContextProvider>
-                          <QueryClientProvider client={queryClient}>
-                            <EntityManagerOverviewPage />
-                          </QueryClientProvider>
-                        </PerformanceContextProvider>
-                      </RedirectAppLinks>
-                    </EuiThemeProvider>
-                  </Router>
-                </PluginContext.Provider>
-              </KibanaContextProvider>
-            </CloudProvider>
-          </KibanaThemeProvider>
-        </ApplicationUsageTrackingProvider>
+                <Router history={history}>
+                  <EuiThemeProvider darkMode={isDarkMode}>
+                    <RedirectAppLinks coreStart={core} data-test-subj="observabilityMainContainer">
+                      <PerformanceContextProvider>
+                        <QueryClientProvider client={queryClient}>
+                          <EntityManagerOverviewPage />
+                        </QueryClientProvider>
+                      </PerformanceContextProvider>
+                    </RedirectAppLinks>
+                  </EuiThemeProvider>
+                </Router>
+              </PluginContext.Provider>
+            </KibanaContextProvider>
+          </CloudProvider>
+        </KibanaThemeProvider>
       </PresentationContextProvider>
     </KibanaRenderContextProvider>,
     element
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
index 0db381522b020..b4d806d2e9c35 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
@@ -44,7 +44,6 @@ export class Plugin implements EntityManagerAppPluginClass {
         core: coreStart,
         isDev: this.context.env.mode.dev,
         kibanaVersion,
-        usageCollection: pluginSetup.usageCollection,
         ObservabilityPageTemplate: pluginsStart.observabilityShared.navigation.PageTemplate,
         plugins: pluginsStart,
         isServerless: !!pluginsStart.serverless,
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
index b735771d79f80..89baf5f7a9305 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
@@ -12,13 +12,11 @@ import {
   ObservabilitySharedPluginSetup,
   ObservabilitySharedPluginStart,
 } from '@kbn/observability-shared-plugin/public';
-import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import { EntityManagerPublicPluginSetup } from '@kbn/entityManager-plugin/public/types';
 
 export interface EntityManagerPluginSetup {
   observabilityShared: ObservabilitySharedPluginSetup;
   serverless?: ServerlessPluginSetup;
-  usageCollection: UsageCollectionSetup;
   entityManager: EntityManagerPublicPluginSetup;
 }
 
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
index a286a1ec7b2aa..7a0825ada4034 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
+++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
@@ -22,7 +22,6 @@
     "@kbn/react-kibana-context-render",
     "@kbn/react-kibana-context-theme",
     "@kbn/shared-ux-link-redirect-app",
-    "@kbn/usage-collection-plugin",
     "@kbn/shared-ux-router",
     "@kbn/presentation-util-plugin",
     "@kbn/cloud-plugin",

From 59ab75928dc5569008dae0cf155662f547f81d4a Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Wed, 20 Nov 2024 10:52:46 +0100
Subject: [PATCH 23/37] register entity_manager in application_usage

---
 .../collectors/application_usage/schema.ts    |  1 +
 .../entity_manager_app/kibana.jsonc           |  1 +
 .../entity_manager_app/public/application.tsx | 77 +++++++++++--------
 .../entity_manager_app/public/plugin.ts       |  1 +
 .../entity_manager_app/public/types.ts        |  2 +
 .../entity_manager_app/tsconfig.json          |  1 +
 6 files changed, 50 insertions(+), 33 deletions(-)

diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
index 88d60b1a86b2e..0cc56676137e5 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
@@ -143,6 +143,7 @@ export const applicationUsageSchema = {
   enterpriseSearchSemanticSearch: commonSchema,
   enterpriseSearchVectorSearch: commonSchema,
   enterpriseSearchElasticsearch: commonSchema,
+  entity_manager: commonSchema,
   appSearch: commonSchema,
   workplaceSearch: commonSchema,
   searchExperiences: commonSchema,
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
index 006ce6f1c4c33..93e6687f9b4c3 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
+++ b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc
@@ -14,6 +14,7 @@
       "entityManager",
       "observabilityShared",
       "presentationUtil",
+      "usageCollection",
       "licensing"
     ],
     "optionalPlugins": [
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
index 8779c461cb10f..e6d2d94d21562 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
@@ -17,6 +17,7 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import { EntityClient } from '@kbn/entityManager-plugin/public';
 import React from 'react';
 import ReactDOM from 'react-dom';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import { Router } from '@kbn/shared-ux-router';
 import { PluginContext } from './context/plugin_context';
 import { EntityManagerPluginStart } from './types';
@@ -27,6 +28,7 @@ export function renderApp({
   plugins,
   appMountParameters,
   ObservabilityPageTemplate,
+  usageCollection,
   isDev,
   kibanaVersion,
   isServerless,
@@ -36,6 +38,7 @@ export function renderApp({
   plugins: EntityManagerPluginStart;
   appMountParameters: AppMountParameters;
   ObservabilityPageTemplate: React.ComponentType<LazyObservabilityPageTemplateProps>;
+  usageCollection: UsageCollectionSetup;
   isDev?: boolean;
   kibanaVersion: string;
   isServerless?: boolean;
@@ -49,6 +52,9 @@ export function renderApp({
 
   const queryClient = new QueryClient();
 
+  const ApplicationUsageTrackingProvider =
+    usageCollection?.components.ApplicationUsageTrackingProvider ?? React.Fragment;
+
   const CloudProvider = plugins.cloud?.CloudContextProvider ?? React.Fragment;
 
   const PresentationContextProvider = React.Fragment;
@@ -56,43 +62,48 @@ export function renderApp({
   ReactDOM.render(
     <KibanaRenderContextProvider {...core}>
       <PresentationContextProvider>
-        <KibanaThemeProvider {...{ theme: { theme$ } }}>
-          <CloudProvider>
-            <KibanaContextProvider
-              services={{
-                ...core,
-                ...plugins,
-                storage: new Storage(localStorage),
-                entityClient: new EntityClient(core),
-                isDev,
-                kibanaVersion,
-                isServerless,
-              }}
-            >
-              <PluginContext.Provider
-                value={{
+        <ApplicationUsageTrackingProvider>
+          <KibanaThemeProvider {...{ theme: { theme$ } }}>
+            <CloudProvider>
+              <KibanaContextProvider
+                services={{
+                  ...core,
+                  ...plugins,
+                  storage: new Storage(localStorage),
+                  entityClient: new EntityClient(core),
                   isDev,
+                  kibanaVersion,
                   isServerless,
-                  appMountParameters,
-                  ObservabilityPageTemplate,
-                  entityClient,
                 }}
               >
-                <Router history={history}>
-                  <EuiThemeProvider darkMode={isDarkMode}>
-                    <RedirectAppLinks coreStart={core} data-test-subj="observabilityMainContainer">
-                      <PerformanceContextProvider>
-                        <QueryClientProvider client={queryClient}>
-                          <EntityManagerOverviewPage />
-                        </QueryClientProvider>
-                      </PerformanceContextProvider>
-                    </RedirectAppLinks>
-                  </EuiThemeProvider>
-                </Router>
-              </PluginContext.Provider>
-            </KibanaContextProvider>
-          </CloudProvider>
-        </KibanaThemeProvider>
+                <PluginContext.Provider
+                  value={{
+                    isDev,
+                    isServerless,
+                    appMountParameters,
+                    ObservabilityPageTemplate,
+                    entityClient,
+                  }}
+                >
+                  <Router history={history}>
+                    <EuiThemeProvider darkMode={isDarkMode}>
+                      <RedirectAppLinks
+                        coreStart={core}
+                        data-test-subj="observabilityMainContainer"
+                      >
+                        <PerformanceContextProvider>
+                          <QueryClientProvider client={queryClient}>
+                            <EntityManagerOverviewPage />
+                          </QueryClientProvider>
+                        </PerformanceContextProvider>
+                      </RedirectAppLinks>
+                    </EuiThemeProvider>
+                  </Router>
+                </PluginContext.Provider>
+              </KibanaContextProvider>
+            </CloudProvider>
+          </KibanaThemeProvider>
+        </ApplicationUsageTrackingProvider>
       </PresentationContextProvider>
     </KibanaRenderContextProvider>,
     element
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
index b4d806d2e9c35..0db381522b020 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts
@@ -44,6 +44,7 @@ export class Plugin implements EntityManagerAppPluginClass {
         core: coreStart,
         isDev: this.context.env.mode.dev,
         kibanaVersion,
+        usageCollection: pluginSetup.usageCollection,
         ObservabilityPageTemplate: pluginsStart.observabilityShared.navigation.PageTemplate,
         plugins: pluginsStart,
         isServerless: !!pluginsStart.serverless,
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
index 89baf5f7a9305..b735771d79f80 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts
@@ -12,11 +12,13 @@ import {
   ObservabilitySharedPluginSetup,
   ObservabilitySharedPluginStart,
 } from '@kbn/observability-shared-plugin/public';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
 import { EntityManagerPublicPluginSetup } from '@kbn/entityManager-plugin/public/types';
 
 export interface EntityManagerPluginSetup {
   observabilityShared: ObservabilitySharedPluginSetup;
   serverless?: ServerlessPluginSetup;
+  usageCollection: UsageCollectionSetup;
   entityManager: EntityManagerPublicPluginSetup;
 }
 
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
index 7a0825ada4034..a286a1ec7b2aa 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
+++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
@@ -22,6 +22,7 @@
     "@kbn/react-kibana-context-render",
     "@kbn/react-kibana-context-theme",
     "@kbn/shared-ux-link-redirect-app",
+    "@kbn/usage-collection-plugin",
     "@kbn/shared-ux-router",
     "@kbn/presentation-util-plugin",
     "@kbn/cloud-plugin",

From 6035b272391285d371728bae8dbb8874ee9ffb43 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Wed, 20 Nov 2024 11:17:08 +0100
Subject: [PATCH 24/37] update oss_plugins

---
 src/plugins/telemetry/schema/oss_plugins.json | 131 ++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json
index 78f8b4f2f7b38..a9f685c3040ec 100644
--- a/src/plugins/telemetry/schema/oss_plugins.json
+++ b/src/plugins/telemetry/schema/oss_plugins.json
@@ -3015,6 +3015,137 @@
             }
           }
         },
+        "entity_manager": {
+          "properties": {
+            "appId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "The application being tracked"
+              }
+            },
+            "viewId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "Always `main`"
+              }
+            },
+            "clicks_total": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application since we started counting them"
+              }
+            },
+            "clicks_7_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 7 days"
+              }
+            },
+            "clicks_30_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 30 days"
+              }
+            },
+            "clicks_90_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 90 days"
+              }
+            },
+            "minutes_on_screen_total": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen since we started counting them."
+              }
+            },
+            "minutes_on_screen_7_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 7 days"
+              }
+            },
+            "minutes_on_screen_30_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 30 days"
+              }
+            },
+            "minutes_on_screen_90_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 90 days"
+              }
+            },
+            "views": {
+              "type": "array",
+              "items": {
+                "properties": {
+                  "appId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application being tracked"
+                    }
+                  },
+                  "viewId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application view being tracked"
+                    }
+                  },
+                  "clicks_total": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the application sub view since we started counting them"
+                    }
+                  },
+                  "clicks_7_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 7 days"
+                    }
+                  },
+                  "clicks_30_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 30 days"
+                    }
+                  },
+                  "clicks_90_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 90 days"
+                    }
+                  },
+                  "minutes_on_screen_total": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application sub view is active and on-screen since we started counting them."
+                    }
+                  },
+                  "minutes_on_screen_7_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 7 days"
+                    }
+                  },
+                  "minutes_on_screen_30_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 30 days"
+                    }
+                  },
+                  "minutes_on_screen_90_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 90 days"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "appSearch": {
           "properties": {
             "appId": {

From 698debf0d3bf1ed5ec5af2a1521601bb54a22aab Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Wed, 20 Nov 2024 19:21:40 +0100
Subject: [PATCH 25/37] remove queryclient

---
 .../entity_manager_app/public/application.tsx | 78 ++++++++-----------
 1 file changed, 33 insertions(+), 45 deletions(-)

diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
index e6d2d94d21562..8f2e9e2213ba0 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx
@@ -13,7 +13,6 @@ import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shar
 import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
 import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme';
 import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app';
-import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import { EntityClient } from '@kbn/entityManager-plugin/public';
 import React from 'react';
 import ReactDOM from 'react-dom';
@@ -50,61 +49,50 @@ export function renderApp({
   // ensure all divs are .kbnAppWrappers
   element.classList.add(APP_WRAPPER_CLASS);
 
-  const queryClient = new QueryClient();
-
   const ApplicationUsageTrackingProvider =
     usageCollection?.components.ApplicationUsageTrackingProvider ?? React.Fragment;
 
   const CloudProvider = plugins.cloud?.CloudContextProvider ?? React.Fragment;
 
-  const PresentationContextProvider = React.Fragment;
-
   ReactDOM.render(
     <KibanaRenderContextProvider {...core}>
-      <PresentationContextProvider>
-        <ApplicationUsageTrackingProvider>
-          <KibanaThemeProvider {...{ theme: { theme$ } }}>
-            <CloudProvider>
-              <KibanaContextProvider
-                services={{
-                  ...core,
-                  ...plugins,
-                  storage: new Storage(localStorage),
-                  entityClient: new EntityClient(core),
+      <ApplicationUsageTrackingProvider>
+        <KibanaThemeProvider {...{ theme: { theme$ } }}>
+          <CloudProvider>
+            <KibanaContextProvider
+              services={{
+                ...core,
+                ...plugins,
+                storage: new Storage(localStorage),
+                entityClient: new EntityClient(core),
+                isDev,
+                kibanaVersion,
+                isServerless,
+              }}
+            >
+              <PluginContext.Provider
+                value={{
                   isDev,
-                  kibanaVersion,
                   isServerless,
+                  appMountParameters,
+                  ObservabilityPageTemplate,
+                  entityClient,
                 }}
               >
-                <PluginContext.Provider
-                  value={{
-                    isDev,
-                    isServerless,
-                    appMountParameters,
-                    ObservabilityPageTemplate,
-                    entityClient,
-                  }}
-                >
-                  <Router history={history}>
-                    <EuiThemeProvider darkMode={isDarkMode}>
-                      <RedirectAppLinks
-                        coreStart={core}
-                        data-test-subj="observabilityMainContainer"
-                      >
-                        <PerformanceContextProvider>
-                          <QueryClientProvider client={queryClient}>
-                            <EntityManagerOverviewPage />
-                          </QueryClientProvider>
-                        </PerformanceContextProvider>
-                      </RedirectAppLinks>
-                    </EuiThemeProvider>
-                  </Router>
-                </PluginContext.Provider>
-              </KibanaContextProvider>
-            </CloudProvider>
-          </KibanaThemeProvider>
-        </ApplicationUsageTrackingProvider>
-      </PresentationContextProvider>
+                <Router history={history}>
+                  <EuiThemeProvider darkMode={isDarkMode}>
+                    <RedirectAppLinks coreStart={core} data-test-subj="observabilityMainContainer">
+                      <PerformanceContextProvider>
+                        <EntityManagerOverviewPage />
+                      </PerformanceContextProvider>
+                    </RedirectAppLinks>
+                  </EuiThemeProvider>
+                </Router>
+              </PluginContext.Provider>
+            </KibanaContextProvider>
+          </CloudProvider>
+        </KibanaThemeProvider>
+      </ApplicationUsageTrackingProvider>
     </KibanaRenderContextProvider>,
     element
   );

From 506ce78a06a31cb65d4657d36d8580b206e9a3ab Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Wed, 20 Nov 2024 20:00:38 +0100
Subject: [PATCH 26/37] fix merge and tests

---
 .../server/lib/queries/utils.test.ts          | 135 ++++++++++++++++++
 .../server/lib/queries/utils.ts               |   7 +-
 2 files changed, 139 insertions(+), 3 deletions(-)
 create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts
new file mode 100644
index 0000000000000..9d487c68fd64e
--- /dev/null
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mergeEntitiesList } from './utils';
+
+describe(__filename, () => {
+  describe('mergeEntitiesList', () => {
+    it('merges entities on entity.id', () => {
+      const entities = [
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+          'entity.type': 'service',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+          'entity.type': 'service',
+        },
+      ];
+
+      const mergedEntities = mergeEntitiesList(entities);
+      expect(mergedEntities.length).toEqual(1);
+      expect(mergedEntities[0]).toEqual({
+        'entity.id': 'foo',
+        'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+        'entity.type': 'service',
+      });
+    });
+
+    it('merges metadata fields', () => {
+      const entities = [
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': 'host-1',
+          'metadata.agent.name': 'agent-1',
+          'metadata.service.environment': ['dev', 'staging'],
+          'metadata.only_in_record_1': 'foo',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': ['host-2', 'host-3'],
+          'metadata.agent.name': 'agent-2',
+          'metadata.service.environment': 'prod',
+          'metadata.only_in_record_2': 'bar',
+        },
+      ];
+
+      const mergedEntities = mergeEntitiesList(entities);
+      expect(mergedEntities.length).toEqual(1);
+      expect(mergedEntities[0]).toEqual({
+        'entity.id': 'foo',
+        'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+        'entity.type': 'service',
+        'metadata.host.name': ['host-1', 'host-2', 'host-3'],
+        'metadata.agent.name': ['agent-1', 'agent-2'],
+        'metadata.service.environment': ['dev', 'staging', 'prod'],
+        'metadata.only_in_record_1': 'foo',
+        'metadata.only_in_record_2': 'bar',
+      });
+    });
+
+    it('picks most recent timestamp when merging', () => {
+      const entities = [
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': 'host-1',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': 'host-2',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': 'host-3',
+        },
+      ];
+
+      const mergedEntities = mergeEntitiesList(entities);
+      expect(mergedEntities.length).toEqual(1);
+      expect(mergedEntities[0]).toEqual({
+        'entity.id': 'foo',
+        'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
+        'entity.type': 'service',
+        'metadata.host.name': ['host-1', 'host-2', 'host-3'],
+      });
+    });
+
+    it('deduplicates metadata values', () => {
+      const entities = [
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': 'host-1',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': 'host-2',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z',
+          'entity.type': 'service',
+          'metadata.host.name': ['host-1', 'host-2'],
+        },
+      ];
+
+      const mergedEntities = mergeEntitiesList(entities);
+      expect(mergedEntities.length).toEqual(1);
+      expect(mergedEntities[0]).toEqual({
+        'entity.id': 'foo',
+        'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
+        'entity.type': 'service',
+        'metadata.host.name': ['host-1', 'host-2'],
+      });
+    });
+  });
+});
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
index 38cf525d7c9a3..1956daccaa07c 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
@@ -25,9 +25,10 @@ function mergeEntities(entity1: Entity, entity2: Entity): Entity {
     _key.startsWith('metadata.')
   )) {
     if (merged[key]) {
-      merged[key] = uniq(
-        Array.isArray(merged[key]) ? [...merged[key], value] : [merged[key], value]
-      );
+      merged[key] = uniq([
+        ...(Array.isArray(merged[key]) ? merged[key] : [merged[key]]),
+        ...(Array.isArray(value) ? value : [value]),
+      ]);
     } else {
       merged[key] = value;
     }

From ad1c530c4f20f99e6d6b29cebf988a2ab6a72c93 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Wed, 20 Nov 2024 20:10:44 +0100
Subject: [PATCH 27/37] query test

---
 .../server/lib/queries/index.test.ts          | 38 +++++++++++++++++++
 .../server/lib/queries/index.ts               |  2 +-
 2 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/index.test.ts

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
new file mode 100644
index 0000000000000..5231ac3043402
--- /dev/null
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEntityInstancesQuery } from './';
+
+describe(__filename, () => {
+  describe('getEntityInstancesQuery', () => {
+    it('generates a valid esql query', () => {
+      const query = getEntityInstancesQuery({
+        source: {
+          type: 'service',
+          index_patterns: ['logs-*', 'metrics-*'],
+          identity_fields: ['service.name'],
+          metadata_fields: ['host.name'],
+          filters: [],
+          timestamp_field: 'custom_timestamp_field',
+        },
+        limit: 5,
+        start: '2024-11-20T19:00:00.000Z',
+        end: '2024-11-20T20:00:00.000Z',
+      });
+
+      expect(query).toEqual(
+        'FROM logs-*,metrics-*|' +
+          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z"|' +
+          'WHERE custom_timestamp_field <= "2024-11-20T20:00:00.000Z"|' +
+          'WHERE service.name IS NOT NULL|' +
+          'STATS entity.last_seen_timestamp=MAX(custom_timestamp_field),metadata.host.name=VALUES(host.name) BY service.name|' +
+          'SORT entity.last_seen_timestamp DESC|' +
+          'LIMIT 5'
+      );
+    });
+  });
+});
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
index f0d7ad913d2df..9fc7ae00c9aa6 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts
@@ -65,7 +65,7 @@ const statsCommand = ({ source }: { source: EntitySource }) => {
       .map((field) => `metadata.${field}=VALUES(${field})`),
   ];
 
-  return `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(',')}`;
+  return `STATS ${aggs.join(',')} BY ${source.identity_fields.join(',')}`;
 };
 
 export function getEntityInstancesQuery({

From 48add7453a268cfd7843d209a14b371e04bef1ea Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Wed, 20 Nov 2024 19:32:08 +0000
Subject: [PATCH 28/37] [CI] Auto-commit changed files from 'node
 scripts/eslint --no-cache --fix'

---
 x-pack/plugins/entity_manager/server/lib/queries/index.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
index 5231ac3043402..ff5aab6678a3c 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { getEntityInstancesQuery } from './';
+import { getEntityInstancesQuery } from '.';
 
 describe(__filename, () => {
   describe('getEntityInstancesQuery', () => {

From 79a1c2007ebec26918b127a2b54a588a5a262618 Mon Sep 17 00:00:00 2001
From: Kevin Lacabane <klacabane@gmail.com>
Date: Thu, 21 Nov 2024 14:56:26 +0100
Subject: [PATCH 29/37] Update
 x-pack/plugins/entity_manager/server/lib/queries/index.test.ts

Co-authored-by: Milton Hultgren <miltonhultgren@gmail.com>
---
 x-pack/plugins/entity_manager/server/lib/queries/index.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
index ff5aab6678a3c..539d20c464794 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts
@@ -7,7 +7,7 @@
 
 import { getEntityInstancesQuery } from '.';
 
-describe(__filename, () => {
+describe('getEntityInstancesQuery', () => {
   describe('getEntityInstancesQuery', () => {
     it('generates a valid esql query', () => {
       const query = getEntityInstancesQuery({

From 9a14b7798426bf4c7322db00eaac225c9daa1ad2 Mon Sep 17 00:00:00 2001
From: Kevin Lacabane <klacabane@gmail.com>
Date: Thu, 21 Nov 2024 14:56:48 +0100
Subject: [PATCH 30/37] Update
 x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts

Co-authored-by: Milton Hultgren <miltonhultgren@gmail.com>
---
 x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts
index 9d487c68fd64e..5d57025671726 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts
@@ -7,7 +7,7 @@
 
 import { mergeEntitiesList } from './utils';
 
-describe(__filename, () => {
+describe('mergeEntitiesList', () => {
   describe('mergeEntitiesList', () => {
     it('merges entities on entity.id', () => {
       const entities = [

From 26bf3d8032ec6883c3ebe9be747232c7011b7ae6 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Thu, 21 Nov 2024 15:48:46 +0100
Subject: [PATCH 31/37] add type level metadata/filters

---
 .../entity_manager/server/lib/entity_client.ts    | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 67ab4a92f1b74..cca7b5ad1aa8e 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -190,19 +190,24 @@ export class EntityClient {
     sources,
     start,
     end,
+    metadataFields = [],
+    filters = [],
     limit = 10,
   }: {
     sources: EntitySource[];
     start: string;
     end: string;
+    metadataFields?: string[];
+    filters?: string[];
     limit?: number;
   }) {
     const entities = await Promise.all(
       sources.map(async (source) => {
         const mandatoryFields = [source.timestamp_field, ...source.identity_fields];
+        const metaFields = [...metadataFields, ...source.metadata_fields];
         const { fields } = await this.options.esClient.fieldCaps({
           index: source.index_patterns,
-          fields: [...mandatoryFields, ...source.metadata_fields],
+          fields: [...mandatoryFields, ...metaFields],
         });
 
         const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
@@ -216,10 +221,14 @@ export class EntityClient {
         }
 
         // but metadata field not being available is fine
-        const availableMetadataFields = source.metadata_fields.filter((field) => fields[field]);
+        const availableMetadataFields = metaFields.filter((field) => fields[field]);
 
         const query = getEntityInstancesQuery({
-          source: { ...source, metadata_fields: availableMetadataFields },
+          source: {
+            ...source,
+            metadata_fields: availableMetadataFields,
+            filters: [...source.filters, ...filters],
+          },
           start,
           end,
           limit,

From 5898952f53d0d82f5bb73d71229031905c8e2f92 Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Thu, 21 Nov 2024 15:49:02 +0100
Subject: [PATCH 32/37] v2 routes

---
 .../server/routes/{entities => v2}/search.ts           | 10 ++++++----
 .../entity_manager_app/public/pages/overview/index.tsx |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)
 rename x-pack/plugins/entity_manager/server/routes/{entities => v2}/search.ts (86%)

diff --git a/x-pack/plugins/entity_manager/server/routes/entities/search.ts b/x-pack/plugins/entity_manager/server/routes/v2/search.ts
similarity index 86%
rename from x-pack/plugins/entity_manager/server/routes/entities/search.ts
rename to x-pack/plugins/entity_manager/server/routes/v2/search.ts
index 8edfef17f5973..3156775b5f9ba 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/v2/search.ts
@@ -11,10 +11,12 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
 import { entitySourceSchema } from '../../lib/queries';
 
 export const searchEntitiesRoute = createEntityManagerServerRoute({
-  endpoint: 'POST /internal/entities/_search',
+  endpoint: 'POST /internal/entities/v2/_search',
   params: z.object({
     body: z.object({
       type: z.string(),
+      metadata_fields: z.optional(z.array(z.string())).default([]),
+      filters: z.optional(z.array(z.string())).default([]),
       start: z
         .optional(z.string())
         .default(() => moment().subtract(5, 'minutes').toISOString())
@@ -32,7 +34,7 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
     try {
-      const { type, start, end, limit } = params.body;
+      const { type, start, end, limit, filters, metadata_fields: metadataFields } = params.body;
 
       const client = await getScopedClient({ request });
 
@@ -41,7 +43,7 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
         return response.notFound({ body: { message: `No sources found for type [${type}]` } });
       }
 
-      const entities = await client.searchEntities({ sources, start, end, limit });
+      const entities = await client.searchEntities({ sources, filters, metadataFields, start, end, limit });
 
       return response.ok({ body: { entities } });
     } catch (e) {
@@ -52,7 +54,7 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
 });
 
 export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
-  endpoint: 'POST /internal/entities/_search/preview',
+  endpoint: 'POST /internal/entities/v2/_search/preview',
   params: z.object({
     body: z.object({
       sources: z.array(entitySourceSchema),
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
index 6e64f288d9c08..14e69b638d28a 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
@@ -164,7 +164,7 @@ export function EntityManagerOverviewPage() {
 
     try {
       const { entities } = await entityClient.repositoryClient(
-        'POST /internal/entities/_search/preview',
+        'POST /internal/entities/v2/_search/preview',
         {
           params: {
             body: {

From baab5b3d46d8749a421382af6d042a544b24e20e Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Thu, 21 Nov 2024 15:59:35 +0100
Subject: [PATCH 33/37] add searchEntitiesBySources

---
 .../errors/unknown_entity_type.ts}            |  9 +++---
 .../server/lib/entity_client.ts               | 31 +++++++++++++++++++
 .../entity_manager/server/routes/v2/search.ts | 23 +++++++++-----
 3 files changed, 50 insertions(+), 13 deletions(-)
 rename x-pack/plugins/entity_manager/server/lib/{errors.ts => entities/errors/unknown_entity_type.ts} (51%)

diff --git a/x-pack/plugins/entity_manager/server/lib/errors.ts b/x-pack/plugins/entity_manager/server/lib/entities/errors/unknown_entity_type.ts
similarity index 51%
rename from x-pack/plugins/entity_manager/server/lib/errors.ts
rename to x-pack/plugins/entity_manager/server/lib/entities/errors/unknown_entity_type.ts
index e0d341f87a9fd..5d29e24a1cca2 100644
--- a/x-pack/plugins/entity_manager/server/lib/errors.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entities/errors/unknown_entity_type.ts
@@ -5,10 +5,9 @@
  * 2.0.
  */
 
-export class AssetNotFoundError extends Error {
-  constructor(ean: string) {
-    super(`Asset with ean (${ean}) not found in the provided time range`);
-    Object.setPrototypeOf(this, new.target.prototype);
-    this.name = 'AssetNotFoundError';
+export class UnknownEntityType extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'UnknownEntityType';
   }
 }
diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index cca7b5ad1aa8e..0d0b141e6dc08 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -25,6 +25,7 @@ import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
 import { EntitySource, getEntityInstancesQuery } from './queries';
 import { mergeEntitiesList, runESQLQuery } from './queries/utils';
+import { UnknownEntityType } from './entities/errors/unknown_entity_type';
 
 export class EntityClient {
   constructor(
@@ -187,6 +188,36 @@ export class EntityClient {
   }
 
   async searchEntities({
+    type,
+    start,
+    end,
+    metadataFields = [],
+    filters = [],
+    limit = 10,
+  }: {
+    type: string;
+    start: string;
+    end: string;
+    metadataFields?: string[];
+    filters?: string[];
+    limit?: number;
+  }) {
+    const sources = await this.getEntitySources({ type });
+    if (sources.length === 0) {
+      throw new UnknownEntityType(`No sources found for entity type [${type}]`);
+    }
+
+    return this.searchEntitiesBySources({
+      sources,
+      start,
+      end,
+      metadataFields,
+      filters,
+      limit,
+    });
+  }
+
+  async searchEntitiesBySources({
     sources,
     start,
     end,
diff --git a/x-pack/plugins/entity_manager/server/routes/v2/search.ts b/x-pack/plugins/entity_manager/server/routes/v2/search.ts
index 3156775b5f9ba..7ed4b45d30afd 100644
--- a/x-pack/plugins/entity_manager/server/routes/v2/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/v2/search.ts
@@ -9,6 +9,7 @@ import moment from 'moment';
 import { z } from '@kbn/zod';
 import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
 import { entitySourceSchema } from '../../lib/queries';
+import { UnknownEntityType } from '../../lib/entities/errors/unknown_entity_type';
 
 export const searchEntitiesRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/v2/_search',
@@ -37,17 +38,23 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
       const { type, start, end, limit, filters, metadata_fields: metadataFields } = params.body;
 
       const client = await getScopedClient({ request });
-
-      const sources = await client.getEntitySources({ type });
-      if (sources.length === 0) {
-        return response.notFound({ body: { message: `No sources found for type [${type}]` } });
-      }
-
-      const entities = await client.searchEntities({ sources, filters, metadataFields, start, end, limit });
+      const entities = await client.searchEntities({
+        type,
+        filters,
+        metadataFields,
+        start,
+        end,
+        limit,
+      });
 
       return response.ok({ body: { entities } });
     } catch (e) {
       logger.error(e);
+
+      if (e instanceof UnknownEntityType) {
+        return response.notFound({ body: e });
+      }
+
       return response.customError({ body: e, statusCode: 500 });
     }
   },
@@ -78,7 +85,7 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
       const { sources, start, end, limit } = params.body;
 
       const client = await getScopedClient({ request });
-      const entities = await client.searchEntities({
+      const entities = await client.searchEntitiesBySources({
         sources,
         start,
         end,

From 51af524bccd1abeb06a4bcd513a2d62545f1974d Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Thu, 21 Nov 2024 16:07:18 +0100
Subject: [PATCH 34/37] update search module path

---
 x-pack/plugins/entity_manager/server/routes/entities/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/entity_manager/server/routes/entities/index.ts b/x-pack/plugins/entity_manager/server/routes/entities/index.ts
index 5ed21b5112f6f..52300ab2601b6 100644
--- a/x-pack/plugins/entity_manager/server/routes/entities/index.ts
+++ b/x-pack/plugins/entity_manager/server/routes/entities/index.ts
@@ -10,7 +10,7 @@ import { deleteEntityDefinitionRoute } from './delete';
 import { getEntityDefinitionRoute } from './get';
 import { resetEntityDefinitionRoute } from './reset';
 import { updateEntityDefinitionRoute } from './update';
-import { searchEntitiesRoute, searchEntitiesPreviewRoute } from './search';
+import { searchEntitiesRoute, searchEntitiesPreviewRoute } from '../v2/search';
 
 export const entitiesRoutes = {
   ...createEntityDefinitionRoute,

From a1ae809ce4c012d80237767388ac4f452ec23bcc Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Thu, 21 Nov 2024 16:10:05 +0100
Subject: [PATCH 35/37] types

---
 .../entity_manager/server/routes/v2/search.ts | 23 ++++++++-----------
 .../public/pages/overview/index.tsx           |  3 ++-
 2 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugins/entity_manager/server/routes/v2/search.ts b/x-pack/plugins/entity_manager/server/routes/v2/search.ts
index 7ed4b45d30afd..0b975da748a86 100644
--- a/x-pack/plugins/entity_manager/server/routes/v2/search.ts
+++ b/x-pack/plugins/entity_manager/server/routes/v2/search.ts
@@ -81,21 +81,16 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
     }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
-    try {
-      const { sources, start, end, limit } = params.body;
+    const { sources, start, end, limit } = params.body;
 
-      const client = await getScopedClient({ request });
-      const entities = await client.searchEntitiesBySources({
-        sources,
-        start,
-        end,
-        limit,
-      });
+    const client = await getScopedClient({ request });
+    const entities = await client.searchEntitiesBySources({
+      sources,
+      start,
+      end,
+      limit,
+    });
 
-      return response.ok({ body: { entities } });
-    } catch (e) {
-      logger.error(e);
-      return response.customError({ body: e, statusCode: 500 });
-    }
+    return response.ok({ body: { entities } });
   },
 });
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
index 14e69b638d28a..d04f581a8d98b 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
@@ -21,6 +21,7 @@ import {
   EuiSpacer,
   EuiTitle,
 } from '@elastic/eui';
+import { Entity } from '@kbn/entities-schema';
 import { usePluginContext } from '../../hooks/use_plugin_context';
 
 function EntitySourceForm({
@@ -136,7 +137,7 @@ const newEntitySource = ({
 
 export function EntityManagerOverviewPage() {
   const { ObservabilityPageTemplate, entityClient } = usePluginContext();
-  const [previewEntities, setPreviewEntities] = useState([]);
+  const [previewEntities, setPreviewEntities] = useState<Entity[]>([]);
   const [isSearchingEntities, setIsSearchingEntities] = useState(false);
   const [previewError, setPreviewError] = useState(null);
   const [formErrors, setFormErrors] = useState<string[]>([]);

From 67902d9c5d3a382bc6d8237b99233d3443e390fc Mon Sep 17 00:00:00 2001
From: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:22:27 +0000
Subject: [PATCH 36/37] [CI] Auto-commit changed files from 'node
 scripts/lint_ts_projects --fix'

---
 .../observability_solution/entity_manager_app/tsconfig.json      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
index a286a1ec7b2aa..64c0a293a4e2e 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
+++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json
@@ -28,5 +28,6 @@
     "@kbn/cloud-plugin",
     "@kbn/serverless",
     "@kbn/entityManager-plugin",
+    "@kbn/entities-schema",
   ]
 }

From f904abb3d3e6af725dd73b34e4b75bf63f77711f Mon Sep 17 00:00:00 2001
From: klacabane <kevin.lacabane@elastic.co>
Date: Fri, 22 Nov 2024 12:06:52 +0100
Subject: [PATCH 37/37] rename entityv2

---
 .../packages/kbn-entities-schema/src/schema/entity.ts  |  2 +-
 .../plugins/entity_manager/server/lib/entity_client.ts |  7 +++++--
 .../plugins/entity_manager/server/lib/queries/utils.ts | 10 +++++-----
 .../entity_manager_app/public/pages/overview/index.tsx |  4 ++--
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
index 738453a7ddc7b..5df10e11bb7ed 100644
--- a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
+++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
@@ -23,7 +23,7 @@ export interface MetadataRecord {
   [key: string]: string[] | MetadataRecord | string;
 }
 
-export interface Entity {
+export interface EntityV2 {
   'entity.id': string;
   'entity.last_seen_timestamp': string;
   'entity.type': string;
diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
index 0d0b141e6dc08..7045bee1fc538 100644
--- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts
+++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { Entity, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
+import { EntityV2, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
@@ -266,7 +266,10 @@ export class EntityClient {
         });
         this.options.logger.debug(`Entity query: ${query}`);
 
-        const rawEntities = await runESQLQuery<Entity>({ query, esClient: this.options.esClient });
+        const rawEntities = await runESQLQuery<EntityV2>({
+          query,
+          esClient: this.options.esClient,
+        });
 
         return rawEntities.map((entity) => {
           entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
index 1956daccaa07c..68f5b0f11aff2 100644
--- a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
+++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts
@@ -6,12 +6,12 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
-import { Entity } from '@kbn/entities-schema';
+import { EntityV2 } from '@kbn/entities-schema';
 import { ESQLSearchResponse } from '@kbn/es-types';
 import { uniq } from 'lodash';
 
-function mergeEntities(entity1: Entity, entity2: Entity): Entity {
-  const merged: Entity = {
+function mergeEntities(entity1: EntityV2, entity2: EntityV2): EntityV2 {
+  const merged: EntityV2 = {
     ...entity1,
     'entity.last_seen_timestamp': new Date(
       Math.max(
@@ -36,8 +36,8 @@ function mergeEntities(entity1: Entity, entity2: Entity): Entity {
   return merged;
 }
 
-export function mergeEntitiesList(entities: Entity[]): Entity[] {
-  const instances: { [key: string]: Entity } = {};
+export function mergeEntitiesList(entities: EntityV2[]): EntityV2[] {
+  const instances: { [key: string]: EntityV2 } = {};
 
   for (let i = 0; i < entities.length; i++) {
     const entity = entities[i];
diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
index d04f581a8d98b..8c978db6f6751 100644
--- a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
+++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx
@@ -21,7 +21,7 @@ import {
   EuiSpacer,
   EuiTitle,
 } from '@elastic/eui';
-import { Entity } from '@kbn/entities-schema';
+import { EntityV2 } from '@kbn/entities-schema';
 import { usePluginContext } from '../../hooks/use_plugin_context';
 
 function EntitySourceForm({
@@ -137,7 +137,7 @@ const newEntitySource = ({
 
 export function EntityManagerOverviewPage() {
   const { ObservabilityPageTemplate, entityClient } = usePluginContext();
-  const [previewEntities, setPreviewEntities] = useState<Entity[]>([]);
+  const [previewEntities, setPreviewEntities] = useState<EntityV2[]>([]);
   const [isSearchingEntities, setIsSearchingEntities] = useState(false);
   const [previewError, setPreviewError] = useState(null);
   const [formErrors, setFormErrors] = useState<string[]>([]);