[Ingest] Make all saved objects namespace agnostic, move to NP plugin (…

…#64360) (#64465)

* Move saved object type registration & mapping to NP plugin; normalize and prefix saved object type names

* Replace hardcoded saved object type names with constants

* Adjust output type name

* Review and import fixes

* Update tests, fixtures, and instructions

Co-authored-by: Elastic Machine <[email protected]>

x-pack/legacy/plugins/ingest_manager/index.ts

@@ -4,43 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { resolve } from 'path';
-import {
-  savedObjectMappings,
-  OUTPUT_SAVED_OBJECT_TYPE,
-  AGENT_CONFIG_SAVED_OBJECT_TYPE,
-  DATASOURCE_SAVED_OBJECT_TYPE,
-  PACKAGES_SAVED_OBJECT_TYPE,
-} from '../../../plugins/ingest_manager/server';
-
-// TODO https://github.com/elastic/kibana/issues/46373
-// const INDEX_NAMES = {
-//   INGEST: '.kibana',
-// };
 
 export function ingestManager(kibana: any) {
   return new kibana.Plugin({
     id: 'ingestManager',
     publicDir: resolve(__dirname, '../../../plugins/ingest_manager/public'),
-    uiExports: {
-      savedObjectSchemas: {
-        [AGENT_CONFIG_SAVED_OBJECT_TYPE]: {
-          isNamespaceAgnostic: true,
-          // indexPattern: INDEX_NAMES.INGEST,
-        },
-        [OUTPUT_SAVED_OBJECT_TYPE]: {
-          isNamespaceAgnostic: true,
-          // indexPattern: INDEX_NAMES.INGEST,
-        },
-        [DATASOURCE_SAVED_OBJECT_TYPE]: {
-          isNamespaceAgnostic: true,
-          // indexPattern: INDEX_NAMES.INGEST,
-        },
-        [PACKAGES_SAVED_OBJECT_TYPE]: {
-          isNamespaceAgnostic: true,
-          // indexPattern: INDEX_NAMES.INGEST,
-        },
-      },
-      mappings: savedObjectMappings,
-    },
   });
 }

x-pack/plugins/ingest_manager/README.md

@@ -1,60 +1,70 @@
 # Ingest Manager
+
 ## Plugin
-  - The plugin is disabled by default. See the TypeScript type for the [the available plugin configuration options](https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/common/types/index.ts#L9-L27)
-  - Setting `xpack.ingestManager.enabled=true` enables the plugin including the EPM and Fleet features. It also adds the `DATASOURCE_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts)
-  - Adding `--xpack.ingestManager.epm.enabled=false` will disable the EPM API & UI
-  - Adding `--xpack.ingestManager.fleet.enabled=false` will disable the Fleet API & UI
-    - [code for adding the routes](https://github.com/elastic/kibana/blob/1f27d349533b1c2865c10c45b2cf705d7416fb36/x-pack/plugins/ingest_manager/server/plugin.ts#L115-L133)
-    - [Integration tests](server/integration_tests/router.test.ts)
-  - Both EPM and Fleet require `ingestManager` be enabled. They are not standalone features.
+
+- The plugin is disabled by default. See the TypeScript type for the [the available plugin configuration options](https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/common/types/index.ts#L9-L27)
+- Setting `xpack.ingestManager.enabled=true` enables the plugin including the EPM and Fleet features. It also adds the `DATASOURCE_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts)
+- Adding `--xpack.ingestManager.epm.enabled=false` will disable the EPM API & UI
+- Adding `--xpack.ingestManager.fleet.enabled=false` will disable the Fleet API & UI
+  - [code for adding the routes](https://github.com/elastic/kibana/blob/1f27d349533b1c2865c10c45b2cf705d7416fb36/x-pack/plugins/ingest_manager/server/plugin.ts#L115-L133)
+  - [Integration tests](server/integration_tests/router.test.ts)
+- Both EPM and Fleet require `ingestManager` be enabled. They are not standalone features.
 
 ## Development
 
 ### Getting started
-See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-kibana)
 
+See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-kibana)
 
 One common development workflow is:
- - Bootstrap Kibana
-     ```
-    yarn kbn bootstrap
-    ```
- - Start Elasticsearch in one shell
-    ```
-    yarn es snapshot -E xpack.security.authc.api_key.enabled=true
-    ```
- - Start Kibana in another shell
-    ```
-    yarn start --xpack.ingestManager.enabled=true --no-base-path --xpack.endpoint.enabled=true
-    ```
+
+- Bootstrap Kibana
+  ```
+  yarn kbn bootstrap
+  ```
+- Start Elasticsearch in one shell
+  ```
+  yarn es snapshot -E xpack.security.authc.api_key.enabled=true
+  ```
+- Start Kibana in another shell
+  ```
+  yarn start --xpack.ingestManager.enabled=true --no-base-path --xpack.endpoint.enabled=true
+  ```
 
 This plugin follows the `common`, `server`, `public` structure from the [Architecture Style Guide
 ](https://github.com/elastic/kibana/blob/master/style_guides/architecture_style_guide.md#file-and-folder-structure). We also follow the pattern of developing feature branches under your personal fork of Kibana.
 
 ### API Tests
+
 #### Ingest & Fleet
-  1. In one terminal, change to the `x-pack` directory and start the test server with
-      ```
-      node scripts/functional_tests_server.js --config test/api_integration/config.ts
-      ```
-
-  1. in a second terminal, run the tests from the Kibana root directory with
-      ```
-      node scripts/functional_test_runner.js --config x-pack/test/api_integration/config.ts
-      ```
+
+1. In one terminal, change to the `x-pack` directory and start the test server with
+
+   ```
+   node scripts/functional_tests_server.js --config test/api_integration/config.js
+   ```
+
+1. in a second terminal, run the tests from the Kibana root directory with
+   ```
+   node scripts/functional_test_runner.js --config x-pack/test/api_integration/config.js
+   ```
+
 #### EPM
-  1. In one terminal, change to the `x-pack` directory and start the test server with
-      ```
-      node scripts/functional_tests_server.js --config test/epm_api_integration/config.ts
-      ```
 
-  1. in a second terminal, run the tests from the Kibana root directory with
-      ```
-      node scripts/functional_test_runner.js --config x-pack/test/epm_api_integration/config.ts
-      ```
+1. In one terminal, change to the `x-pack` directory and start the test server with
+
+   ```
+   node scripts/functional_tests_server.js --config test/epm_api_integration/config.ts
+   ```
 
- ### Staying up-to-date with `master`
- While we're developing in the `feature-ingest` feature branch, here's is more information on keeping up to date with upstream kibana.
+1. in a second terminal, run the tests from the Kibana root directory with
+   ```
+   node scripts/functional_test_runner.js --config x-pack/test/epm_api_integration/config.ts
+   ```
+
+### Staying up-to-date with `master`
+
+While we're developing in the `feature-ingest` feature branch, here's is more information on keeping up to date with upstream kibana.
 
 <details>
   <summary>merge upstream <code>master</code> into <code>feature-ingest</code></summary>
@@ -80,6 +90,7 @@ git push origin
 ## push your changes to upstream feature branch from the terminal; not GitHub UI
 git push upstream
 ```
+
 </details>
 
 See https://github.com/elastic/kibana/pull/37950 for an example.

x-pack/plugins/ingest_manager/common/constants/agent.ts

@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const AGENT_SAVED_OBJECT_TYPE = 'agents';
-export const AGENT_EVENT_SAVED_OBJECT_TYPE = 'agent_events';
-export const AGENT_ACTION_SAVED_OBJECT_TYPE = 'agent_actions';
+export const AGENT_SAVED_OBJECT_TYPE = 'fleet-agents';
+export const AGENT_EVENT_SAVED_OBJECT_TYPE = 'fleet-agent-events';
+export const AGENT_ACTION_SAVED_OBJECT_TYPE = 'fleet-agent-actions';
 
 export const AGENT_TYPE_PERMANENT = 'PERMANENT';
 export const AGENT_TYPE_EPHEMERAL = 'EPHEMERAL';

x-pack/plugins/ingest_manager/common/constants/agent_config.ts

@@ -5,7 +5,7 @@
  */
 import { AgentConfigStatus, DefaultPackages } from '../types';
 
-export const AGENT_CONFIG_SAVED_OBJECT_TYPE = 'agent_configs';
+export const AGENT_CONFIG_SAVED_OBJECT_TYPE = 'ingest-agent-configs';
 
 export const DEFAULT_AGENT_CONFIG = {
   name: 'Default config',

x-pack/plugins/ingest_manager/common/constants/datasource.ts

@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const DATASOURCE_SAVED_OBJECT_TYPE = 'datasources';
+export const DATASOURCE_SAVED_OBJECT_TYPE = 'ingest-datasources';

x-pack/plugins/ingest_manager/common/constants/enrollment_api_key.ts

@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE = 'enrollment_api_keys';
+export const ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE = 'fleet-enrollment-api-keys';

x-pack/plugins/ingest_manager/common/constants/epm.ts

@@ -4,5 +4,5 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-package';
+export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';

x-pack/plugins/ingest_manager/common/constants/output.ts

@@ -5,7 +5,7 @@
  */
 import { OutputType } from '../types';
 
-export const OUTPUT_SAVED_OBJECT_TYPE = 'outputs';
+export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs';
 
 export const DEFAULT_OUTPUT = {
   name: 'default',

x-pack/plugins/ingest_manager/common/services/agent_status.ts

@@ -8,20 +8,22 @@ import {
   AGENT_TYPE_TEMPORARY,
   AGENT_POLLING_THRESHOLD_MS,
   AGENT_TYPE_PERMANENT,
+  AGENT_SAVED_OBJECT_TYPE,
 } from '../constants';
 
 export function buildKueryForOnlineAgents() {
-  return `agents.last_checkin >= now-${(3 * AGENT_POLLING_THRESHOLD_MS) / 1000}s`;
+  return `${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${(3 * AGENT_POLLING_THRESHOLD_MS) /
+    1000}s`;
 }
 
 export function buildKueryForOfflineAgents() {
-  return `agents.type:${AGENT_TYPE_TEMPORARY} AND agents.last_checkin < now-${(3 *
+  return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${(3 *
     AGENT_POLLING_THRESHOLD_MS) /
     1000}s`;
 }
 
 export function buildKueryForErrorAgents() {
-  return `agents.type:${AGENT_TYPE_PERMANENT} AND agents.last_checkin < now-${(4 *
+  return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${(4 *
     AGENT_POLLING_THRESHOLD_MS) /
     1000}s`;
 }

x-pack/plugins/ingest_manager/dev_docs/api/agents_list.md

@@ -18,5 +18,5 @@
 ## Example
 
 ```js
-GET /api/ingest_manager/fleet/agents?kuery=agents.last_checkin:2019-10-01T13:42:54.323Z
+GET /api/ingest_manager/fleet/agents?kuery=fleet-agents.last_checkin:2019-10-01T13:42:54.323Z
 ```

x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx

@@ -8,12 +8,11 @@ import React, { useState, useEffect } from 'react';
 import { IFieldType } from 'src/plugins/data/public';
 // @ts-ignore
 import { EuiSuggest, EuiSuggestItemProps } from '@elastic/eui';
-import { useDebounce } from '../hooks';
-import { useStartDeps } from '../hooks/use_deps';
-import { INDEX_NAME } from '../constants';
+import { useDebounce, useStartDeps } from '../hooks';
+import { INDEX_NAME, AGENT_SAVED_OBJECT_TYPE } from '../constants';
 
 const DEBOUNCE_SEARCH_MS = 150;
-const HIDDEN_FIELDS = ['agents.actions'];
+const HIDDEN_FIELDS = [`${AGENT_SAVED_OBJECT_TYPE}.actions`];
 
 interface Suggestion {
   label: string;

x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts

@@ -3,7 +3,16 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-export { PLUGIN_ID, EPM_API_ROUTES, AGENT_CONFIG_SAVED_OBJECT_TYPE } from '../../../../common';
+export {
+  PLUGIN_ID,
+  EPM_API_ROUTES,
+  AGENT_API_ROUTES,
+  AGENT_CONFIG_SAVED_OBJECT_TYPE,
+  AGENT_EVENT_SAVED_OBJECT_TYPE,
+  AGENT_SAVED_OBJECT_TYPE,
+  ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
+  DATASOURCE_SAVED_OBJECT_TYPE,
+} from '../../../../common';
 
 export const BASE_PATH = '/app/ingestManager';
 export const EPM_PATH = '/epm';

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_delete_provider.tsx

@@ -8,6 +8,7 @@ import React, { Fragment, useRef, useState } from 'react';
 import { EuiConfirmModal, EuiOverlayMask } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
+import { AGENT_SAVED_OBJECT_TYPE } from '../../../constants';
 import { sendDeleteAgentConfigs, useCore, sendRequest } from '../../../hooks';
 
 interface Props {
@@ -122,7 +123,7 @@ export const AgentConfigDeleteProvider: React.FunctionComponent<Props> = ({ chil
       path: `/api/ingest_manager/fleet/agents`,
       method: 'get',
       query: {
-        kuery: `agents.config_id : (${agentConfigsToCheck.join(' or ')})`,
+        kuery: `${AGENT_SAVED_OBJECT_TYPE}.config_id : (${agentConfigsToCheck.join(' or ')})`,
       },
     });
     setAgentsCount(data?.total || 0);

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/datasource_delete_provider.tsx

@@ -9,8 +9,8 @@ import { EuiCallOut, EuiConfirmModal, EuiOverlayMask, EuiSpacer } from '@elastic
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { useCore, sendRequest, sendDeleteDatasource, useConfig } from '../../../hooks';
-import { AGENT_API_ROUTES } from '../../../../../../common/constants';
-import { AgentConfig } from '../../../../../../common/types/models';
+import { AGENT_API_ROUTES, AGENT_SAVED_OBJECT_TYPE } from '../../../constants';
+import { AgentConfig } from '../../../types';
 
 interface Props {
   agentConfig: AgentConfig;
@@ -51,7 +51,7 @@ export const DatasourceDeleteProvider: React.FunctionComponent<Props> = ({
         query: {
           page: 1,
           perPage: 1,
-          kuery: `agents.config_id : ${agentConfig.id}`,
+          kuery: `${AGENT_SAVED_OBJECT_TYPE}.config_id : ${agentConfig.id}`,
         },
       });
       setAgentsCount(data?.total || 0);

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/settings_panel.tsx

@@ -8,10 +8,11 @@ import React from 'react';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiTitle, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui';
 import { EuiSpacer } from '@elastic/eui';
-import { useGetPackageInstallStatus } from '../../hooks';
 import { InstallStatus, PackageInfo } from '../../../../types';
-import { InstallationButton } from './installation_button';
 import { useGetDatasources } from '../../../../hooks';
+import { DATASOURCE_SAVED_OBJECT_TYPE } from '../../../../constants';
+import { useGetPackageInstallStatus } from '../../hooks';
+import { InstallationButton } from './installation_button';
 
 const NoteLabel = () => (
   <FormattedMessage
@@ -26,7 +27,7 @@ export const SettingsPanel = (
   const { data: datasourcesData } = useGetDatasources({
     perPage: 0,
     page: 1,
-    kuery: `datasources.package.name:${props.name}`,
+    kuery: `${DATASOURCE_SAVED_OBJECT_TYPE}.package.name:${props.name}`,
   });
   const { name, title, removable } = props;
   const packageInstallStatus = getPackageInstallStatus(name);

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/agent_events_table.tsx

@@ -17,6 +17,7 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage, FormattedTime } from '@kbn/i18n/react';
+import { AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../../constants';
 import { Agent, AgentEvent } from '../../../../types';
 import { usePagination, useGetOneAgentEvents } from '../../../../hooks';
 import { SearchBar } from '../../../../components/search_bar';
@@ -130,7 +131,11 @@ export const AgentEventsTable: React.FunctionComponent<{ agent: Agent }> = ({ ag
       <EuiSpacer size="l" />
       <EuiFlexGroup>
         <EuiFlexItem>
-          <SearchBar value={search} onChange={setSearch} fieldPrefix={'agent_events'} />
+          <SearchBar
+            value={search}
+            onChange={setSearch}
+            fieldPrefix={AGENT_EVENT_SAVED_OBJECT_TYPE}
+          />
         </EuiFlexItem>
         <EuiFlexItem grow={null}>
           <EuiButton color="secondary" iconType="refresh" onClick={onClickRefresh}>

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx

@@ -40,7 +40,11 @@ import { SearchBar } from '../../../components/search_bar';
 import { AgentHealth } from '../components/agent_health';
 import { AgentUnenrollProvider } from '../components/agent_unenroll_provider';
 import { AgentStatusKueryHelper } from '../../../services';
-import { FLEET_AGENT_DETAIL_PATH, AGENT_CONFIG_DETAILS_PATH } from '../../../constants';
+import {
+  FLEET_AGENT_DETAIL_PATH,
+  AGENT_CONFIG_DETAILS_PATH,
+  AGENT_SAVED_OBJECT_TYPE,
+} from '../../../constants';
 
 const NO_WRAP_TRUNCATE_STYLE: CSSProperties = Object.freeze({
   overflow: 'hidden',
@@ -180,7 +184,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
     if (kuery) {
       kuery = `(${kuery}) and`;
     }
-    kuery = `${kuery} agents.config_id : (${selectedConfigs
+    kuery = `${kuery} ${AGENT_SAVED_OBJECT_TYPE}.config_id : (${selectedConfigs
       .map(config => `"${config}"`)
       .join(' or ')})`;
   }
@@ -394,7 +398,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
                   });
                   setSearch(newSearch);
                 }}
-                fieldPrefix="agents"
+                fieldPrefix={AGENT_SAVED_OBJECT_TYPE}
               />
             </EuiFlexItem>
             <EuiFlexItem grow={2}>