[7.x] Add Ingest Manager plugin to master (#56567)

src/dev/typescript/projects.ts

@@ -19,7 +19,6 @@
 
 import glob from 'glob';
 import { resolve } from 'path';
-
 import { REPO_ROOT } from '../constants';
 import { Project } from './project';
 

src/plugins/es_ui_shared/public/index.ts

@@ -18,3 +18,12 @@
  */
 
 export { JsonEditor, OnJsonEditorUpdateHandler } from './components/json_editor';
+
+export {
+  SendRequestConfig,
+  SendRequestResponse,
+  UseRequestConfig,
+  UseRequestResponse,
+  sendRequest,
+  useRequest,
+} from './request/np_ready_request';

src/plugins/es_ui_shared/public/request/np_ready_request.ts

@@ -28,8 +28,8 @@ export interface SendRequestConfig {
   body?: any;
 }
 
-export interface SendRequestResponse {
-  data: any;
+export interface SendRequestResponse<D = any> {
+  data: D | null;
   error: Error | null;
 }
 
@@ -39,18 +39,18 @@ export interface UseRequestConfig extends SendRequestConfig {
   deserializer?: (data: any) => any;
 }
 
-export interface UseRequestResponse {
+export interface UseRequestResponse<D = any> {
   isInitialRequest: boolean;
   isLoading: boolean;
-  error: null | unknown;
-  data: any;
-  sendRequest: (...args: any[]) => Promise<SendRequestResponse>;
+  error: Error | null;
+  data: D | null;
+  sendRequest: (...args: any[]) => Promise<SendRequestResponse<D>>;
 }
 
-export const sendRequest = async (
+export const sendRequest = async <D = any>(
   httpClient: HttpSetup,
   { path, method, body, query }: SendRequestConfig
-): Promise<SendRequestResponse> => {
+): Promise<SendRequestResponse<D>> => {
   try {
     const response = await httpClient[method](path, { body, query });
 
@@ -66,7 +66,7 @@ export const sendRequest = async (
   }
 };
 
-export const useRequest = (
+export const useRequest = <D = any>(
   httpClient: HttpSetup,
   {
     path,
@@ -77,8 +77,8 @@ export const useRequest = (
     initialData,
     deserializer = (data: any): any => data,
   }: UseRequestConfig
-): UseRequestResponse => {
-  const sendRequestRef = useRef<() => Promise<SendRequestResponse>>();
+): UseRequestResponse<D> => {
+  const sendRequestRef = useRef<() => Promise<SendRequestResponse<D>>>();
 
   // Main states for tracking request status and data
   const [error, setError] = useState<null | any>(null);

x-pack/index.js

@@ -37,6 +37,7 @@ import { transform } from './legacy/plugins/transform';
 import { actions } from './legacy/plugins/actions';
 import { alerting } from './legacy/plugins/alerting';
 import { lens } from './legacy/plugins/lens';
+import { ingestManager } from './legacy/plugins/ingest_manager';
 import { triggersActionsUI } from './legacy/plugins/triggers_actions_ui';
 
 module.exports = function(kibana) {
@@ -74,6 +75,7 @@ module.exports = function(kibana) {
     snapshotRestore(kibana),
     actions(kibana),
     alerting(kibana),
+    ingestManager(kibana),
     triggersActionsUI(kibana),
   ];
 };

x-pack/legacy/plugins/ingest_manager/index.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import {
+  savedObjectMappings,
+  OUTPUT_SAVED_OBJECT_TYPE,
+  AGENT_CONFIG_SAVED_OBJECT_TYPE,
+  DATASOURCE_SAVED_OBJECT_TYPE,
+} from '../../../plugins/ingest_manager/server';
+
+// TODO https://github.com/elastic/kibana/issues/46373
+// const INDEX_NAMES = {
+//   INGEST: '.kibana',
+// };
+
+export function ingestManager(kibana: any) {
+  return new kibana.Plugin({
+    id: 'ingestManager',
+    uiExports: {
+      savedObjectSchemas: {
+        [AGENT_CONFIG_SAVED_OBJECT_TYPE]: {
+          isNamespaceAgnostic: true,
+          // indexPattern: INDEX_NAMES.INGEST,
+        },
+        [OUTPUT_SAVED_OBJECT_TYPE]: {
+          isNamespaceAgnostic: true,
+          // indexPattern: INDEX_NAMES.INGEST,
+        },
+        [DATASOURCE_SAVED_OBJECT_TYPE]: {
+          isNamespaceAgnostic: true,
+          // indexPattern: INDEX_NAMES.INGEST,
+        },
+      },
+      mappings: savedObjectMappings,
+    },
+  });
+}

x-pack/plugins/ingest_manager/README.md

@@ -0,0 +1,20 @@
+# Ingest Manager
+
+## Getting started
+See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-kibana).
+
+One common workflow is:
+
+ 1. `yarn es snapshot`
+ 1. In another shell: `yarn start --xpack.ingestManager.enabled=true` (or set in `config.yml`)
+
+## HTTP API
+  1. Nothing by default. If `xpack.ingestManager.enabled=true`, it adds the `DATASOURCE_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts)
+  1. [Integration tests](../../test/api_integration/apis/ingest_manager/endpoints.ts)
+  1. In later versions the EPM and Fleet routes will be added when their flags are enabled. See the [currently disabled logic to add those routes](https://github.com/jfsiii/kibana/blob/feature-ingest-manager/x-pack/plugins/ingest_manager/server/plugin.ts#L86-L90).
+
+## Plugin architecture
+Follows the `common`, `server`, `public` structure from the [Architecture Style Guide
+](https://github.com/elastic/kibana/blob/master/style_guides/architecture_style_guide.md#file-and-folder-structure).
+
+We use New Platform approach (structure, APIs, etc) where possible. There's a `kibana.json` manifest, and the server uses the `server/{index,plugin}.ts` approach from [`MIGRATION.md`](https://github.com/elastic/kibana/blob/master/src/core/MIGRATION.md#architecture). 

x-pack/plugins/ingest_manager/common/constants/agent_config.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { AgentConfigStatus } from '../types';
+
+export const AGENT_CONFIG_SAVED_OBJECT_TYPE = 'agent_configs';
+
+export const DEFAULT_AGENT_CONFIG_ID = 'default';
+
+export const DEFAULT_AGENT_CONFIG = {
+  name: 'Default config',
+  namespace: 'default',
+  description: 'Default agent configuration created by Kibana',
+  status: AgentConfigStatus.Active,
+  datasources: [],
+};

x-pack/plugins/ingest_manager/common/constants/datasource.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const DATASOURCE_SAVED_OBJECT_TYPE = 'datasources';

x-pack/plugins/ingest_manager/common/constants/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './plugin';
+export * from './routes';
+
+export * from './agent_config';
+export * from './datasource';
+export * from './output';

x-pack/plugins/ingest_manager/common/constants/output.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { OutputType } from '../types';
+
+export const OUTPUT_SAVED_OBJECT_TYPE = 'outputs';
+
+export const DEFAULT_OUTPUT_ID = 'default';
+
+export const DEFAULT_OUTPUT = {
+  name: DEFAULT_OUTPUT_ID,
+  type: OutputType.Elasticsearch,
+  hosts: [''],
+  ingest_pipeline: DEFAULT_OUTPUT_ID,
+  api_key: '',
+};

x-pack/plugins/ingest_manager/common/constants/plugin.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const PLUGIN_ID = 'ingestManager';

x-pack/plugins/ingest_manager/common/constants/routes.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+// Base API paths
+export const API_ROOT = `/api/ingest_manager`;
+export const DATASOURCE_API_ROOT = `${API_ROOT}/datasources`;
+export const AGENT_CONFIG_API_ROOT = `${API_ROOT}/agent_configs`;
+export const EPM_API_ROOT = `${API_ROOT}/epm`;
+export const FLEET_API_ROOT = `${API_ROOT}/fleet`;
+
+// EPM API routes
+export const EPM_API_ROUTES = {
+  LIST_PATTERN: `${EPM_API_ROOT}/list`,
+  INFO_PATTERN: `${EPM_API_ROOT}/package/{pkgkey}`,
+  INSTALL_PATTERN: `${EPM_API_ROOT}/install/{pkgkey}`,
+  DELETE_PATTERN: `${EPM_API_ROOT}/delete/{pkgkey}`,
+  CATEGORIES_PATTERN: `${EPM_API_ROOT}/categories`,
+};
+
+// Datasource API routes
+export const DATASOURCE_API_ROUTES = {
+  LIST_PATTERN: `${DATASOURCE_API_ROOT}`,
+  INFO_PATTERN: `${DATASOURCE_API_ROOT}/{datasourceId}`,
+  CREATE_PATTERN: `${DATASOURCE_API_ROOT}`,
+  UPDATE_PATTERN: `${DATASOURCE_API_ROOT}/{datasourceId}`,
+  DELETE_PATTERN: `${DATASOURCE_API_ROOT}/delete`,
+};
+
+// Agent config API routes
+export const AGENT_CONFIG_API_ROUTES = {
+  LIST_PATTERN: `${AGENT_CONFIG_API_ROOT}`,
+  INFO_PATTERN: `${AGENT_CONFIG_API_ROOT}/{agentConfigId}`,
+  CREATE_PATTERN: `${AGENT_CONFIG_API_ROOT}`,
+  UPDATE_PATTERN: `${AGENT_CONFIG_API_ROOT}/{agentConfigId}`,
+  DELETE_PATTERN: `${AGENT_CONFIG_API_ROOT}/delete`,
+};
+
+// Fleet setup API routes
+export const FLEET_SETUP_API_ROUTES = {
+  INFO_PATTERN: `${FLEET_API_ROOT}/setup`,
+  CREATE_PATTERN: `${FLEET_API_ROOT}/setup`,
+};

x-pack/plugins/ingest_manager/common/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './constants';
+export * from './services';
+export * from './types';

x-pack/plugins/ingest_manager/common/services/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+export * from './routes';

x-pack/plugins/ingest_manager/common/services/routes.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import {
+  EPM_API_ROOT,
+  EPM_API_ROUTES,
+  DATASOURCE_API_ROUTES,
+  AGENT_CONFIG_API_ROUTES,
+} from '../constants';
+
+export const epmRouteService = {
+  getCategoriesPath: () => {
+    return EPM_API_ROUTES.CATEGORIES_PATTERN;
+  },
+
+  getListPath: () => {
+    return EPM_API_ROUTES.LIST_PATTERN;
+  },
+
+  getInfoPath: (pkgkey: string) => {
+    return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey);
+  },
+
+  getFilePath: (filePath: string) => {
+    return `${EPM_API_ROOT}${filePath}`;
+  },
+
+  getInstallPath: (pkgkey: string) => {
+    return EPM_API_ROUTES.INSTALL_PATTERN.replace('{pkgkey}', pkgkey).replace(/\/$/, ''); // trim trailing slash
+  },
+
+  getRemovePath: (pkgkey: string) => {
+    return EPM_API_ROUTES.DELETE_PATTERN.replace('{pkgkey}', pkgkey).replace(/\/$/, ''); // trim trailing slash
+  },
+};
+
+export const datasourceRouteService = {
+  getListPath: () => {
+    return DATASOURCE_API_ROUTES.LIST_PATTERN;
+  },
+
+  getInfoPath: (datasourceId: string) => {
+    return DATASOURCE_API_ROUTES.INFO_PATTERN.replace('{datasourceId}', datasourceId);
+  },
+
+  getCreatePath: () => {
+    return DATASOURCE_API_ROUTES.CREATE_PATTERN;
+  },
+
+  getUpdatePath: (datasourceId: string) => {
+    return DATASOURCE_API_ROUTES.UPDATE_PATTERN.replace('{datasourceId}', datasourceId);
+  },
+};
+
+export const agentConfigRouteService = {
+  getListPath: () => {
+    return AGENT_CONFIG_API_ROUTES.LIST_PATTERN;
+  },
+
+  getInfoPath: (agentConfigId: string) => {
+    return AGENT_CONFIG_API_ROUTES.INFO_PATTERN.replace('{agentConfigId}', agentConfigId);
+  },
+
+  getCreatePath: () => {
+    return AGENT_CONFIG_API_ROUTES.CREATE_PATTERN;
+  },
+
+  getUpdatePath: (agentConfigId: string) => {
+    return AGENT_CONFIG_API_ROUTES.UPDATE_PATTERN.replace('{agentConfigId}', agentConfigId);
+  },
+
+  getDeletePath: () => {
+    return AGENT_CONFIG_API_ROUTES.DELETE_PATTERN;
+  },
+};

x-pack/plugins/ingest_manager/common/types/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './models';
+export * from './rest_spec';
+
+export interface IngestManagerConfigType {
+  enabled: boolean;
+  epm: {
+    enabled: boolean;
+    registryUrl: string;
+  };
+  fleet: {
+    enabled: boolean;
+    defaultOutputHost: string;
+  };
+}