Add support for GeoIP processor databases in Ingest Pipelines (#190830)

Fixes #190818

## Summary

Elasticsearch has added support for GeoIP, enabling the use of paid
GeoIP databases from MaxMind/IPInfo for more accurate and granular
geolocation data. As such we should add support to ingest pipelines UI
for making this available to the user.


* If the user doesn't have enough privileges, the "Manage Pipelines"
link and UI won't show.
* Users can add two types of databases through the UI: MaxMind and
IPinfo. Database names are predefined by ES, and the user cannot enter
their own.
* Certain types of databases (local and web) can be configured through
ES, and these will appear in the UI, but they cannot be deleted as they
are read-only.
* When configuring a `IP location` processor, the database field will
display a list of available and configured databases that the user can
select. It also allows for free-text input if the user wants to
configure a database that does not yet exist.
* The new IP location processor is essentially a clone of the GeoIP
processor, which we are moving away from due to copyright issues.
However, it was decided that GeoIP will remain as is for backward
compatibility, and all new work will only be added to IP location going
forward.
* I left a few mocks in the `server/routes/api/geoip_database/list.ts `
to try `local/web` types

## Release note
The Ingest Pipelines app now supports adding and managing databases for
the GeoIP processor. Additionally, the pipeline creation flow now
includes support for the IP Location processor.

<details>
<summary>Screenshots</summary>

![Screenshot 2024-10-07 at 09 36
31](https://github.com/user-attachments/assets/60d438cc-6658-4475-bd27-036c7d13d496)
![Screenshot 2024-10-07 at 09 38
58](https://github.com/user-attachments/assets/7c08e94f-b35c-4e78-a204-1fb456d88181)
![Screenshot 2024-10-07 at 09 47
08](https://github.com/user-attachments/assets/2baca0bd-811d-4dd5-9eb6-9b3f41579249)
![Screenshot 2024-10-07 at 09 47
20](https://github.com/user-attachments/assets/74d8664c-8c73-41f3-8cd5-e0670f3ada77)
![Screenshot 2024-10-07 at 09 48
19](https://github.com/user-attachments/assets/9fb4c186-6224-404c-a8d6-5c44c14da951)
![Screenshot 2024-10-07 at 09 48
25](https://github.com/user-attachments/assets/07e4909d-2613-45aa-918b-11a189e14f6f)


</details>

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Ignacio Rivas <[email protected]>
Co-authored-by: Elena Stoeva <[email protected]>
Co-authored-by: Elena Stoeva <[email protected]>
Co-authored-by: Matthew Kime <[email protected]>

config/serverless.yml

@@ -113,6 +113,9 @@ xpack.index_management.enableTogglingDataRetention: false
 # Disable project level rentention checks in DSL form from Index Management UI
 xpack.index_management.enableProjectLevelRetentionChecks: false
 
+# Disable Manage Processors UI in Ingest Pipelines
+xpack.ingest_pipelines.enableManageProcessors: false
+
 # Keep deeplinks visible so that they are shown in the sidenav
 dev_tools.deeplinks.navLinkStatus: visible
 management.deeplinks.navLinkStatus: visible

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -314,6 +314,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.ml.nlp.modelDeployment.vCPURange.medium.static (number?)',
         'xpack.osquery.actionEnabled (boolean?)',
         'xpack.remote_clusters.ui.enabled (boolean?)',
+        'xpack.ingest_pipelines.enableManageProcessors (boolean?|never)',
         /**
          * NOTE: The Reporting plugin is currently disabled in functional tests (see test/functional/config.base.js).
          * It will be re-enabled once #102552 is completed.

x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/http_requests.ts

@@ -73,12 +73,27 @@ const registerHttpRequestMockHelpers = (
   const setParseCsvResponse = (response?: object, error?: ResponseError) =>
     mockResponse('POST', `${API_BASE_PATH}/parse_csv`, response, error);
 
+  const setLoadDatabasesResponse = (response?: object[], error?: ResponseError) =>
+    mockResponse('GET', `${API_BASE_PATH}/databases`, response, error);
+
+  const setDeleteDatabasesResponse = (
+    databaseName: string,
+    response?: object,
+    error?: ResponseError
+  ) => mockResponse('DELETE', `${API_BASE_PATH}/databases/${databaseName}`, response, error);
+
+  const setCreateDatabasesResponse = (response?: object, error?: ResponseError) =>
+    mockResponse('POST', `${API_BASE_PATH}/databases`, response, error);
+
   return {
     setLoadPipelinesResponse,
     setLoadPipelineResponse,
     setDeletePipelineResponse,
     setCreatePipelineResponse,
     setParseCsvResponse,
+    setLoadDatabasesResponse,
+    setDeleteDatabasesResponse,
+    setCreateDatabasesResponse,
   };
 };
 

x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/index.ts

@@ -10,8 +10,9 @@ import { setup as pipelinesCreateSetup } from './pipelines_create.helpers';
 import { setup as pipelinesCloneSetup } from './pipelines_clone.helpers';
 import { setup as pipelinesEditSetup } from './pipelines_edit.helpers';
 import { setup as pipelinesCreateFromCsvSetup } from './pipelines_create_from_csv.helpers';
+import { setup as manageProcessorsSetup } from './manage_processors.helpers';
 
-export { nextTick, getRandomString, findTestSubject } from '@kbn/test-jest-helpers';
+export { getRandomString, findTestSubject } from '@kbn/test-jest-helpers';
 
 export { setupEnvironment } from './setup_environment';
 
@@ -21,4 +22,5 @@ export const pageHelpers = {
   pipelinesClone: { setup: pipelinesCloneSetup },
   pipelinesEdit: { setup: pipelinesEditSetup },
   pipelinesCreateFromCsv: { setup: pipelinesCreateFromCsvSetup },
+  manageProcessors: { setup: manageProcessorsSetup },
 };

x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/manage_processors.helpers.ts

@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { act } from 'react-dom/test-utils';
+import { HttpSetup } from '@kbn/core/public';
+
+import { registerTestBed, TestBed, AsyncTestBedConfig } from '@kbn/test-jest-helpers';
+import { ManageProcessors } from '../../../public/application/sections';
+import { WithAppDependencies } from './setup_environment';
+import { getManageProcessorsPath, ROUTES } from '../../../public/application/services/navigation';
+
+const testBedConfig: AsyncTestBedConfig = {
+  memoryRouter: {
+    initialEntries: [getManageProcessorsPath()],
+    componentRoutePath: ROUTES.manageProcessors,
+  },
+  doMountAsync: true,
+};
+
+export type ManageProcessorsTestBed = TestBed<ManageProcessorsTestSubjects> & {
+  actions: ReturnType<typeof createActions>;
+};
+
+const createActions = (testBed: TestBed) => {
+  const { component, find, form } = testBed;
+
+  const clickDeleteDatabaseButton = async (index: number) => {
+    const allDeleteButtons = find('deleteGeoipDatabaseButton');
+    const deleteButton = allDeleteButtons.at(index);
+    await act(async () => {
+      deleteButton.simulate('click');
+    });
+
+    component.update();
+  };
+
+  const confirmDeletingDatabase = async () => {
+    await act(async () => {
+      form.setInputValue('geoipDatabaseConfirmation', 'delete');
+    });
+
+    component.update();
+
+    const confirmButton: HTMLButtonElement | null = document.body.querySelector(
+      '[data-test-subj="deleteGeoipDatabaseSubmit"]'
+    );
+
+    expect(confirmButton).not.toBe(null);
+    expect(confirmButton!.disabled).toBe(false);
+    expect(confirmButton!.textContent).toContain('Delete');
+
+    await act(async () => {
+      confirmButton!.click();
+    });
+
+    component.update();
+  };
+
+  const clickAddDatabaseButton = async () => {
+    const button = find('addGeoipDatabaseButton');
+    expect(button).not.toBe(undefined);
+    await act(async () => {
+      button.simulate('click');
+    });
+
+    component.update();
+  };
+
+  const fillOutDatabaseValues = async (
+    databaseType: string,
+    databaseName: string,
+    maxmind?: string
+  ) => {
+    await act(async () => {
+      form.setSelectValue('databaseTypeSelect', databaseType);
+    });
+    component.update();
+
+    if (maxmind) {
+      await act(async () => {
+        form.setInputValue('maxmindField', maxmind);
+      });
+    }
+    await act(async () => {
+      form.setSelectValue('databaseNameSelect', databaseName);
+    });
+
+    component.update();
+  };
+
+  const confirmAddingDatabase = async () => {
+    const confirmButton: HTMLButtonElement | null = document.body.querySelector(
+      '[data-test-subj="addGeoipDatabaseSubmit"]'
+    );
+
+    expect(confirmButton).not.toBe(null);
+    expect(confirmButton!.disabled).toBe(false);
+
+    await act(async () => {
+      confirmButton!.click();
+    });
+
+    component.update();
+  };
+
+  return {
+    clickDeleteDatabaseButton,
+    confirmDeletingDatabase,
+    clickAddDatabaseButton,
+    fillOutDatabaseValues,
+    confirmAddingDatabase,
+  };
+};
+
+export const setup = async (httpSetup: HttpSetup): Promise<ManageProcessorsTestBed> => {
+  const initTestBed = registerTestBed(
+    WithAppDependencies(ManageProcessors, httpSetup),
+    testBedConfig
+  );
+  const testBed = await initTestBed();
+
+  return {
+    ...testBed,
+    actions: createActions(testBed),
+  };
+};
+
+export type ManageProcessorsTestSubjects =
+  | 'manageProcessorsTitle'
+  | 'addGeoipDatabaseForm'
+  | 'addGeoipDatabaseButton'
+  | 'geoipDatabaseList'
+  | 'databaseTypeSelect'
+  | 'maxmindField'
+  | 'databaseNameSelect'
+  | 'addGeoipDatabaseSubmit'
+  | 'deleteGeoipDatabaseButton'
+  | 'geoipDatabaseConfirmation'
+  | 'geoipEmptyListPrompt'
+  | 'geoipListLoadingError';

x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx

@@ -70,6 +70,9 @@ const appServices = {
   },
   overlays: overlayServiceMock.createStartContract(),
   http: httpServiceMock.createStartContract({ basePath: '/mock' }),
+  config: {
+    enableManageProcessors: true,
+  },
 };
 
 export const setupEnvironment = () => {