[dot-kibana-split] Generalise FTR and integration tests to support mu…

…ltiple SO indices (#154884) Part of #154888 There are a bunch of FTR and integration tests that interact with the SO indices directly, assuming there are only two SO indices: `.kibana` and `.kibana_task_manager`. The goal of this PR is to factorise that knowledge and make these tests use a list of SO indices instead.
elastic · Apr 18, 2023 · c7956c6 · c7956c6
1 parent e13e3cb
commit c7956c6
Show file tree

Hide file tree

Showing 12 changed files with 109 additions and 37 deletions.
diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts
@@ -52,7 +52,11 @@ export type {
   SavedObjectsExportablePredicate,
 } from './src/saved_objects_management';
 export type { SavedObjectStatusMeta } from './src/saved_objects_status';
-export { MAIN_SAVED_OBJECT_INDEX } from './src/saved_objects_index_pattern';
+export {
+  MAIN_SAVED_OBJECT_INDEX,
+  TASK_MANAGER_SAVED_OBJECT_INDEX,
+  SavedObjectsIndexPatterns,
+} from './src/saved_objects_index_pattern';
 export type {
   SavedObjectsType,
   SavedObjectTypeExcludeFromUpgradeFilterHook,

diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts
@@ -6,4 +6,13 @@
  * Side Public License, v 1.
  */
 
+/**
+ * Collect and centralize the names of the different saved object indices.
+ * Note that all of them start with the '.kibana' prefix.
+ * There are multiple places in the code that these indices have the form .kibana*.
+ * However, beware that there are some system indices that have the same prefix
+ * but are NOT used to store saved objects, e.g.: .kibana_security_session_1
+ */
 export const MAIN_SAVED_OBJECT_INDEX = '.kibana';
+export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_manager`;
+export const SavedObjectsIndexPatterns = [MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX];
diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts
@@ -8,9 +8,6 @@
 
 import path from 'path';
 import { unlink } from 'fs/promises';
-import { REPO_ROOT } from '@kbn/repo-info';
-import { Env } from '@kbn/config';
-import { getEnvOptions } from '@kbn/config-mocks';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
 import { Root } from '@kbn/core-root-server-internal';
@@ -19,8 +16,8 @@ import {
   createRootWithCorePlugins,
   type TestElasticsearchUtils,
 } from '@kbn/core-test-helpers-kbn-server';
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 
-const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version;
 const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log');
 
 async function removeLogFile() {
@@ -105,8 +102,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {
     await new Promise((resolve) => setTimeout(resolve, 10000));
   };
 
-  const migratedIndex = `.kibana_${kibanaVersion}_001`;
-
   beforeAll(async () => {
     await removeLogFile();
     await startServers({
@@ -121,7 +116,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {
 
   it('copies all the document of the previous index to the new one', async () => {
     const migratedIndexResponse = await esClient.count({
-      index: migratedIndex,
+      index: SavedObjectsIndexPatterns,
     });
     const oldIndexResponse = await esClient.count({
       index: '.kibana_1',

diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts
@@ -9,7 +9,7 @@
 import Hapi from '@hapi/hapi';
 import h2o2 from '@hapi/h2o2';
 import { URL } from 'url';
-import type { SavedObject } from '@kbn/core-saved-objects-server';
+import { SavedObject, SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server';
 import type { InternalCoreSetup, InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
 import { Root } from '@kbn/core-root-server-internal';
@@ -18,6 +18,7 @@ import {
   createTestServers,
   type TestElasticsearchUtils,
 } from '@kbn/core-test-helpers-kbn-server';
+import { kibanaPackageJson as pkg } from '@kbn/repo-info';
 import {
   declareGetRoute,
   declareDeleteRoute,
@@ -30,6 +31,7 @@ import {
   declarePostUpdateByQueryRoute,
   declarePassthroughRoute,
   setProxyInterrupt,
+  allCombinationsPermutations,
 } from './repository_with_proxy_utils';
 
 let esServer: TestElasticsearchUtils;
@@ -98,17 +100,24 @@ describe('404s from proxies', () => {
     await hapiServer.register(h2o2);
     // register specific routes to modify the response and a catch-all to relay the request/response as-is
 
-    declareGetRoute(hapiServer, esHostname, esPort);
-    declareDeleteRoute(hapiServer, esHostname, esPort);
-    declarePostUpdateRoute(hapiServer, esHostname, esPort);
+    allCombinationsPermutations(
+      SavedObjectsIndexPatterns.map((indexPattern) => `${indexPattern}_${pkg.version}`)
+    )
+      .map((indices) => indices.join(','))
+      .forEach((kbnIndexPath) => {
+        declareGetRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+        declareDeleteRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+        declarePostUpdateRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+
+        declareGetSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+        declarePostSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+        declarePostPitRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+        declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, kbnIndexPath);
+      });
 
-    declareGetSearchRoute(hapiServer, esHostname, esPort);
-    declarePostSearchRoute(hapiServer, esHostname, esPort);
+    // register index-agnostic routes
     declarePostBulkRoute(hapiServer, esHostname, esPort);
     declarePostMgetRoute(hapiServer, esHostname, esPort);
-    declarePostPitRoute(hapiServer, esHostname, esPort);
-    declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort);
-
     declarePassthroughRoute(hapiServer, esHostname, esPort);
 
     await hapiServer.start();

diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts
@@ -7,7 +7,6 @@
  */
 import Hapi from '@hapi/hapi';
 import { IncomingMessage } from 'http';
-import { kibanaPackageJson as pkg } from '@kbn/repo-info';
 
 // proxy setup
 const defaultProxyOptions = (hostname: string, port: string) => ({
@@ -52,10 +51,13 @@ const proxyOnResponseHandler = async (res: IncomingMessage, h: Hapi.ResponseTool
     .code(404);
 };
 
-const kbnIndex = `.kibana_${pkg.version}`;
-
 // GET /.kibana_8.0.0/_doc/{type*} route (repository.get calls)
-export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
+export const declareGetRoute = (
+  hapiServer: Hapi.Server,
+  hostname: string,
+  port: string,
+  kbnIndex: string
+) =>
   hapiServer.route({
     method: 'GET',
     path: `/${kbnIndex}/_doc/{type*}`,
@@ -70,7 +72,12 @@ export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port:
     },
   });
 // DELETE /.kibana_8.0.0/_doc/{type*} route (repository.delete calls)
-export const declareDeleteRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
+export const declareDeleteRoute = (
+  hapiServer: Hapi.Server,
+  hostname: string,
+  port: string,
+  kbnIndex: string
+) =>
   hapiServer.route({
     method: 'DELETE',
     path: `/${kbnIndex}/_doc/{_id*}`,
@@ -133,7 +140,12 @@ export const declarePostMgetRoute = (hapiServer: Hapi.Server, hostname: string,
     },
   });
 // GET _search route
-export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
+export const declareGetSearchRoute = (
+  hapiServer: Hapi.Server,
+  hostname: string,
+  port: string,
+  kbnIndex: string
+) =>
   hapiServer.route({
     method: 'GET',
     path: `/${kbnIndex}/_search`,
@@ -149,7 +161,12 @@ export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string,
     },
   });
 // POST _search route (`find` calls)
-export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
+export const declarePostSearchRoute = (
+  hapiServer: Hapi.Server,
+  hostname: string,
+  port: string,
+  kbnIndex: string
+) =>
   hapiServer.route({
     method: 'POST',
     path: `/${kbnIndex}/_search`,
@@ -168,7 +185,12 @@ export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string
     },
   });
 // POST _update
-export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
+export const declarePostUpdateRoute = (
+  hapiServer: Hapi.Server,
+  hostname: string,
+  port: string,
+  kbnIndex: string
+) =>
   hapiServer.route({
     method: 'POST',
     path: `/${kbnIndex}/_update/{_id*}`,
@@ -187,7 +209,12 @@ export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string
     },
   });
 // POST _pit
-export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
+export const declarePostPitRoute = (
+  hapiServer: Hapi.Server,
+  hostname: string,
+  port: string,
+  kbnIndex: string
+) =>
   hapiServer.route({
     method: 'POST',
     path: `/${kbnIndex}/_pit`,
@@ -209,7 +236,8 @@ export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, p
 export const declarePostUpdateByQueryRoute = (
   hapiServer: Hapi.Server,
   hostname: string,
-  port: string
+  port: string,
+  kbnIndex: string
 ) =>
   hapiServer.route({
     method: 'POST',
@@ -244,3 +272,22 @@ export const declarePassthroughRoute = (hapiServer: Hapi.Server, hostname: strin
       },
     },
   });
+
+export function allCombinationsPermutations<T>(collection: T[]): T[][] {
+  const recur = (subcollection: T[], size: number): T[][] => {
+    if (size <= 0) {
+      return [[]];
+    }
+    const permutations: T[][] = [];
+    subcollection.forEach((value, index, array) => {
+      array = array.slice();
+      array.splice(index, 1);
+      recur(array, size - 1).forEach((permutation) => {
+        permutation.unshift(value);
+        permutations.push(permutation);
+      });
+    });
+    return permutations;
+  };
+  return collection.map((_, n) => recur(collection, n + 1)).flat();
+}
diff --git a/x-pack/test/api_integration/apis/security/index_fields.ts b/x-pack/test/api_integration/apis/security/index_fields.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
@@ -24,7 +25,7 @@ export default function ({ getService }: FtrProviderContext) {
     describe('GET /internal/security/fields/{query}', () => {
       it('should return a list of available index mapping fields', async () => {
         await supertest
-          .get('/internal/security/fields/.kibana')
+          .get(`/internal/security/fields/${SavedObjectsIndexPatterns.join(',')}`)
           .set('kbn-xsrf', 'xxx')
           .send()
           .expect(200)

diff --git a/x-pack/test/api_integration/apis/security_solution/utils.ts b/x-pack/test/api_integration/apis/security_solution/utils.ts
@@ -9,6 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { TransportResult } from '@elastic/elasticsearch';
 import type { Client } from '@elastic/elasticsearch';
 import { JsonObject } from '@kbn/utility-types';
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 
 export async function getSavedObjectFromES<T>(
   es: Client,
@@ -17,7 +18,7 @@ export async function getSavedObjectFromES<T>(
 ): Promise<TransportResult<estypes.SearchResponse<T>, unknown>> {
   return await es.search<T>(
     {
-      index: '.kibana',
+      index: SavedObjectsIndexPatterns,
       body: {
         query: {
           bool: {

diff --git a/x-pack/test/common/lib/test_data_loader.ts b/x-pack/test/common/lib/test_data_loader.ts
@@ -6,6 +6,7 @@
  */
 
 import { LegacyUrlAlias } from '@kbn/core-saved-objects-base-server-internal';
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 import Fs from 'fs/promises';
 import { FtrProviderContext } from '../ftr_provider_context';
 
@@ -176,7 +177,7 @@ export function getTestDataLoader({ getService }: Pick<FtrProviderContext, 'getS
 
     deleteAllSavedObjectsFromKibanaIndex: async () => {
       await es.deleteByQuery({
-        index: '.kibana',
+        index: SavedObjectsIndexPatterns,
         wait_for_completion: true,
         body: {
           // @ts-expect-error

diff --git a/x-pack/test/spaces_api_integration/common/lib/space_test_utils.ts b/x-pack/test/spaces_api_integration/common/lib/space_test_utils.ts
@@ -6,6 +6,7 @@
  */
 
 import type { Client } from '@elastic/elasticsearch';
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
 
 export function getUrlPrefix(spaceId?: string) {
@@ -39,7 +40,7 @@ export function getTestScenariosForSpace(spaceId: string) {
 
 export function getAggregatedSpaceData(es: Client, objectTypes: string[]) {
   return es.search({
-    index: '.kibana',
+    index: SavedObjectsIndexPatterns,
     body: {
       size: 0,
       runtime_mappings: {

diff --git a/x-pack/test/spaces_api_integration/common/suites/delete.ts b/x-pack/test/spaces_api_integration/common/suites/delete.ts
@@ -8,6 +8,7 @@
 import expect from '@kbn/expect';
 import { SuperTest } from 'supertest';
 import type { Client } from '@elastic/elasticsearch';
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 import { getAggregatedSpaceData, getTestScenariosForSpace } from '../lib/space_test_utils';
 import { MULTI_NAMESPACE_SAVED_OBJECT_TEST_CASES as CASES } from '../lib/saved_object_test_cases';
 import { DescribeFn, TestDefinitionAuthentication } from '../lib/types';
@@ -105,7 +106,7 @@ export function deleteTestSuiteFactory(es: Client, esArchiver: any, supertest: S
     // Since Space 2 was deleted, any multi-namespace objects that existed in that space
     // are updated to remove it, and of those, any that don't exist in any space are deleted.
     const multiNamespaceResponse = await es.search<Record<string, any>>({
-      index: '.kibana',
+      index: SavedObjectsIndexPatterns,
       size: 100,
       body: { query: { terms: { type: ['sharedtype'] } } },
     });

diff --git a/x-pack/test/spaces_api_integration/common/suites/disable_legacy_url_aliases.ts b/x-pack/test/spaces_api_integration/common/suites/disable_legacy_url_aliases.ts
@@ -6,12 +6,13 @@
  */
 
 import expect from '@kbn/expect';
-import { SuperTest } from 'supertest';
+import type { SuperTest } from 'supertest';
 import type { Client } from '@elastic/elasticsearch';
 import type { LegacyUrlAlias } from '@kbn/core-saved-objects-base-server-internal';
+import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
 import { SPACES } from '../lib/spaces';
 import { getUrlPrefix } from '../../../saved_object_api_integration/common/lib/saved_object_test_utils';
-import {
+import type {
   ExpectResponseBody,
   TestDefinition,
   TestSuite,
@@ -62,7 +63,8 @@ export function disableLegacyUrlAliasesTestSuiteFactory(
       }
       const esResponse = await es.get<RawLegacyUrlAlias>(
         {
-          index: '.kibana',
+          // affected by the .kibana split, assumes LEGACY_URL_ALIAS_TYPE is stored in .kibana
+          index: MAIN_SAVED_OBJECT_INDEX,
           id: `${LEGACY_URL_ALIAS_TYPE}:${targetSpace}:${targetType}:${sourceId}`,
         },
         { ignore: [404] }

diff --git a/x-pack/test/spaces_api_integration/common/suites/update_objects_spaces.ts b/x-pack/test/spaces_api_integration/common/suites/update_objects_spaces.ts
@@ -14,6 +14,7 @@ import {
   SavedObjectsErrorHelpers,
   SavedObjectsUpdateObjectsSpacesResponse,
 } from '@kbn/core/server';
+import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server';
 import { SPACES } from '../lib/spaces';
 import {
   expectResponses,
@@ -98,11 +99,11 @@ export function updateObjectsSpacesTestSuiteFactory(
             if (expectAliasDifference !== undefined) {
               // if we deleted an object that had an alias pointing to it, the alias should have been deleted as well
               if (!hasRefreshed) {
-                await es.indices.refresh({ index: '.kibana' }); // alias deletion uses refresh: false, so we need to manually refresh the index before searching
+                await es.indices.refresh({ index: SavedObjectsIndexPatterns }); // alias deletion uses refresh: false, so we need to manually refresh the index before searching
                 hasRefreshed = true;
               }
               const searchResponse = await es.search({
-                index: '.kibana',
+                index: SavedObjectsIndexPatterns,
                 body: {
                   size: 0,
                   query: { terms: { type: ['legacy-url-alias'] } },