diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip deleted file mode 100644 index d079624d95988..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.3.0_xpack_sample_saved_objects.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.3.0_xpack_sample_saved_objects.zip deleted file mode 100644 index b79a497d06941..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.3.0_xpack_sample_saved_objects.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts index 5981c2759d5ca..1cb6b12650961 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts @@ -22,18 +22,20 @@ import { readLog, clearLog, currentVersion, + nextMinor, } from '../kibana_migrator_test_kit'; import { + BASELINE_COMPLEX_DOCUMENTS_500K_AFTER, BASELINE_DOCUMENTS_PER_TYPE_500K, BASELINE_TEST_ARCHIVE_500K, } from '../kibana_migrator_archive_utils'; import { - baselineTypes, getReindexingBaselineTypes, getReindexingMigratorTestKit, getUpToDateMigratorTestKit, } from '../kibana_migrator_test_kit.fixtures'; import { delay } from '../test_utils'; +import { expectDocumentsMigratedToHighestVersion } from '../kibana_migrator_test_kit.expect'; const logFilePath = join(__dirname, 'v2_migration.log'); @@ -83,11 +85,11 @@ describe('v2 migration', () => { expect(migrationResults.map((result) => omit(result, 'elapsedMs'))).toMatchInlineSnapshot(` Array [ Object { - "destIndex": ".kibana_migrator_9.0.0_001", + "destIndex": ".kibana_migrator_${currentVersion}_001", "status": "patched", }, Object { - "destIndex": ".kibana_migrator_tasks_9.0.0_001", + "destIndex": ".kibana_migrator_tasks_${currentVersion}_001", "status": "patched", }, ] @@ -214,38 +216,10 @@ describe('v2 migration', () => { }); it('migrates documents to the highest version', async () => { - const typeMigrationVersions: Record = { - basic: '10.1.0', // did not define any model versions - complex: '10.2.0', - task: '10.2.0', - }; - - const resultSets = await Promise.all( - baselineTypes.map(({ name: type }) => - kit.client.search({ - index: [defaultKibanaIndex, defaultKibanaTaskIndex], - query: { - bool: { - should: [ - { - term: { type }, - }, - ], - }, - }, - }) - ) - ); - - expect( - resultSets - .flatMap((result) => result.hits.hits) - .every( - (document) => - document._source.typeMigrationVersion === - typeMigrationVersions[document._source.type] - ) - ).toEqual(true); + await expectDocumentsMigratedToHighestVersion(kit.client, [ + defaultKibanaIndex, + defaultKibanaTaskIndex, + ]); }); describe('a migrator performing a compatible upgrade migration', () => { @@ -338,11 +312,7 @@ describe('v2 migration', () => { }); it('executes the excludeOnUpgrade hook', () => { - // we discard the second half with exclude on upgrade (firstHalf !== true) - // then we discard half all multiples of 100 (1% of them) - expect(primaryIndexCounts.complex).toEqual( - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 / 100 - ); + expect(primaryIndexCounts.complex).toEqual(BASELINE_COMPLEX_DOCUMENTS_500K_AFTER); }); }); @@ -352,13 +322,13 @@ describe('v2 migration', () => { .toMatchInlineSnapshot(` Array [ Object { - "destIndex": ".kibana_migrator_9.1.0_001", - "sourceIndex": ".kibana_migrator_9.0.0_001", + "destIndex": ".kibana_migrator_${nextMinor}_001", + "sourceIndex": ".kibana_migrator_${currentVersion}_001", "status": "migrated", }, Object { - "destIndex": ".kibana_migrator_tasks_9.0.0_001", - "sourceIndex": ".kibana_migrator_tasks_9.0.0_001", + "destIndex": ".kibana_migrator_tasks_${currentVersion}_001", + "sourceIndex": ".kibana_migrator_tasks_${currentVersion}_001", "status": "migrated", }, ] diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/multi_node_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/multi_node_split.test.ts new file mode 100644 index 0000000000000..c10bae0a1b6c9 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/multi_node_split.test.ts @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { join } from 'path'; +import { omit, sortBy } from 'lodash'; +import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; +import type { Client } from '@elastic/elasticsearch'; +import { + clearLog, + defaultKibanaIndex, + defaultKibanaTaskIndex, + getAggregatedTypesCount, + getEsClient, + nextMinor, + startElasticsearch, +} from '../kibana_migrator_test_kit'; +import { + BASELINE_COMPLEX_DOCUMENTS_500K_AFTER, + BASELINE_DOCUMENTS_PER_TYPE_500K, + BASELINE_TEST_ARCHIVE_500K, +} from '../kibana_migrator_archive_utils'; +import { + getRelocatingMigratorTestKit, + kibanaSplitIndex, +} from '../kibana_migrator_test_kit.fixtures'; +import { delay, parseLogFile } from '../test_utils'; +import '../jest_matchers'; +import { expectDocumentsMigratedToHighestVersion } from '../kibana_migrator_test_kit.expect'; + +const PARALLEL_MIGRATORS = 4; +type Job = () => Promise; + +const getLogFile = (node: number) => join(__dirname, `multi_node_split_${node}.log`); +const logFileSecondRun = join(__dirname, `multi_node_split_second_run.log`); + +describe('multiple Kibana nodes performing a reindexing migration', () => { + let esServer: TestElasticsearchUtils['es']; + let jobs: Array>; + let client: Client; + let results: MigrationResult[][]; + + beforeEach(async () => { + for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { + await clearLog(getLogFile(i)); + } + await clearLog(logFileSecondRun); + + esServer = await startElasticsearch({ dataArchive: BASELINE_TEST_ARCHIVE_500K }); + jobs = await createMigratorJobs(PARALLEL_MIGRATORS); + client = await getEsClient(); + await checkBeforeState(); + }); + + it('migrate saved objects normally when started at the same time', async () => { + results = await startWithDelay(jobs, 0); + }); + + it('migrate saved objects normally when started with a small interval', async () => { + results = await startWithDelay(jobs, 1); + }); + + it('migrate saved objects normally when started with an average interval', async () => { + results = await startWithDelay(jobs, 5); + }); + + it('migrate saved objects normally when started with a bigger interval', async () => { + results = await startWithDelay(jobs, 20); + }); + + afterEach(async () => { + checkMigratorsResults(); + await checkIndicesInfo(); + await checkSavedObjectDocuments(); + await checkMigratorsSteps(); + await checkUpToDateOnRestart(); + }); + + afterEach(async () => { + await esServer?.stop(); + await delay(5); // give it a few seconds... cause we always do ¯\_(ツ)_/¯ + }); + + async function checkBeforeState() { + await expect(getAggregatedTypesCount(client, [defaultKibanaIndex])).resolves.toEqual({ + basic: BASELINE_DOCUMENTS_PER_TYPE_500K, + complex: BASELINE_DOCUMENTS_PER_TYPE_500K, + deprecated: BASELINE_DOCUMENTS_PER_TYPE_500K, + server: BASELINE_DOCUMENTS_PER_TYPE_500K, + }); + await expect(getAggregatedTypesCount(client, [defaultKibanaTaskIndex])).resolves.toEqual({ + task: BASELINE_DOCUMENTS_PER_TYPE_500K, + }); + await expect(getAggregatedTypesCount(client, [kibanaSplitIndex])).resolves.toEqual({}); + } + + function checkMigratorsResults() { + const flatResults = results.flat(); // multiple nodes, multiple migrators each + + // each migrator should take less than 60 seconds + expect( + (flatResults as Array<{ elapsedMs?: number }>).every( + ({ elapsedMs }) => !elapsedMs || elapsedMs < 60000 + ) + ).toEqual(true); + + // each migrator has either migrated or patched + expect( + flatResults.every(({ status }) => status === 'migrated' || status === 'patched') + ).toEqual(true); + } + + async function checkIndicesInfo() { + const indicesInfo = await client.indices.get({ index: '.kibana*' }); + [defaultKibanaIndex, kibanaSplitIndex].forEach((index) => + expect(indicesInfo[`${index}_${nextMinor}_001`]).toEqual( + expect.objectContaining({ + aliases: expect.objectContaining({ [index]: expect.any(Object) }), + mappings: { + dynamic: 'strict', + _meta: { + mappingVersions: expect.any(Object), + indexTypesMap: expect.any(Object), + }, + properties: expect.any(Object), + }, + settings: { index: expect.any(Object) }, + }) + ) + ); + + const typesMap = + indicesInfo[`${defaultKibanaIndex}_${nextMinor}_001`].mappings?._meta?.indexTypesMap; + expect(typesMap[defaultKibanaIndex]).toEqual(['complex', 'server']); // 'deprecated' no longer present + expect(typesMap[kibanaSplitIndex]).toEqual(['basic', 'task']); + } + + async function checkSavedObjectDocuments() { + // check documents have been migrated + await expect(getAggregatedTypesCount(client, [defaultKibanaIndex])).resolves.toEqual({ + complex: BASELINE_COMPLEX_DOCUMENTS_500K_AFTER, + }); + await expect(getAggregatedTypesCount(client, [defaultKibanaTaskIndex])).resolves.toEqual({}); + await expect(getAggregatedTypesCount(client, [kibanaSplitIndex])).resolves.toEqual({ + basic: BASELINE_DOCUMENTS_PER_TYPE_500K, + task: BASELINE_DOCUMENTS_PER_TYPE_500K, + }); + await expectDocumentsMigratedToHighestVersion(client, [defaultKibanaIndex, kibanaSplitIndex]); + } + + async function checkMigratorsSteps() { + for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { + const logs = await parseLogFile(getLogFile(i)); + // '.kibana_migrator_split' is a new index, all nodes' migrators must attempt to create it + expect(logs).toContainLogEntries( + [ + `[${kibanaSplitIndex}] INIT -> CREATE_REINDEX_TEMP.`, + `[${kibanaSplitIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`, + // no docs to reindex, as source index did NOT exist + `[${kibanaSplitIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`, + ], + { ordered: true } + ); + + // '.kibana_migrator' and '.kibana_migrator_tasks' are involved in a relocation + [defaultKibanaIndex, defaultKibanaTaskIndex].forEach((index) => { + expect(logs).toContainLogEntries( + [ + `[${index}] INIT -> WAIT_FOR_YELLOW_SOURCE.`, + `[${index}] WAIT_FOR_YELLOW_SOURCE -> CHECK_CLUSTER_ROUTING_ALLOCATION.`, + `[${index}] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.`, + `[${index}] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.`, + `[${index}] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.`, + `[${index}] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.`, + `[${index}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`, + `[${index}] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.`, + `[${index}] Starting to process`, + `[${index}] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.`, + `[${index}] Processed`, + // if the index is closed by another node, we will have instead: REINDEX_SOURCE_TO_TEMP_TRANSFORM => REINDEX_SOURCE_TO_TEMP_CLOSE_PIT. + // `[${index}] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.`, + ], + { ordered: true } + ); + }); + + // after the relocation, all migrators share the final part of the flow + [defaultKibanaIndex, defaultKibanaTaskIndex, kibanaSplitIndex].forEach((index) => { + expect(logs).toContainLogEntries( + [ + `[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`, + `[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`, + `[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`, + `[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`, + `[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`, + `[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`, + `[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.`, + `[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`, + `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`, + `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`, + `[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`, + `[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY_SYNC.`, + `[${index}] MARK_VERSION_INDEX_READY_SYNC`, // all migrators try to update all aliases, all but one will have conclicts + `[${index}] Migration completed after`, + ], + { ordered: true } + ); + }); + + // should NOT retransform anything (we reindexed, thus we transformed already) + [defaultKibanaIndex, defaultKibanaTaskIndex, kibanaSplitIndex].forEach((index) => { + expect(logs).not.toContainLogEntry(`[${index}] OUTDATED_DOCUMENTS_TRANSFORM`); + expect(logs).not.toContainLogEntry( + `[${index}] Kibana is performing a compatible update and it will update the following SO types so that ES can pickup the updated mappings` + ); + }); + } + } + + async function checkUpToDateOnRestart() { + // run a new migrator to ensure everything is up to date + const { runMigrations } = await getRelocatingMigratorTestKit({ + logFilePath: logFileSecondRun, + // no need to filter deprecated this time, they should not be there anymore + }); + const secondRunResults = await runMigrations(); + expect( + sortBy( + secondRunResults.map((result) => omit(result, 'elapsedMs')), + 'destIndex' + ) + ).toEqual([ + { + destIndex: `.kibana_migrator_${nextMinor}_001`, + status: 'patched', + }, + { + destIndex: `.kibana_migrator_split_${nextMinor}_001`, + status: 'patched', + }, + ]); + const logs = await parseLogFile(logFileSecondRun); + expect(logs).not.toContainLogEntries(['REINDEX', 'CREATE', 'UPDATE_TARGET_MAPPINGS']); + } +}); + +async function createMigratorJobs(nodes: number): Promise>> { + const jobs: Array> = []; + + for (let i = 0; i < nodes; ++i) { + const kit = await getRelocatingMigratorTestKit({ + logFilePath: getLogFile(i), + filterDeprecated: true, + }); + jobs.push(kit.runMigrations); + } + + return jobs; +} + +async function startWithDelay(runnables: Array>, delayInSec: number) { + const promises: Array> = []; + const errors: string[] = []; + for (let i = 0; i < runnables.length; i++) { + promises.push( + runnables[i]().catch((reason) => { + errors.push(reason.message ?? reason); + return reason; + }) + ); + if (i < runnables.length - 2) { + // We wait between instances, but not after the last one + await delay(delayInSec); + } + } + const results = await Promise.all(promises); + if (errors.length) { + throw new Error(`Failed to run all parallel jobs: ${errors.join(',')}`); + } else { + return results; + } +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts deleted file mode 100644 index 9bd876166c246..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import del from 'del'; -import { esTestConfig, kibanaServerTestUser } from '@kbn/test'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; -import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { - createTestServers, - createRoot as createkbnTestServerRoot, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; - -const LOG_FILE_PREFIX = 'migration_test_multiple_kibana_nodes'; - -async function removeLogFiles() { - await del([Path.join(__dirname, `${LOG_FILE_PREFIX}_*.log`)], { force: true }); -} - -function extractSortNumberFromId(id: string): number { - const parsedId = parseInt(id.split(':')[1], 10); // "foo:123" -> 123 - if (isNaN(parsedId)) { - throw new Error(`Failed to parse Saved Object ID [${id}]. Result is NaN`); - } - return parsedId; -} - -async function fetchDocs(esClient: ElasticsearchClient, index: string) { - const body = await esClient.search({ - index, - size: 10000, - body: { - query: { - bool: { - should: [ - { - term: { type: 'foo' }, - }, - ], - }, - }, - }, - }); - - return body.hits.hits - .map((h) => ({ - ...h._source, - id: h._id, - })) - .sort((a, b) => extractSortNumberFromId(a.id) - extractSortNumberFromId(b.id)); -} - -interface CreateRootConfig { - logFileName: string; -} - -async function createRoot({ logFileName }: CreateRootConfig) { - const root = createkbnTestServerRoot({ - elasticsearch: { - hosts: [esTestConfig.getUrl()], - username: kibanaServerTestUser.username, - password: kibanaServerTestUser.password, - }, - migrations: { - skip: false, - batchSize: 100, // fixture contains 5000 docs - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFileName, - layout: { - type: 'pattern', - }, - }, - }, - loggers: [ - { - name: 'root', - appenders: ['file'], - level: 'info', - }, - { - name: 'savedobjects-service', - appenders: ['file'], - level: 'debug', - }, - ], - }, - }); - - await root.preboot(); - - return root; -} - -// suite is very long, the 10mins default can cause timeouts -jest.setTimeout(15 * 60 * 1000); - -describe('migration v2', () => { - let esServer: TestElasticsearchUtils; - let rootA: Root; - let rootB: Root; - let rootC: Root; - - const migratedIndexAlias = `.kibana_${pkg.version}`; - const fooType: SavedObjectsType = { - name: 'foo', - hidden: false, - mappings: { properties: { status: { type: 'text' } } }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => { - if (doc.attributes?.status) { - doc.attributes.status = doc.attributes.status.replace('unmigrated', 'migrated'); - } - return doc; - }, - }, - }; - - const delay = (timeInMs: number) => new Promise((resolve) => setTimeout(resolve, timeInMs)); - - beforeEach(async () => { - await removeLogFiles(); - - rootA = await createRoot({ - logFileName: Path.join(__dirname, `${LOG_FILE_PREFIX}_A.log`), - }); - rootB = await createRoot({ - logFileName: Path.join(__dirname, `${LOG_FILE_PREFIX}_B.log`), - }); - rootC = await createRoot({ - logFileName: Path.join(__dirname, `${LOG_FILE_PREFIX}_C.log`), - }); - - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // original SOs: 5k of `foo` docs with this structure: - // [ - // { id: 'foo:1', type: 'foo', foo: { status: 'unmigrated' }, migrationVersion: { foo: '7.13.0' } }, - // { id: 'foo:2', type: 'foo', foo: { status: 'unmigrated' }, migrationVersion: { foo: '7.13.0' } }, - // { id: 'foo:3', type: 'foo', foo: { status: 'unmigrated' }, migrationVersion: { foo: '7.13.0' } }, - // ]; - dataArchive: Path.join(__dirname, '..', 'archives', '7.13.0_concurrent_5k_foo.zip'), - }, - }, - }); - esServer = await startES(); - }); - - afterEach(async () => { - try { - await Promise.all([rootA.shutdown(), rootB.shutdown(), rootC.shutdown()]); - } catch (e) { - /* trap */ - } - - if (esServer) { - await esServer.stop(); - } - }); - - const startWithDelay = async (instances: Root[], delayInSec: number) => { - const promises: Array> = []; - const errors: string[] = []; - for (let i = 0; i < instances.length; i++) { - promises.push( - instances[i].start().catch((err) => { - errors.push(err.message); - }) - ); - if (i < instances.length - 2) { - // We wait between instances, but not after the last one - await delay(delayInSec * 1000); - } - } - await Promise.all(promises); - if (errors.length) { - throw new Error(`Failed to start all instances: ${errors.join(',')}`); - } - }; - - it('migrates saved objects normally when multiple Kibana instances are started at the same time', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 0); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); - - it('migrates saved objects normally when multiple Kibana instances are started with a small interval', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 1); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); - - it('migrates saved objects normally when multiple Kibana instances are started with an average interval', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 5); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); - - it('migrates saved objects normally when multiple Kibana instances are started with a bigger interval', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 20); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); -}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts deleted file mode 100644 index 58dfb5d4e433e..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - type ISavedObjectTypeRegistry, - type SavedObjectsType, - MAIN_SAVED_OBJECT_INDEX, - ALL_SAVED_OBJECT_INDICES, - TASK_MANAGER_SAVED_OBJECT_INDEX, -} from '@kbn/core-saved-objects-server'; -import { DEFAULT_INDEX_TYPES_MAP } from '@kbn/core-saved-objects-base-server-internal'; -import { - clearLog, - startElasticsearch, - getKibanaMigratorTestKit, - getCurrentVersionTypeRegistry, - overrideTypeRegistry, - getAggregatedTypesCount, - currentVersion, - type KibanaMigratorTestKit, - getEsClient, -} from '../kibana_migrator_test_kit'; -import { delay, parseLogFile } from '../test_utils'; -import '../jest_matchers'; - -// define a type => index distribution -const RELOCATE_TYPES: Record = { - dashboard: '.kibana_so_ui', - visualization: '.kibana_so_ui', - 'canvas-workpad': '.kibana_so_ui', - search: '.kibana_so_search', - task: '.kibana_task_manager', - // the remaining types will be forced to go to '.kibana', - // overriding `indexPattern: foo` defined in the registry -}; - -const PARALLEL_MIGRATORS = 6; -export const logFilePathFirstRun = Path.join(__dirname, 'dot_kibana_split_1st_run.test.log'); -export const logFilePathSecondRun = Path.join(__dirname, 'dot_kibana_split_2nd_run.test.log'); - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('split .kibana index into multiple system indices', () => { - let esServer: TestElasticsearchUtils['es']; - let typeRegistry: ISavedObjectTypeRegistry; - - beforeAll(async () => { - typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); - }); - - beforeEach(async () => { - await clearLog(logFilePathFirstRun); - await clearLog(logFilePathSecondRun); - }); - - describe('when migrating from a legacy version', () => { - let migratorTestKitFactory: (logFilePath: string) => Promise; - - beforeAll(async () => { - esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'), - timeout: 60000, - }); - }); - - it('performs v1 migration and then relocates saved objects into different indices, depending on their types', async () => { - const updatedTypeRegistry = overrideTypeRegistry( - typeRegistry, - (type: SavedObjectsType) => { - return { - ...type, - indexPattern: RELOCATE_TYPES[type.name] ?? MAIN_SAVED_OBJECT_INDEX, - }; - } - ); - - migratorTestKitFactory = (logFilePath: string) => - getKibanaMigratorTestKit({ - types: updatedTypeRegistry.getAllTypes(), - kibanaIndex: '.kibana', - logFilePath, - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, - }); - - const { runMigrations, client } = await migratorTestKitFactory(logFilePathFirstRun); - - // count of types in the legacy index - expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({ - 'canvas-workpad': 3, - config: 1, - dashboard: 3, - 'index-pattern': 3, - map: 3, - 'maps-telemetry': 1, - 'sample-data-telemetry': 3, - search: 2, - telemetry: 1, - space: 1, - visualization: 39, - }); - - await runMigrations(); - - await client.indices.refresh({ - index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'], - }); - - expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({ - 'index-pattern': 3, - map: 3, - 'sample-data-telemetry': 3, - config: 1, - telemetry: 1, - space: 1, - }); - expect(await getAggregatedTypesCount(client, '.kibana_so_search')).toEqual({ - search: 2, - }); - expect(await getAggregatedTypesCount(client, '.kibana_so_ui')).toEqual({ - visualization: 39, - 'canvas-workpad': 3, - dashboard: 3, - }); - - const indicesInfo = await client.indices.get({ index: '.kibana*' }); - expect(indicesInfo[`.kibana_${currentVersion}_001`]).toEqual( - expect.objectContaining({ - aliases: expect.objectContaining({ '.kibana': expect.any(Object) }), - mappings: { - dynamic: 'strict', - _meta: { - mappingVersions: expect.any(Object), - indexTypesMap: expect.any(Object), - }, - properties: expect.any(Object), - }, - settings: { index: expect.any(Object) }, - }) - ); - - expect(indicesInfo[`.kibana_so_search_${currentVersion}_001`]).toEqual( - expect.objectContaining({ - aliases: expect.objectContaining({ '.kibana_so_search': expect.any(Object) }), - mappings: { - dynamic: 'strict', - _meta: { - mappingVersions: expect.any(Object), - indexTypesMap: expect.any(Object), - }, - properties: expect.any(Object), - }, - settings: { index: expect.any(Object) }, - }) - ); - - expect(indicesInfo[`.kibana_so_ui_${currentVersion}_001`]).toEqual( - expect.objectContaining({ - aliases: expect.objectContaining({ '.kibana_so_ui': expect.any(Object) }), - mappings: { - dynamic: 'strict', - _meta: { - mappingVersions: expect.any(Object), - indexTypesMap: expect.any(Object), - }, - properties: expect.any(Object), - }, - settings: { index: expect.any(Object) }, - }) - ); - - const typesMap = indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap; - - expect(Array.isArray(typesMap['.kibana'])).toEqual(true); - expect(typesMap['.kibana'].length > 50).toEqual(true); - expect(typesMap['.kibana'].includes('action')).toEqual(true); - expect(typesMap['.kibana'].includes('cases')).toEqual(true); - expect(typesMap['.kibana_so_search']).toEqual(['search']); - expect(typesMap['.kibana_so_ui']).toEqual(['canvas-workpad', 'dashboard', 'visualization']); - expect(typesMap['.kibana_task_manager']).toEqual(['task']); - - const logs = await parseLogFile(logFilePathFirstRun); - - expect(logs).toContainLogEntries( - [ - // .kibana_task_manager index exists and has no aliases => LEGACY_* migration path - '[.kibana_task_manager] INIT -> LEGACY_CHECK_CLUSTER_ROUTING_ALLOCATION.', - '[.kibana_task_manager] LEGACY_CHECK_CLUSTER_ROUTING_ALLOCATION -> LEGACY_SET_WRITE_BLOCK.', - '[.kibana_task_manager] LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_DELETE.', - '[.kibana_task_manager] LEGACY_DELETE -> SET_SOURCE_WRITE_BLOCK.', - '[.kibana_task_manager] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.', - '[.kibana_task_manager] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.', - '[.kibana_task_manager] CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK.', - '[.kibana_task_manager] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.', - '[.kibana_task_manager] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.', - '[.kibana_task_manager] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.', - '[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.', - '[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.', - '[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.', - '[.kibana_task_manager] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.', - '[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.', - '[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.', - '[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.', - '[.kibana_task_manager] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.', - '[.kibana_task_manager] MARK_VERSION_INDEX_READY -> DONE.', - '[.kibana_task_manager] Migration completed after', - ], - { ordered: true } - ); - - expect(logs).not.toContainLogEntries([ - // .kibana_task_manager migrator is NOT involved in relocation, must not sync with other migrators - '[.kibana_task_manager] READY_TO_REINDEX_SYNC', - '[.kibana_task_manager] DONE_REINDEXING_SYNC', - // .kibana_task_manager migrator performed a REINDEX migration, it must update ALL types - '[.kibana_task_manager] Kibana is performing a compatible update and it will update the following SO types so that ES can pickup the updated mappings', - ]); - - // new indices migrators did not exist, so they all have to reindex (create temp index + sync) - ['.kibana_so_ui', '.kibana_so_search'].forEach((newIndex) => { - expect(logs).toContainLogEntries( - [ - `[${newIndex}] INIT -> CREATE_REINDEX_TEMP.`, - `[${newIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`, - // no docs to reindex, as source index did NOT exist - `[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`, - ], - { ordered: true } - ); - }); - - // the .kibana migrator is involved in a relocation, it must also reindex - expect(logs).toContainLogEntries( - [ - '[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.', - '[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_CLUSTER_ROUTING_ALLOCATION.', - '[.kibana] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.', - '[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.', - '[.kibana] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.', - '[.kibana] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.', - '[.kibana] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.', - '[.kibana] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana] Starting to process 59 documents.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana] Processed 59 documents out of 59.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.', - ], - { ordered: true } - ); - - // after .kibana migrator is done relocating documents - // the 3 migrators share the final part of the flow - ['.kibana', '.kibana_so_ui', '.kibana_so_search'].forEach((index) => { - expect(logs).toContainLogEntries( - [ - `[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`, - `[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`, - `[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`, - `[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`, - `[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`, - `[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`, - `[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.`, - `[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`, - `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`, - `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`, - `[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`, - `[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY_SYNC.`, - `[${index}] MARK_VERSION_INDEX_READY_SYNC`, // all migrators try to update all aliases, all but one will have conclicts - `[${index}] Migration completed after`, - ], - { ordered: true } - ); - }); - - // should NOT retransform anything (we reindexed, thus we transformed already) - ['.kibana', '.kibana_task_manager', '.kibana_so_ui', '.kibana_so_search'].forEach((index) => { - expect(logs).not.toContainLogEntry(`[${index}] OUTDATED_DOCUMENTS_TRANSFORM`); - expect(logs).not.toContainLogEntry( - `[${index}] Kibana is performing a compatible update and it will update the following SO types so that ES can pickup the updated mappings` - ); - }); - }); - - afterEach(async () => { - // we run the migrator again to ensure that the next time state is loaded everything still works as expected - const { runMigrations } = await migratorTestKitFactory(logFilePathSecondRun); - await runMigrations(); - const logs = await parseLogFile(logFilePathSecondRun); - expect(logs).not.toContainLogEntries(['REINDEX', 'CREATE', 'UPDATE_TARGET_MAPPINGS']); - }); - - afterAll(async () => { - await esServer?.stop(); - await delay(2); - }); - }); - - describe('when multiple Kibana migrators run in parallel', () => { - jest.setTimeout(1200000); - it('correctly migrates 7.7.2_xpack_100k_obj.zip archive', async () => { - esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'), - }); - const esClient = await getEsClient(); - - const breakdownBefore = await getAggregatedTypesCount(esClient, [ - MAIN_SAVED_OBJECT_INDEX, - TASK_MANAGER_SAVED_OBJECT_INDEX, - ]); - expect(breakdownBefore).toEqual({ - '.kibana': { - 'apm-telemetry': 1, - application_usage_transactional: 4, - config: 1, - dashboard: 52994, - 'index-pattern': 1, - 'maps-telemetry': 1, - search: 1, - space: 1, - 'ui-metric': 5, - visualization: 53004, - }, - '.kibana_task_manager': { - task: 5, - }, - }); - - for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { - await clearLog(Path.join(__dirname, `dot_kibana_split_instance_${i}.log`)); - } - - const testKits = await Promise.all( - new Array(PARALLEL_MIGRATORS).fill(true).map((_, index) => - getKibanaMigratorTestKit({ - settings: { - migrations: { - discardUnknownObjects: currentVersion, - discardCorruptObjects: currentVersion, - }, - }, - kibanaIndex: MAIN_SAVED_OBJECT_INDEX, - types: typeRegistry.getAllTypes(), - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, - logFilePath: Path.join(__dirname, `dot_kibana_split_instance_${index}.log`), - }) - ) - ); - - const results = await Promise.all(testKits.map((testKit) => testKit.runMigrations())); - expect( - results - .flat() - .every((result) => result.status === 'migrated' || result.status === 'patched') - ).toEqual(true); - - await esClient.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - - const breakdownAfter = await getAggregatedTypesCount(esClient, [ - MAIN_SAVED_OBJECT_INDEX, - TASK_MANAGER_SAVED_OBJECT_INDEX, - ]); - expect(breakdownAfter).toEqual({ - '.kibana': { - 'apm-telemetry': 1, - config: 1, - space: 1, - 'ui-metric': 5, - }, - '.kibana_alerting_cases': {}, - '.kibana_analytics': { - dashboard: 52994, - 'index-pattern': 1, - search: 1, - visualization: 53004, - }, - '.kibana_ingest': {}, - '.kibana_security_solution': {}, - '.kibana_task_manager': { - task: 5, - }, - '.kibana_usage_counters': {}, - }); - }); - - afterEach(async () => { - await esServer?.stop(); - await delay(2); - }); - }); -}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts index 358ceea5c006a..81ef3bd237d6d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts @@ -15,33 +15,19 @@ import { startElasticsearch, defaultKibanaIndex, defaultKibanaTaskIndex, - getKibanaMigratorTestKit, } from '../kibana_migrator_test_kit'; import { delay } from '../test_utils'; import '../jest_matchers'; import { getElasticsearchClientWrapperFactory } from '../elasticsearch_client_wrapper'; import { BASELINE_TEST_ARCHIVE_1K } from '../kibana_migrator_archive_utils'; import { - baselineIndexTypesMap, - getReindexingBaselineTypes, + getRelocatingMigratorTestKit, + kibanaSplitIndex, } from '../kibana_migrator_test_kit.fixtures'; export const logFilePathFirstRun = join(__dirname, 'single_migrator_failures_1st_run.test.log'); export const logFilePathSecondRun = join(__dirname, 'single_migrator_failures_2nd_run.test.log'); -const kibanaSplitIndex = `${defaultKibanaIndex}_split`; -const tasksToNewIndex = getReindexingBaselineTypes(true).map((type) => { - if (type.name !== 'task' && type.name !== 'simple') { - return type; - } - - // relocate 'simple' and 'task' objects to a new index (forces reindex) - return { - ...type, - indexPattern: kibanaSplitIndex, - }; -}); - describe('split .kibana index into multiple system indices', () => { let esServer: TestElasticsearchUtils['es']; @@ -65,19 +51,7 @@ describe('split .kibana index into multiple system indices', () => { errorDelaySeconds: delaySeconds, }); - return await getKibanaMigratorTestKit({ - types: tasksToNewIndex, - logFilePath, - kibanaVersion: nextMinor, - defaultIndexTypesMap: baselineIndexTypesMap, - clientWrapperFactory, - settings: { - migrations: { - discardUnknownObjects: nextMinor, - discardCorruptObjects: nextMinor, - }, - }, - }); + return await getRelocatingMigratorTestKit({ logFilePath, clientWrapperFactory }); }; beforeEach(async () => { @@ -242,17 +216,8 @@ describe('split .kibana index into multiple system indices', () => { }); afterEach(async () => { - const { runMigrations: secondRun } = await getKibanaMigratorTestKit({ + const { runMigrations: secondRun } = await getRelocatingMigratorTestKit({ logFilePath: logFilePathSecondRun, - types: tasksToNewIndex, - kibanaVersion: nextMinor, - defaultIndexTypesMap: baselineIndexTypesMap, - settings: { - migrations: { - discardUnknownObjects: nextMinor, - discardCorruptObjects: nextMinor, - }, - }, }); const results = await secondRun(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts index 3e1fa8664fbfd..e3a0543dcf221 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts @@ -29,6 +29,10 @@ import { baselineTypes, getBaselineDocuments } from './kibana_migrator_test_kit. export const BASELINE_ELASTICSEARCH_VERSION = currentVersion; export const BASELINE_DOCUMENTS_PER_TYPE_1K = 200; export const BASELINE_DOCUMENTS_PER_TYPE_500K = 100_000; +// we discard the second half with exclude on upgrade (firstHalf !== true) +// then we discard half all multiples of 100 (1% of them) +export const BASELINE_COMPLEX_DOCUMENTS_500K_AFTER = + BASELINE_DOCUMENTS_PER_TYPE_500K / 2 - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 / 100; export const BASELINE_TEST_ARCHIVE_1K = join( __dirname, diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.expect.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.expect.ts new file mode 100644 index 0000000000000..2b220aca150e9 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.expect.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { baselineTypes } from './kibana_migrator_test_kit.fixtures'; + +export async function expectDocumentsMigratedToHighestVersion( + client: ElasticsearchClient, + index: string | string[] +) { + const typeMigrationVersions: Record = { + basic: '10.1.0', // did not define any model versions + server: '10.1.0', // did not define any model versions + deprecated: '10.1.0', // did not define any model versions + complex: '10.2.0', + task: '10.2.0', + }; + + const resultSets = await Promise.all( + baselineTypes.map(({ name: type }) => + client.search({ + index, + query: { + bool: { + should: [ + { + term: { type }, + }, + ], + }, + }, + }) + ) + ); + + const notUpgraded = resultSets + .flatMap((result) => result.hits.hits) + .find( + (document) => + document._source.typeMigrationVersion !== typeMigrationVersions[document._source.type] + ); + expect(notUpgraded).toBeUndefined(); +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts index 1ec300c075ff2..74fb34979366b 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts @@ -10,6 +10,7 @@ import type { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { ElasticsearchClientWrapperFactory } from './elasticsearch_client_wrapper'; import { currentVersion, defaultKibanaIndex, @@ -267,6 +268,7 @@ interface GetMutatedMigratorParams { filterDeprecated?: boolean; types?: Array>; settings?: Record; + clientWrapperFactory?: ElasticsearchClientWrapperFactory; } export const getUpToDateMigratorTestKit = async ({ @@ -304,12 +306,50 @@ export const getReindexingMigratorTestKit = async ({ filterDeprecated = false, types = getReindexingBaselineTypes(filterDeprecated), kibanaVersion = nextMinor, + clientWrapperFactory, settings = {}, }: GetMutatedMigratorParams = {}) => { return await getKibanaMigratorTestKit({ logFilePath, types, kibanaVersion, + clientWrapperFactory, + settings: { + ...settings, + migrations: { + discardUnknownObjects: nextMinor, + discardCorruptObjects: nextMinor, + ...settings.migrations, + }, + }, + }); +}; + +export const kibanaSplitIndex = `${defaultKibanaIndex}_split`; +export const getRelocatingMigratorTestKit = async ({ + logFilePath = defaultLogFilePath, + filterDeprecated = false, + types = getReindexingBaselineTypes(filterDeprecated).map((type) => { + if (type.name !== 'task' && type.name !== 'basic') { + return type; + } + + // relocate 'basic' and 'task' objects to a new index (forces reindex) + return { + ...type, + indexPattern: kibanaSplitIndex, + }; + }), + kibanaVersion = nextMinor, + clientWrapperFactory, + settings = {}, +}: GetMutatedMigratorParams = {}) => { + return await getKibanaMigratorTestKit({ + logFilePath, + types, + kibanaVersion, + clientWrapperFactory, + defaultIndexTypesMap: baselineIndexTypesMap, settings: { ...settings, migrations: {