diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts index c2f5f18a07e9b..1238060ffffec 100644 --- a/packages/kbn-es-archiver/src/actions/load.ts +++ b/packages/kbn-es-archiver/src/actions/load.ts @@ -36,6 +36,7 @@ import { migrateKibanaIndex, Progress, createDefaultSpace, + isKibanaIndex, } from '../lib'; // pipe a series of streams into each other so that data and errors @@ -107,7 +108,7 @@ export async function loadAction({ }); // If we affected the Kibana index, we need to ensure it's migrated... - if (Object.keys(result).some((k) => k.startsWith('.kibana'))) { + if (Object.keys(result).some(isKibanaIndex)) { await migrateKibanaIndex({ client, kbnClient }); if (kibanaPluginIds.includes('spaces')) { diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts index 87c166fe275cc..9be9197c01f28 100644 --- a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts @@ -21,6 +21,7 @@ import { Transform } from 'stream'; import { Client, SearchParams, SearchResponse } from 'elasticsearch'; import { Stats } from '../stats'; import { Progress } from '../progress'; +import { isKibanaIndex } from '../indices'; const SCROLL_SIZE = 1000; const SCROLL_TIMEOUT = '1m'; @@ -73,7 +74,7 @@ export function createGenerateDocRecordsStream({ value: { // always rewrite the .kibana_* index to .kibana_1 so that // when it is loaded it can skip migration, if possible - index: hit._index.startsWith('.kibana') ? '.kibana_1' : hit._index, + index: isKibanaIndex(hit._index) ? '.kibana_1' : hit._index, type: hit._type, id: hit._id, source: hit._source, diff --git a/packages/kbn-es-archiver/src/lib/index.ts b/packages/kbn-es-archiver/src/lib/index.ts index 960d51e411859..9d4b0cf0b0c2b 100644 --- a/packages/kbn-es-archiver/src/lib/index.ts +++ b/packages/kbn-es-archiver/src/lib/index.ts @@ -26,6 +26,7 @@ export { deleteKibanaIndices, migrateKibanaIndex, createDefaultSpace, + isKibanaIndex, } from './indices'; export { createFilterRecordsStream } from './records'; diff --git a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts index fa4c95dc73166..2fe63f5e67ed1 100644 --- a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts @@ -24,7 +24,7 @@ import { Client } from 'elasticsearch'; import { ToolingLog } from '@kbn/dev-utils'; import { Stats } from '../stats'; -import { deleteKibanaIndices } from './kibana_index'; +import { deleteKibanaIndices, isKibanaIndex } from './kibana_index'; import { deleteIndex } from './delete_index'; interface DocRecord { @@ -65,7 +65,7 @@ export function createCreateIndexStream({ async function handleIndex(record: DocRecord) { const { index, settings, mappings, aliases } = record.value; - const isKibana = index.startsWith('.kibana'); + const isKibana = isKibanaIndex(index); async function attemptToCreate(attemptNumber = 1) { try { @@ -86,30 +86,41 @@ export function createCreateIndexStream({ stats.createdIndex(index, { settings }); } catch (err) { - if ( - err?.body?.error?.reason?.includes('index exists with the same name as the alias') && - attemptNumber < 3 - ) { - kibanaIndexAlreadyDeleted = false; + const nameAlreadyInUse = err?.body?.error?.reason?.includes( + 'index exists with the same name as the alias' + ); + const indexExists = err?.body?.error?.type !== 'resource_already_exists_exception'; + + if (nameAlreadyInUse || indexExists) { const aliasStr = inspect(aliases); - log.info( - `failed to create aliases [${aliasStr}] because ES indicated an index/alias already exists, trying again` + const errStr = inspect(err?.body?.error ?? ''); + log.error( + `failed to create aliases [${aliasStr}] because ES indicated an index/alias already exists: ${errStr}` ); - await attemptToCreate(attemptNumber + 1); - return; } - if (err?.body?.error?.type !== 'resource_already_exists_exception' || attemptNumber >= 3) { + // bail if we have already done three attempts, or the error is not expected + if (attemptNumber >= 3 || !(nameAlreadyInUse || indexExists)) { throw err; } - if (skipExisting) { + // we empty Kibana indexes weirdly, see https://github.com/elastic/kibana/pull/71910 + if (isKibana && nameAlreadyInUse) { + kibanaIndexAlreadyDeleted = false; + await attemptToCreate(attemptNumber + 1); + return; + } + + // if the concrete index exists and we're skipping existing, move on + if (indexExists && skipExisting) { skipDocsFromIndices.add(index); stats.skippedIndex(index); return; } + // request index delete and try again await deleteIndex({ client, stats, index, log }); + log.error(`Attempting to create [${index}] again`); await attemptToCreate(attemptNumber + 1); return; } diff --git a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts index b4e1e530e1f84..d45c89339d185 100644 --- a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts @@ -23,7 +23,7 @@ import { ToolingLog } from '@kbn/dev-utils'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; -import { cleanKibanaIndices } from './kibana_index'; +import { cleanKibanaIndices, isKibanaIndex } from './kibana_index'; export function createDeleteIndexStream( client: Client, @@ -39,7 +39,7 @@ export function createDeleteIndexStream( if (!record || record.type === 'index') { const { index } = record.value; - if (index.startsWith('.kibana')) { + if (isKibanaIndex(index)) { await cleanKibanaIndices({ client, stats, log, kibanaPluginIds }); } else { await deleteIndex({ client, stats, log, index }); diff --git a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts index 07ee1420741c9..2bb2af71879c2 100644 --- a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts @@ -20,6 +20,7 @@ import { Transform } from 'stream'; import { Client } from 'elasticsearch'; import { Stats } from '../stats'; +import { isKibanaIndex } from './kibana_index'; export function createGenerateIndexRecordsStream(client: Client, stats: Stats) { return new Transform({ @@ -55,7 +56,7 @@ export function createGenerateIndexRecordsStream(client: Client, stats: Stats) { value: { // always rewrite the .kibana_* index to .kibana_1 so that // when it is loaded it can skip migration, if possible - index: index.startsWith('.kibana') ? '.kibana_1' : index, + index: isKibanaIndex(index) ? '.kibana_1' : index, settings, mappings, aliases, diff --git a/packages/kbn-es-archiver/src/lib/indices/index.ts b/packages/kbn-es-archiver/src/lib/indices/index.ts index 289ac87feb9a5..8356bc9560785 100644 --- a/packages/kbn-es-archiver/src/lib/indices/index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/index.ts @@ -20,4 +20,9 @@ export { createCreateIndexStream } from './create_index_stream'; export { createDeleteIndexStream } from './delete_index_stream'; export { createGenerateIndexRecordsStream } from './generate_index_records_stream'; -export { migrateKibanaIndex, deleteKibanaIndices, createDefaultSpace } from './kibana_index'; +export { + migrateKibanaIndex, + deleteKibanaIndices, + createDefaultSpace, + isKibanaIndex, +} from './kibana_index'; diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 3599911735b8d..f0ca581d61617 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -78,6 +78,8 @@ export async function migrateKibanaIndex({ await kbnClient.savedObjects.migrate(); } +export const isKibanaIndex = (index: string) => /^\.kibana(:?_\d*)?$/.test(index); + /** * Migrations mean that the Kibana index will look something like: * .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting @@ -86,7 +88,6 @@ export async function migrateKibanaIndex({ */ async function fetchKibanaIndices(client: Client) { const kibanaIndices = await client.cat.indices({ index: '.kibana*', format: 'json' }); - const isKibanaIndex = (index: string) => /^\.kibana(:?_\d*)?$/.test(index); return kibanaIndices.map((x: { index: string }) => x.index).filter(isKibanaIndex); }