[esArchiver] retry delete/create for non-kibana indexes which conflict with aliases

packages/kbn-es-archiver/src/actions/load.ts

@@ -36,6 +36,7 @@ import {
   migrateKibanaIndex,
   Progress,
   createDefaultSpace,
+  isKibanaIndex,
 } from '../lib';
 
 // pipe a series of streams into each other so that data and errors
@@ -107,7 +108,7 @@ export async function loadAction({
   });
 
   // If we affected the Kibana index, we need to ensure it's migrated...
-  if (Object.keys(result).some((k) => k.startsWith('.kibana'))) {
+  if (Object.keys(result).some(isKibanaIndex)) {
     await migrateKibanaIndex({ client, kbnClient });
 
     if (kibanaPluginIds.includes('spaces')) {

packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts

@@ -21,6 +21,7 @@ import { Transform } from 'stream';
 import { Client, SearchParams, SearchResponse } from 'elasticsearch';
 import { Stats } from '../stats';
 import { Progress } from '../progress';
+import { isKibanaIndex } from '../indices';
 
 const SCROLL_SIZE = 1000;
 const SCROLL_TIMEOUT = '1m';
@@ -73,7 +74,7 @@ export function createGenerateDocRecordsStream({
               value: {
                 // always rewrite the .kibana_* index to .kibana_1 so that
                 // when it is loaded it can skip migration, if possible
-                index: hit._index.startsWith('.kibana') ? '.kibana_1' : hit._index,
+                index: isKibanaIndex(hit._index) ? '.kibana_1' : hit._index,
                 type: hit._type,
                 id: hit._id,
                 source: hit._source,

packages/kbn-es-archiver/src/lib/index.ts

@@ -26,6 +26,7 @@ export {
   deleteKibanaIndices,
   migrateKibanaIndex,
   createDefaultSpace,
+  isKibanaIndex,
 } from './indices';
 
 export { createFilterRecordsStream } from './records';

packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts

@@ -24,7 +24,7 @@ import { Client } from 'elasticsearch';
 import { ToolingLog } from '@kbn/dev-utils';
 
 import { Stats } from '../stats';
-import { deleteKibanaIndices } from './kibana_index';
+import { deleteKibanaIndices, isKibanaIndex } from './kibana_index';
 import { deleteIndex } from './delete_index';
 
 interface DocRecord {
@@ -65,7 +65,7 @@ export function createCreateIndexStream({
 
   async function handleIndex(record: DocRecord) {
     const { index, settings, mappings, aliases } = record.value;
-    const isKibana = index.startsWith('.kibana');
+    const isKibana = isKibanaIndex(index);
 
     async function attemptToCreate(attemptNumber = 1) {
       try {
@@ -86,30 +86,41 @@ export function createCreateIndexStream({
 
         stats.createdIndex(index, { settings });
       } catch (err) {
-        if (
-          err?.body?.error?.reason?.includes('index exists with the same name as the alias') &&
-          attemptNumber < 3
-        ) {
-          kibanaIndexAlreadyDeleted = false;
+        const nameAlreadyInUse = err?.body?.error?.reason?.includes(
+          'index exists with the same name as the alias'
+        );
+        const indexExists = err?.body?.error?.type !== 'resource_already_exists_exception';
+
+        if (nameAlreadyInUse || indexExists) {
           const aliasStr = inspect(aliases);
-          log.info(
-            `failed to create aliases [${aliasStr}] because ES indicated an index/alias already exists, trying again`
+          const errStr = inspect(err?.body?.error ?? '<unknown error>');
+          log.error(
+            `failed to create aliases [${aliasStr}] because ES indicated an index/alias already exists: ${errStr}`
           );
-          await attemptToCreate(attemptNumber + 1);
-          return;
         }
 
-        if (err?.body?.error?.type !== 'resource_already_exists_exception' || attemptNumber >= 3) {
+        // bail if we have already done three attempts, or the error is not expected
+        if (attemptNumber >= 3 || !(nameAlreadyInUse || indexExists)) {
           throw err;
         }
 
-        if (skipExisting) {
+        // we empty Kibana indexes weirdly, see https://github.com/elastic/kibana/pull/71910
+        if (isKibana && nameAlreadyInUse) {
+          kibanaIndexAlreadyDeleted = false;
+          await attemptToCreate(attemptNumber + 1);
+          return;
+        }
+
+        // if the concrete index exists and we're skipping existing, move on
+        if (indexExists && skipExisting) {
           skipDocsFromIndices.add(index);
           stats.skippedIndex(index);
           return;
         }
 
+        // request index delete and try again
         await deleteIndex({ client, stats, index, log });
+        log.error(`Attempting to create [${index}] again`);
         await attemptToCreate(attemptNumber + 1);
         return;
       }

packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts

@@ -23,7 +23,7 @@ import { ToolingLog } from '@kbn/dev-utils';
 
 import { Stats } from '../stats';
 import { deleteIndex } from './delete_index';
-import { cleanKibanaIndices } from './kibana_index';
+import { cleanKibanaIndices, isKibanaIndex } from './kibana_index';
 
 export function createDeleteIndexStream(
   client: Client,
@@ -39,7 +39,7 @@ export function createDeleteIndexStream(
         if (!record || record.type === 'index') {
           const { index } = record.value;
 
-          if (index.startsWith('.kibana')) {
+          if (isKibanaIndex(index)) {
             await cleanKibanaIndices({ client, stats, log, kibanaPluginIds });
           } else {
             await deleteIndex({ client, stats, log, index });

packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts

@@ -20,6 +20,7 @@
 import { Transform } from 'stream';
 import { Client } from 'elasticsearch';
 import { Stats } from '../stats';
+import { isKibanaIndex } from './kibana_index';
 
 export function createGenerateIndexRecordsStream(client: Client, stats: Stats) {
   return new Transform({
@@ -55,7 +56,7 @@ export function createGenerateIndexRecordsStream(client: Client, stats: Stats) {
             value: {
               // always rewrite the .kibana_* index to .kibana_1 so that
               // when it is loaded it can skip migration, if possible
-              index: index.startsWith('.kibana') ? '.kibana_1' : index,
+              index: isKibanaIndex(index) ? '.kibana_1' : index,
               settings,
               mappings,
               aliases,

packages/kbn-es-archiver/src/lib/indices/index.ts

@@ -20,4 +20,9 @@
 export { createCreateIndexStream } from './create_index_stream';
 export { createDeleteIndexStream } from './delete_index_stream';
 export { createGenerateIndexRecordsStream } from './generate_index_records_stream';
-export { migrateKibanaIndex, deleteKibanaIndices, createDefaultSpace } from './kibana_index';
+export {
+  migrateKibanaIndex,
+  deleteKibanaIndices,
+  createDefaultSpace,
+  isKibanaIndex,
+} from './kibana_index';

packages/kbn-es-archiver/src/lib/indices/kibana_index.ts

@@ -78,6 +78,8 @@ export async function migrateKibanaIndex({
   await kbnClient.savedObjects.migrate();
 }
 
+export const isKibanaIndex = (index: string) => /^\.kibana(:?_\d*)?$/.test(index);
+
 /**
  * Migrations mean that the Kibana index will look something like:
  * .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting
@@ -86,7 +88,6 @@ export async function migrateKibanaIndex({
  */
 async function fetchKibanaIndices(client: Client) {
   const kibanaIndices = await client.cat.indices({ index: '.kibana*', format: 'json' });
-  const isKibanaIndex = (index: string) => /^\.kibana(:?_\d*)?$/.test(index);
   return kibanaIndices.map((x: { index: string }) => x.index).filter(isKibanaIndex);
 }