Migrations v2 ignore fleet agent events (#96690)

* migrationsv2: ignore fleet agent events and tsvb telemetry

* migrationsv1: ignore tsvb-validation-telemetry

* Skip fleet test that depends on fleet-agent-events

* Fix typescript errors

Co-authored-by: Kibana Machine <[email protected]>
# Conflicts:
#	src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts

src/core/server/saved_objects/migrations/core/elastic_index.test.ts

@@ -436,11 +436,18 @@ describe('ElasticIndex', () => {
           size: 100,
           query: {
             bool: {
-              must_not: {
-                term: {
-                  type: 'fleet-agent-events',
+              must_not: [
+                {
+                  term: {
+                    type: 'fleet-agent-events',
+                  },
                 },
-              },
+                {
+                  term: {
+                    type: 'tsvb-validation-telemetry',
+                  },
+                },
+              ],
             },
           },
         },

src/core/server/saved_objects/migrations/core/elastic_index.ts

@@ -68,16 +68,19 @@ export function reader(
   let scrollId: string | undefined;
 
   // When migrating from the outdated index we use a read query which excludes
-  // saved objects which are no longer used. These saved objects will still be
-  // kept in the outdated index for backup purposes, but won't be availble in
-  // the upgraded index.
-  const excludeUnusedTypes = {
+  // saved object types which are no longer used. These saved objects will
+  // still be kept in the outdated index for backup purposes, but won't be
+  // availble in the upgraded index.
+  const EXCLUDE_UNUSED_TYPES = [
+    'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869
+    'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617
+  ];
+
+  const excludeUnusedTypesQuery = {
     bool: {
-      must_not: {
-        term: {
-          type: 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869
-        },
-      },
+      must_not: EXCLUDE_UNUSED_TYPES.map((type) => ({
+        term: { type },
+      })),
     },
   };
 
@@ -90,7 +93,7 @@ export function reader(
       : client.search<SearchResponse<SavedObjectsRawDocSource>>({
           body: {
             size: batchSize,
-            query: excludeUnusedTypes,
+            query: excludeUnusedTypesQuery,
           },
           index,
           scroll,

src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts

@@ -321,7 +321,7 @@ describe('KibanaMigrator', () => {
         options.client.tasks.get.mockReturnValue(
           elasticsearchClientMock.createSuccessTransportRequestPromise({
             completed: true,
-            error: { type: 'elatsicsearch_exception', reason: 'task failed with an error' },
+            error: { type: 'elasticsearch_exception', reason: 'task failed with an error' },
             failures: [],
             task: { description: 'task description' },
           })
@@ -331,11 +331,11 @@ describe('KibanaMigrator', () => {
         migrator.prepareMigrations();
         await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(`
                 [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error:
-                {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}]
+                {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}]
               `);
         expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(`
           [Error: Reindex failed with the following error:
-          {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}]
+          {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}]
         `);
       });
     });

src/core/server/saved_objects/migrationsv2/actions/index.test.ts

@@ -85,7 +85,8 @@ describe('actions', () => {
         'my_source_index',
         'my_target_index',
         Option.none,
-        false
+        false,
+        Option.none
       );
       try {
         await task();

src/core/server/saved_objects/migrationsv2/actions/index.ts

@@ -13,6 +13,7 @@ import { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/
 import { pipe } from 'fp-ts/lib/pipeable';
 import { errors as EsErrors } from '@elastic/elasticsearch';
 import { flow } from 'fp-ts/lib/function';
+import { QueryContainer } from '@elastic/eui/src/components/search_bar/query/ast_to_es_query_dsl';
 import { ElasticsearchClient } from '../../../elasticsearch';
 import { IndexMapping } from '../../mappings';
 import { SavedObjectsRawDoc } from '../../serialization';
@@ -436,7 +437,12 @@ export const reindex = (
   sourceIndex: string,
   targetIndex: string,
   reindexScript: Option.Option<string>,
-  requireAlias: boolean
+  requireAlias: boolean,
+  /* When reindexing we use a source query to exclude saved objects types which
+   * are no longer used. These saved objects will still be kept in the outdated
+   * index for backup purposes, but won't be availble in the upgraded index.
+   */
+  unusedTypesToExclude: Option.Option<string[]>
 ): TaskEither.TaskEither<RetryableEsClientError, ReindexResponse> => () => {
   return client
     .reindex({
@@ -451,6 +457,15 @@ export const reindex = (
           index: sourceIndex,
           // Set reindex batch size
           size: BATCH_SIZE,
+          // Exclude saved object types
+          query: Option.fold<string[], QueryContainer | undefined>(
+            () => undefined,
+            (types) => ({
+              bool: {
+                must_not: types.map((type) => ({ term: { type } })),
+              },
+            })
+          )(unusedTypesToExclude),
         },
         dest: {
           index: targetIndex,

src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts

@@ -66,7 +66,8 @@ describe('migration actions', () => {
       { _source: { title: 'doc 1' } },
       { _source: { title: 'doc 2' } },
       { _source: { title: 'doc 3' } },
-      { _source: { title: 'saved object 4' } },
+      { _source: { title: 'saved object 4', type: 'another_unused_type' } },
+      { _source: { title: 'f-agent-event 5', type: 'f_agent_event' } },
     ] as unknown) as SavedObjectsRawDoc[];
     await bulkOverwriteTransformedDocuments(client, 'existing_index_with_docs', sourceDocs)();
 
@@ -344,7 +345,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -365,6 +367,37 @@ describe('migration actions', () => {
           "doc 2",
           "doc 3",
           "saved object 4",
+          "f-agent-event 5",
+        ]
+      `);
+    });
+    it('resolves right and excludes all unusedTypesToExclude documents', async () => {
+      const res = (await reindex(
+        client,
+        'existing_index_with_docs',
+        'reindex_target_excluded_docs',
+        Option.none,
+        false,
+        Option.some(['f_agent_event', 'another_unused_type'])
+      )()) as Either.Right<ReindexResponse>;
+      const task = waitForReindexTask(client, res.right.taskId, '10s');
+      await expect(task()).resolves.toMatchInlineSnapshot(`
+        Object {
+          "_tag": "Right",
+          "right": "reindex_succeeded",
+        }
+      `);
+
+      const results = ((await searchForOutdatedDocuments(client, {
+        batchSize: 1000,
+        targetIndex: 'reindex_target_excluded_docs',
+        outdatedDocumentsQuery: undefined,
+      })()) as Either.Right<SearchResponse>).right.outdatedDocuments;
+      expect(results.map((doc) => doc._source.title)).toMatchInlineSnapshot(`
+        Array [
+          "doc 1",
+          "doc 2",
+          "doc 3",
         ]
       `);
     });
@@ -375,7 +408,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_2',
         Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -395,6 +429,7 @@ describe('migration actions', () => {
           "doc 2_updated",
           "doc 3_updated",
           "saved object 4_updated",
+          "f-agent-event 5_updated",
         ]
       `);
     });
@@ -406,7 +441,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_3',
         Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       let task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -422,7 +458,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_3',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -444,6 +481,7 @@ describe('migration actions', () => {
           "doc 2_updated",
           "doc 3_updated",
           "saved object 4_updated",
+          "f-agent-event 5_updated",
         ]
       `);
     });
@@ -470,7 +508,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_4',
         Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -492,6 +531,7 @@ describe('migration actions', () => {
           "doc 2",
           "doc 3_updated",
           "saved object 4_updated",
+          "f-agent-event 5_updated",
         ]
       `);
     });
@@ -518,7 +558,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_5',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, reindexTaskId, '10s');
 
@@ -552,7 +593,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_6',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, reindexTaskId, '10s');
 
@@ -572,7 +614,8 @@ describe('migration actions', () => {
         'no_such_index',
         'reindex_target',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -592,7 +635,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'existing_index_with_write_block',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
 
       const task = waitForReindexTask(client, res.right.taskId, '10s');
@@ -613,7 +657,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'existing_index_with_write_block',
         Option.none,
-        true
+        true,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
 
       const task = waitForReindexTask(client, res.right.taskId, '10s');
@@ -634,7 +679,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
 
       const task = waitForReindexTask(client, res.right.taskId, '0s');
@@ -660,7 +706,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_7',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       await waitForReindexTask(client, res.right.taskId, '10s')();
 
@@ -710,12 +757,11 @@ describe('migration actions', () => {
       )()) as Either.Right<SearchResponse>).right.outdatedDocuments;
       expect(resultsWithQuery.length).toBe(3);
 
-      const resultsWithoutQuery = ((await searchForOutdatedDocuments(
-        client,
-        'existing_index_with_docs',
-        undefined as any
-      )()) as Either.Right<SearchResponse>).right.outdatedDocuments;
-      expect(resultsWithoutQuery.length).toBe(4);
+      const resultsWithoutQuery = ((await searchForOutdatedDocuments(client, {
+        targetIndex: 'existing_index_with_docs',
+        outdatedDocumentsQuery: undefined,
+      })()) as Either.Right<SearchResponse>).right.outdatedDocuments;
+      expect(resultsWithoutQuery.length).toBe(5);
     });
     it('resolves with _id, _source, _seq_no and _primary_term', async () => {
       expect.assertions(1);

src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts

@@ -248,6 +248,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },
@@ -308,6 +315,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },
@@ -453,6 +467,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },
@@ -508,6 +529,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },