[Endpoint] Fix endpoint tests with data streams

src/es_archiver/lib/docs/index_doc_records_stream.ts

@@ -30,7 +30,7 @@ export function createIndexDocRecordsStream(client: Client, stats: Stats, progre
       stats.indexedDoc(doc.index);
       body.push(
         {
-          index: {
+          create: {
             _index: doc.index,
             _id: doc.id,
           },

x-pack/test/api_integration/apis/endpoint/alerts/index.ts

@@ -7,6 +7,7 @@ import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { AlertData } from '../../../../../plugins/security_solution/common/endpoint_alerts/types';
 import { AlertId } from '../../../../../plugins/security_solution/server/endpoint/alerts/handlers/lib/index';
+import { deleteEventsStream, deleteMetadataStream } from '../data_stream_helper';
 
 /**
  * The number of alert documents in the es archive.
@@ -66,26 +67,27 @@ export default function ({ getService }: FtrProviderContext) {
   const nextPrevPrefixOrder = 'order=desc';
   const nextPrevPrefixPageSize = 'page_size=10';
   const nextPrevPrefix = `${nextPrevPrefixQuery}&${nextPrevPrefixDateRange}&${nextPrevPrefixSort}&${nextPrevPrefixOrder}&${nextPrevPrefixPageSize}`;
-  const alertIndex = 'events-endpoint-1';
+  const alertIndex = '.ds-events-endpoint-1-000001';
 
   let nullableEventId = '';
 
-  // SKIPPED as it is failing ES PROMOTION: https://github.com/elastic/kibana/issues/68613
-  describe.skip('Endpoint alert API', () => {
+  describe('Endpoint alert API', () => {
     describe('when data is in elasticsearch', () => {
       before(async () => {
         await esArchiver.load('endpoint/alerts/api_feature');
         await esArchiver.load('endpoint/alerts/host_api_feature');
         const res = await es.search({
-          index: alertIndex,
+          index: 'events-endpoint-*',
           body: ES_QUERY_MISSING,
         });
         nullableEventId = res.hits.hits[0]._source.event.id;
       });
 
       after(async () => {
-        await esArchiver.unload('endpoint/alerts/api_feature');
-        await esArchiver.unload('endpoint/alerts/host_api_feature');
+        // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+        // to do it manually
+        await deleteEventsStream(getService);
+        await deleteMetadataStream(getService);
       });
 
       it('should not support POST requests', async () => {

x-pack/test/api_integration/apis/endpoint/data_stream_helper.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
+  const client = getService('es');
+  await client.transport.request(
+    {
+      method: 'DELETE',
+      path: `_data_stream/${index}`,
+    },
+    {
+      ignore: [404],
+    }
+  );
+}
+
+export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'metrics-endpoint.metadata-*');
+}
+
+export async function deleteEventsStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'events-endpoint-*');
+}
+
+export async function deletePolicyStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'metrics-endpoint.policy-*');
+}

x-pack/test/api_integration/apis/endpoint/metadata.ts

@@ -5,6 +5,7 @@
  */
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { deleteMetadataStream } from './data_stream_helper';
 
 /**
  * The number of host documents in the es archive.
@@ -14,11 +15,12 @@ const numberOfHostsInFixture = 3;
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  // SKIPPED as it is failing on ES PROMOTION: https://github.com/elastic/kibana/issues/68584
-  describe.skip('test metadata api', () => {
+  describe('test metadata api', () => {
     describe('POST /api/endpoint/metadata when index is empty', () => {
       it('metadata api should return empty result when index is empty', async () => {
-        await esArchiver.unload('endpoint/metadata/api_feature');
+        // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+        // to do it manually
+        await deleteMetadataStream(getService);
         const { body } = await supertest
           .post('/api/endpoint/metadata')
           .set('kbn-xsrf', 'xxx')
@@ -32,8 +34,10 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     describe('POST /api/endpoint/metadata when index is not empty', () => {
-      before(() => esArchiver.load('endpoint/metadata/api_feature'));
-      after(() => esArchiver.unload('endpoint/metadata/api_feature'));
+      before(async () => await esArchiver.load('endpoint/metadata/api_feature'));
+      // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+      // to do it manually
+      after(async () => await deleteMetadataStream(getService));
       it('metadata api should return one entry for each host with default paging', async () => {
         const { body } = await supertest
           .post('/api/endpoint/metadata')

x-pack/test/api_integration/apis/endpoint/policy.ts

@@ -6,16 +6,18 @@
 
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { deletePolicyStream } from './data_stream_helper';
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  // SKIPPED as it is failing ES PROMOTION: https://github.com/elastic/kibana/issues/68638
-  describe.skip('Endpoint policy api', () => {
+  describe('Endpoint policy api', () => {
     describe('GET /api/endpoint/policy_response', () => {
       before(async () => await esArchiver.load('endpoint/policy'));
 
-      after(async () => await esArchiver.unload('endpoint/policy'));
+      // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+      // to do it manually
+      after(async () => await deletePolicyStream(getService));
 
       it('should return one policy response for host', async () => {
         const expectedHostId = '4f3b9858-a96d-49d8-a326-230d7763d767';

x-pack/test/api_integration/apis/endpoint/resolver.ts

@@ -216,6 +216,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC
     });
     after(async () => {
       await resolver.deleteTrees(resolverTrees);
+      // this unload is for an endgame-* index so it does not use data streams
       await esArchiver.unload('endpoint/resolver/api_feature');
     });