[S&R] Support data streams (#68078) (#70595)

* Sort endpoint responses into indices and datastreams

The server endpoint for policies now returns data streams and
filters out backing indices from the indices array it returned
previously

* Refactor indices switch and field out of the step settings file

* Fix indices field form behaviour

* WiP on UI. Added the second table per mockup for add and edit.

* add support for creating a policy that backs up data streams end to end

* wip on restore flow - added data streams to server response

* add logic for detecting whether an index is part of a data stream

* fix public jest tests

* fix server side jest tests

* pivot to different solution in UI while we do not have data streams nicely separated

* added data stream to snapshot summary details

* move the data streams badge file closer to where it used

* add data stream badge when restoring snapshots too

* update restore copy

* fix pattern specification in indices and data streams field

* first iteration of complete policy UX

* First iteration that is ready for review

Given the contraints on working with data streams and indices in policies
at the moment the simplest implementation is to just include data streams
with indices and have the user select them there for now.

The way snapshotting behaviour is currently implemented relies entirely
on what is specified inside of "indices", this is also where data
streams must be placed. This unfortunately means that capture patterns
defined in indices will capture entire data streams too.

* delete unused import

* fix type issue in tests

* added logic for rendering out previous selection as custom pattern

* refactor indices fields to make component smaller

* added CIT for data streams badge

* Data streams > indices

* updates to relevant pieces of copy

* more copy updates

* fix types and remove unused import

* removed backing indices from restore view

* Added data stream restore warning message

* restore CITs

* first round of copy feedback

* refactor help text to provide clearer feedback, for both restore and policy forms

* Restore updates

- added spacer between title and data streams callout
- added copy to the restore settings tab to indicate that settings
  also apply to backing indices

* further copy refinements

* second round of copy feedback

* fix i18n

* added comment to mock

* line spacing fixes and created issue for tracking backing index discovery in snaphots

* refactor collapsible list logic and tests

* refactor editing managed policy check

* refactor copy to be clearer about pluralisation of data streams

* refactor file structure in components for data stream badge

* added tests for indices and data streams field helper

* refactored types and fixed i18n id per guidelines

Co-authored-by: Elastic Machine <[email protected]>

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/index.ts

@@ -3,12 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
+import './mocks';
 import { setup as homeSetup } from './home.helpers';
 import { setup as repositoryAddSetup } from './repository_add.helpers';
 import { setup as repositoryEditSetup } from './repository_edit.helpers';
 import { setup as policyAddSetup } from './policy_add.helpers';
 import { setup as policyEditSetup } from './policy_edit.helpers';
+import { setup as restoreSnapshotSetup } from './restore_snapshot.helpers';
 
 export { nextTick, getRandomString, findTestSubject, TestBed } from '../../../../../test_utils';
 
@@ -20,4 +21,5 @@ export const pageHelpers = {
   repositoryEdit: { setup: repositoryEditSetup },
   policyAdd: { setup: policyAddSetup },
   policyEdit: { setup: policyEditSetup },
+  restoreSnapshot: { setup: restoreSnapshotSetup },
 };

x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/mocks.tsx

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+/*
+ * Mocking AutoSizer of the react-virtualized because it does not render children in JS DOM.
+ * This seems related to not being able to properly discover height and width.
+ */
+jest.mock('react-virtualized', () => {
+  const original = jest.requireActual('react-virtualized');
+
+  return {
+    ...original,
+    AutoSizer: ({ children }: { children: any }) => (
+      <div>{children({ height: 500, width: 500 })}</div>
+    ),
+  };
+});

x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/policy_form.helpers.ts

@@ -41,6 +41,8 @@ export type PolicyFormTestSubjects =
   | 'allIndicesToggle'
   | 'backButton'
   | 'deselectIndicesLink'
+  | 'allDataStreamsToggle'
+  | 'deselectDataStreamLink'
   | 'expireAfterValueInput'
   | 'expireAfterUnitSelect'
   | 'ignoreUnavailableIndicesToggle'
@@ -53,4 +55,5 @@ export type PolicyFormTestSubjects =
   | 'selectIndicesLink'
   | 'showAdvancedCronLink'
   | 'snapshotNameInput'
+  | 'dataStreamBadge'
   | 'submitButton';

x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/restore_snapshot.helpers.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @kbn/eslint/no-restricted-paths */
+import { registerTestBed, TestBed, TestBedConfig } from '../../../../../test_utils';
+import { RestoreSnapshot } from '../../../public/application/sections/restore_snapshot';
+import { WithAppDependencies } from './setup_environment';
+
+const testBedConfig: TestBedConfig = {
+  memoryRouter: {
+    initialEntries: ['/add_policy'],
+    componentRoutePath: '/add_policy',
+  },
+  doMountAsync: true,
+};
+
+const initTestBed = registerTestBed<RestoreSnapshotFormTestSubject>(
+  WithAppDependencies(RestoreSnapshot),
+  testBedConfig
+);
+
+const setupActions = (testBed: TestBed<RestoreSnapshotFormTestSubject>) => {
+  const { find } = testBed;
+  return {
+    findDataStreamCallout() {
+      return find('dataStreamWarningCallOut');
+    },
+  };
+};
+
+type Actions = ReturnType<typeof setupActions>;
+
+export type RestoreSnapshotTestBed = TestBed<RestoreSnapshotFormTestSubject> & {
+  actions: Actions;
+};
+
+export const setup = async (): Promise<RestoreSnapshotTestBed> => {
+  const testBed = await initTestBed();
+
+  return {
+    ...testBed,
+    actions: setupActions(testBed),
+  };
+};
+
+export type RestoreSnapshotFormTestSubject =
+  | 'snapshotRestoreStepLogistics'
+  | 'dataStreamWarningCallOut';

x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/setup_environment.tsx

@@ -64,6 +64,14 @@ export const setupEnvironment = () => {
   };
 };
 
+/**
+ * Suppress error messages about Worker not being available in JS DOM.
+ */
+(window as any).Worker = function Worker() {
+  this.postMessage = () => {};
+  this.terminate = () => {};
+};
+
 export const WithAppDependencies = (Comp: any) => (props: any) => (
   <AppContextProvider value={appDependencies as any}>
     <Comp {...props} />

x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_add.test.ts

@@ -3,11 +3,14 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
+// import helpers first, this also sets up the mocks
+import { setupEnvironment, pageHelpers, nextTick, getRandomString } from './helpers';
+
 import { act } from 'react-dom/test-utils';
 
 import * as fixtures from '../../test/fixtures';
 
-import { setupEnvironment, pageHelpers, nextTick, getRandomString } from './helpers';
 import { PolicyFormTestBed } from './helpers/policy_form.helpers';
 import { DEFAULT_POLICY_SCHEDULE } from '../../public/application/constants';
 
@@ -37,7 +40,10 @@ describe('<PolicyAdd />', () => {
   describe('on component mount', () => {
     beforeEach(async () => {
       httpRequestsMockHelpers.setLoadRepositoriesResponse({ repositories: [repository] });
-      httpRequestsMockHelpers.setLoadIndicesResponse({ indices: ['my_index'] });
+      httpRequestsMockHelpers.setLoadIndicesResponse({
+        indices: ['my_index'],
+        dataStreams: ['my_data_stream', 'my_other_data_stream'],
+      });
 
       testBed = await setup();
       await nextTick();
@@ -96,7 +102,7 @@ describe('<PolicyAdd />', () => {
           actions.clickNextButton();
         });
 
-        test('should require at least one index', async () => {
+        test('should require at least one index if no data streams are provided', async () => {
           const { find, form, component } = testBed;
 
           await act(async () => {
@@ -109,7 +115,22 @@ describe('<PolicyAdd />', () => {
           // Deselect all indices from list
           find('deselectIndicesLink').simulate('click');
 
-          expect(form.getErrorsMessages()).toEqual(['You must select at least one index.']);
+          expect(form.getErrorsMessages()).toEqual([
+            'You must select at least one data stream or index.',
+          ]);
+        });
+
+        test('should correctly indicate data streams with a badge', async () => {
+          const { find, component, form } = testBed;
+
+          await act(async () => {
+            // Toggle "All indices" switch
+            form.toggleEuiSwitch('allIndicesToggle', false);
+            await nextTick();
+          });
+          component.update();
+
+          expect(find('dataStreamBadge').length).toBe(2);
         });
       });
 

x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_edit.test.ts

@@ -37,7 +37,10 @@ describe('<PolicyEdit />', () => {
   describe('on component mount', () => {
     beforeEach(async () => {
       httpRequestsMockHelpers.setGetPolicyResponse({ policy: POLICY_EDIT });
-      httpRequestsMockHelpers.setLoadIndicesResponse({ indices: ['my_index'] });
+      httpRequestsMockHelpers.setLoadIndicesResponse({
+        indices: ['my_index'],
+        dataStreams: ['my_data_stream'],
+      });
       httpRequestsMockHelpers.setLoadRepositoriesResponse({
         repositories: [{ name: POLICY_EDIT.repository }],
       });

x-pack/plugins/snapshot_restore/__jest__/client_integration/restore_snapshot.test.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { nextTick, pageHelpers, setupEnvironment } from './helpers';
+import { RestoreSnapshotTestBed } from './helpers/restore_snapshot.helpers';
+import * as fixtures from '../../test/fixtures';
+
+const {
+  restoreSnapshot: { setup },
+} = pageHelpers;
+
+describe('<RestoreSnapshot />', () => {
+  const { server, httpRequestsMockHelpers } = setupEnvironment();
+  let testBed: RestoreSnapshotTestBed;
+
+  afterAll(() => {
+    server.restore();
+  });
+  describe('with data streams', () => {
+    beforeEach(async () => {
+      httpRequestsMockHelpers.setGetSnapshotResponse(fixtures.getSnapshot());
+      testBed = await setup();
+      await nextTick();
+      testBed.component.update();
+    });
+
+    it('shows the data streams warning when the snapshot has data streams', () => {
+      const { exists } = testBed;
+      expect(exists('dataStreamWarningCallOut')).toBe(true);
+    });
+  });
+
+  describe('without data streams', () => {
+    beforeEach(async () => {
+      httpRequestsMockHelpers.setGetSnapshotResponse(fixtures.getSnapshot({ totalDataStreams: 0 }));
+      testBed = await setup();
+      await nextTick();
+      testBed.component.update();
+    });
+
+    it('hides the data streams warning when the snapshot has data streams', () => {
+      const { exists } = testBed;
+      expect(exists('dataStreamWarningCallOut')).toBe(false);
+    });
+  });
+});

x-pack/plugins/snapshot_restore/common/lib/index.ts

@@ -16,3 +16,5 @@ export {
   serializeSnapshotRetention,
 } from './snapshot_serialization';
 export { deserializePolicy, serializePolicy } from './policy_serialization';
+export { csvToArray } from './utils';
+export { isDataStreamBackingIndex } from './is_data_stream_backing_index';

x-pack/plugins/snapshot_restore/common/lib/is_data_stream_backing_index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * @remark
+ * WARNING!
+ *
+ * This is a very hacky way of determining whether an index is a backing index.
+ *
+ * We only do this so that we can show users during a snapshot restore workflow
+ * that an index is part of a data stream. At the moment there is no way for us
+ * to get this information from the snapshot itself, even though it contains the
+ * metadata for the data stream that information is fully opaque to us until after
+ * we have done the snapshot restore.
+ *
+ * Issue for tracking this discussion here: https://github.com/elastic/elasticsearch/issues/58890
+ */
+export const isDataStreamBackingIndex = (indexName: string) => {
+  return indexName.startsWith('.ds');
+};

x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.test.ts

@@ -97,6 +97,7 @@ describe('deserializeSnapshotDetails', () => {
       version: 'version',
       // Indices are sorted.
       indices: ['index1', 'index2', 'index3'],
+      dataStreams: [],
       includeGlobalState: false,
       // Failures are grouped and sorted by index, and the failures themselves are sorted by shard.
       indexFailures: [

x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.ts

@@ -17,6 +17,8 @@ import {
 
 import { deserializeTime, serializeTime } from './time_serialization';
 
+import { csvToArray } from './utils';
+
 export function deserializeSnapshotDetails(
   repository: string,
   snapshotDetailsEs: SnapshotDetailsEs,
@@ -33,6 +35,7 @@ export function deserializeSnapshotDetails(
     version_id: versionId,
     version,
     indices = [],
+    data_streams: dataStreams = [],
     include_global_state: includeGlobalState,
     state,
     start_time: startTime,
@@ -77,6 +80,7 @@ export function deserializeSnapshotDetails(
     versionId,
     version,
     indices: [...indices].sort(),
+    dataStreams: [...dataStreams].sort(),
     includeGlobalState,
     state,
     startTime,
@@ -127,8 +131,10 @@ export function deserializeSnapshotConfig(snapshotConfigEs: SnapshotConfigEs): S
 export function serializeSnapshotConfig(snapshotConfig: SnapshotConfig): SnapshotConfigEs {
   const { indices, ignoreUnavailable, includeGlobalState, partial, metadata } = snapshotConfig;
 
+  const indicesArray = csvToArray(indices);
+
   const snapshotConfigEs: SnapshotConfigEs = {
-    indices,
+    indices: indicesArray,
     ignore_unavailable: ignoreUnavailable,
     include_global_state: includeGlobalState,
     partial,

x-pack/plugins/snapshot_restore/common/lib/utils.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const csvToArray = (indices?: string | string[]): string[] => {
+  return indices && Array.isArray(indices)
+    ? indices
+    : typeof indices === 'string'
+    ? indices.split(',')
+    : [];
+};

x-pack/plugins/snapshot_restore/common/types/index.ts

@@ -8,3 +8,4 @@ export * from './repository';
 export * from './snapshot';
 export * from './restore';
 export * from './policy';
+export * from './indices';

x-pack/plugins/snapshot_restore/common/types/indices.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface PolicyIndicesResponse {
+  indices: string[];
+  dataStreams: string[];
+}

x-pack/plugins/snapshot_restore/common/types/snapshot.ts

@@ -30,6 +30,7 @@ export interface SnapshotDetails {
   versionId: number;
   version: string;
   indices: string[];
+  dataStreams: string[];
   includeGlobalState: boolean;
   state: string;
   /** e.g. '2019-04-05T21:56:40.438Z' */
@@ -52,6 +53,7 @@ export interface SnapshotDetailsEs {
   version_id: number;
   version: string;
   indices: string[];
+  data_streams?: string[];
   include_global_state: boolean;
   state: string;
   /** e.g. '2019-04-05T21:56:40.438Z' */