[ILM] Data tiers for 7.10 (elastic#76126)

* wip

* Revert "wip"

This reverts commit 54b6f7ff3ec8b0f57b150ab2276d617686da9fb5.

* Revert "Revert "wip""

This reverts commit 63868b44ec60d7431c3a0189b16aeece1db2d38e.

* Refactor to using EUI button group component

- also moved node attr and node allocation component to inside
  new folder that contains all allocation components.
- only focussed on updating warm phase for now

* WIP: moved form UX more in line with EUI

- The described form group now has a switch for showing
  controls on the left.
- Refactored DataTierAllocation to CustomDataTierAllocation
- Removed 'node-roles' option
- Updated copy
- Moved JSX around a bit in the edit policy section to make logic
  simpler

* Refactor UI to reflect custom-ness of "Custom" and "None" options

- Still only implemented for warm, cold and frozen are still
  under way

* server side changes for getting node data

* double opt-in

* Refactored data tier allocation type

- Made types more explicit 'default', 'custom' and 'none'
- Fixed issue introduced by use useCallback on state setter -
  need to use the function setter pattern to not have stale data
  being set.

* Some refacoring, but main point is to add warning detection for
node roles.

- Refactored way we get node data to a provider component so that
  phases still have flexibility in how they render. Currently
  this also means that we fetch node stats data for each phase
  we render
- Created a callout for when there is no node role to which data
  can be allocated for the default setting
- Also updated the behaviour to render the entire form even when
  we cannot fetch node data for some reason. It is not ideal to
  not have node data, but we should not block the entire form.

* fix i18n

* fix type issue with deafult policies missing allocation type

* remove "undefined" as option for setting phase data

* Create referentially stable data setter for all phases - prevent infini-update

* fix type issue

* refactor data -> nodesData

* refactor cold phase for data tiers

* refactor frozen phase for data tiers

* fixed existing tests for warm section

* restored existing test coverage for cold phase and added test coverage for frozen

* fix api integration test

* remove unused translations

* slight UX update to turning on custom attribute allocation

* added scss file for data tier advanced section and other style
updates

* added tests for new warning

* fix types

* added correct copy for cold and frozen phases

* fix types and i18n

* implement copy feedback

* added spacer after the enable data tier allocation switch

* refactor to super select

* fix replicas copy

* update phase serialization for cold and frozen

* Refactor so that logic determining warnings lives together

- also factor out the warning of the node allocation component
- revisit copy for the allocation warning

* tier -> phase

* Added some much needed policy serialization test coverage

- also factored out policy allocation action serialization

* fix import paths and added required file header

* fix existing test coverage

* refine copy for data tier allocation recommended option

* fix showing warning for no node attrs

* fix inverted warning logic 🤦🏼‍♂️

* fix typing

* implement CJs copy feedback

* fix i18n

* remove unused or invalid translations

* provide ability to not alter original policy

* do not alter the original policy in the serilalization process

* fix jest tests

* Remove duplicate type and refactor NodeRole to NodeDataRole

Also deleted unused component "AdvancedSection" for now

* added comment to "false" typing

* revised and refactored copy based on feedback

* address copy feedback

* update kibana schema to allow migrate: { enabled: false }

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/index_lifecycle_management/__jest__/components/helpers/edit_policy.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { init as initHttpRequests } from './http_requests';
+
+export type EditPolicySetup = ReturnType<typeof setup>;
+
+export const setup = () => {
+  const { httpRequestsMockHelpers, server } = initHttpRequests();
+
+  const setupNodeListResponse = (
+    response: Record<string, any> = {
+      nodesByAttributes: { 'attribute:true': ['node1'] },
+      nodesByRoles: { data: ['node1'] },
+    }
+  ) => {
+    httpRequestsMockHelpers.setNodesListResponse(response);
+  };
+
+  return {
+    http: {
+      setupNodeListResponse,
+      httpRequestsMockHelpers,
+      server,
+    },
+  };
+};

x-pack/plugins/index_lifecycle_management/__jest__/components/helpers/http_requests.ts

@@ -40,6 +40,8 @@ const registerHttpRequestMockHelpers = (server: SinonFakeServer) => {
   };
 };
 
+export type HttpRequestMockHelpers = ReturnType<typeof registerHttpRequestMockHelpers>;
+
 export const init = () => {
   const server = sinon.fakeServer.create();
 

x-pack/plugins/index_lifecycle_management/__jest__/components/helpers/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as editPolicyHelpers from './edit_policy';
+
+export { HttpRequestMockHelpers, init } from './http_requests';
+
+export { editPolicyHelpers };

x-pack/plugins/index_lifecycle_management/common/types/api.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export type NodeDataRole = 'data' | 'data_hot' | 'data_warm' | 'data_cold' | 'data_frozen';
+
+export interface ListNodesRouteResponse {
+  nodesByAttributes: { [attributePair: string]: string[] };
+  nodesByRoles: { [role in NodeDataRole]?: string[] };
+}

x-pack/plugins/index_lifecycle_management/common/types/index.ts

@@ -4,4 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export * from './api';
+
 export * from './policies';

x-pack/plugins/index_lifecycle_management/common/types/policies.ts

@@ -6,6 +6,8 @@
 
 import { Index as IndexInterface } from '../../../index_management/common/types';
 
+export type PhaseWithAllocation = 'warm' | 'cold' | 'frozen';
+
 export interface SerializedPolicy {
   name: string;
   phases: Phases;
@@ -62,6 +64,7 @@ export interface SerializedWarmPhase extends SerializedPhase {
     set_priority?: {
       priority: number | null;
     };
+    migrate?: { enabled: boolean };
   };
 }
 
@@ -72,6 +75,7 @@ export interface SerializedColdPhase extends SerializedPhase {
     set_priority?: {
       priority: number | null;
     };
+    migrate?: { enabled: boolean };
   };
 }
 
@@ -82,6 +86,7 @@ export interface SerializedFrozenPhase extends SerializedPhase {
     set_priority?: {
       priority: number | null;
     };
+    migrate?: { enabled: boolean };
   };
 }
 
@@ -103,6 +108,13 @@ export interface AllocateAction {
   require?: {
     [attribute: string]: string;
   };
+  migrate?: {
+    /**
+     * If enabled is ever set it will only be set to `false` because the default value
+     * for this is `true`. Rather leave unspecified for true.
+     */
+    enabled: false;
+  };
 }
 
 export interface Policy {
@@ -125,9 +137,23 @@ export interface PhaseWithMinAge {
   selectedMinimumAgeUnits: string;
 }
 
+/**
+ * Different types of allocation markers we use in deserialized policies.
+ *
+ * default - use data tier based data allocation based on node roles -- this is ES best practice mode.
+ * custom - use node_attrs to allocate data to specific nodes
+ * none - do not move data anywhere when entering a phase
+ */
+export type DataTierAllocationType = 'default' | 'custom' | 'none';
+
 export interface PhaseWithAllocationAction {
   selectedNodeAttrs: string;
   selectedReplicaCount: string;
+  /**
+   * A string value indicating allocation type. If unspecified we assume the user
+   * wants to use default allocation.
+   */
+  dataTierAllocationType: DataTierAllocationType;
 }
 
 export interface PhaseWithIndexPriority {

x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts

@@ -38,6 +38,7 @@ export const defaultNewWarmPhase: WarmPhase = {
   selectedReplicaCount: '',
   warmPhaseOnRollover: true,
   phaseIndexPriority: '50',
+  dataTierAllocationType: 'default',
 };
 
 export const defaultNewColdPhase: ColdPhase = {
@@ -48,6 +49,7 @@ export const defaultNewColdPhase: ColdPhase = {
   selectedReplicaCount: '',
   freezeEnabled: false,
   phaseIndexPriority: '0',
+  dataTierAllocationType: 'default',
 };
 
 export const defaultNewFrozenPhase: FrozenPhase = {
@@ -58,6 +60,7 @@ export const defaultNewFrozenPhase: FrozenPhase = {
   selectedReplicaCount: '',
   freezeEnabled: false,
   phaseIndexPriority: '0',
+  dataTierAllocationType: 'default',
 };
 
 export const defaultNewDeletePhase: DeletePhase = {

x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/check_phase_compatibility.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  NodeDataRole,
+  ListNodesRouteResponse,
+  PhaseWithAllocation,
+} from '../../../../common/types';
+
+/**
+ * Given a phase and current node roles, determine whether the phase
+ * can use default data tier allocation.
+ *
+ * This can only be checked for phases that have an allocate action.
+ */
+export const isPhaseDefaultDataAllocationCompatible = (
+  phase: PhaseWithAllocation,
+  nodesByRoles: ListNodesRouteResponse['nodesByRoles']
+): boolean => {
+  // The 'data' role covers all node roles, so if we have at least one node with the data role
+  // we can use default allocation.
+  if (nodesByRoles.data?.length) {
+    return true;
+  }
+
+  // Otherwise we need to check whether a node role for the specific phase exists
+  if (nodesByRoles[`data_${phase}` as NodeDataRole]?.length) {
+    return true;
+  }
+
+  // Otherwise default allocation has nowhere to allocate new shards to in this phase.
+  return false;
+};

x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DataTierAllocationType, AllocateAction } from '../../../../common/types';
+
+/**
+ * Determine what deserialized state the policy config represents.
+ *
+ * See {@DataTierAllocationType} for more information.
+ */
+export const determineDataTierAllocationType = (
+  allocateAction?: AllocateAction
+): DataTierAllocationType => {
+  if (!allocateAction) {
+    return 'default';
+  }
+
+  if (allocateAction.migrate?.enabled === false) {
+    return 'none';
+  }
+
+  if (
+    (allocateAction.require && Object.keys(allocateAction.require).length) ||
+    (allocateAction.include && Object.keys(allocateAction.include).length) ||
+    (allocateAction.exclude && Object.keys(allocateAction.exclude).length)
+  ) {
+    return 'custom';
+  }
+
+  return 'default';
+};

x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './determine_allocation_type';
+
+export * from './check_phase_compatibility';

x-pack/plugins/index_lifecycle_management/public/application/lib/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './data_tiers';

x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/data_tier_allocation.scss

@@ -0,0 +1,9 @@
+.indexLifecycleManagement__phase__dataTierAllocation {
+  &__controlSection {
+    background-color: $euiColorLightestShade;
+    padding-top: $euiSizeM;
+    padding-left: $euiSizeM;
+    padding-right: $euiSizeM;
+    padding-bottom: $euiSizeM;
+  }
+}