[Index Management] Add support for effective retention in DSL (#181375)

x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts

@@ -114,4 +114,5 @@ export type TestSubjects =
   | 'indicesSearch'
   | 'noIndicesMessage'
   | 'clearIndicesSearch'
+  | 'usingMaxRetention'
   | 'componentTemplatesLink';

x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts

@@ -13,7 +13,7 @@ import {
   breadcrumbService,
   IndexManagementBreadcrumb,
 } from '../../../public/application/services/breadcrumbs';
-import { API_BASE_PATH } from '../../../common/constants';
+import { API_BASE_PATH, MAX_DATA_RETENTION } from '../../../common/constants';
 import * as fixtures from '../../../test/fixtures';
 import { setupEnvironment } from '../helpers';
 import { notificationService } from '../../../public/application/services/notification';
@@ -164,6 +164,12 @@ describe('Data Streams tab', () => {
           name: 'dataStream2',
           storageSize: '1kb',
           storageSizeBytes: 1000,
+          lifecycle: {
+            enabled: true,
+            data_retention: '7d',
+            effective_retention: '5d',
+            retention_determined_by: MAX_DATA_RETENTION,
+          },
         }),
       ]);
 
@@ -192,10 +198,16 @@ describe('Data Streams tab', () => {
 
       expect(tableCellsValues).toEqual([
         ['', 'dataStream1', 'green', '1', '7 days', 'Delete'],
-        ['', 'dataStream2', 'green', '1', '7 days', 'Delete'],
+        ['', 'dataStream2', 'green', '1', '5 days ', 'Delete'],
       ]);
     });
 
+    test('highlights datastreams who are using max retention', async () => {
+      const { exists } = testBed;
+
+      expect(exists('usingMaxRetention')).toBe(true);
+    });
+
     test('has a button to reload the data streams', async () => {
       const { exists, actions } = testBed;
 
@@ -247,7 +259,7 @@ describe('Data Streams tab', () => {
           'December 31st, 1969 7:00:00 PM',
           '1kb',
           '1',
-          '7 days',
+          '5 days ',
           'Delete',
         ],
       ]);
@@ -285,7 +297,7 @@ describe('Data Streams tab', () => {
           'December 31st, 1969 7:00:00 PM',
           '1kb',
           '1',
-          '7 days',
+          '5 days ',
           'Delete',
         ],
       ]);
@@ -316,7 +328,7 @@ describe('Data Streams tab', () => {
       const { tableCellsValues } = table.getMetaData('dataStreamTable');
       expect(tableCellsValues).toEqual([
         ['', 'dataStream1', 'green', 'December 31st, 1969 7:00:00 PM', '1', '7 days', 'Delete'],
-        ['', 'dataStream2', 'green', 'December 31st, 1969 7:00:00 PM', '1', '7 days', 'Delete'],
+        ['', 'dataStream2', 'green', 'December 31st, 1969 7:00:00 PM', '1', '5 days ', 'Delete'],
       ]);
     });
 

x-pack/plugins/index_management/common/constants/index.ts

@@ -14,6 +14,8 @@ export * from './index_statuses';
 // the request is 4096 bytes we can fit a max of 16 indices in a single request.
 export const MAX_INDICES_PER_REQUEST = 16;
 
+export const MAX_DATA_RETENTION = 'max_retention';
+
 export {
   UIM_APP_NAME,
   UIM_APP_LOAD,

x-pack/plugins/index_management/common/types/data_streams.ts

@@ -61,6 +61,8 @@ export interface DataStream {
   nextGenerationManagedBy: string;
   lifecycle?: IndicesDataStreamLifecycleWithRollover & {
     enabled?: boolean;
+    effective_retention?: string;
+    retention_determined_by?: string;
   };
 }
 

x-pack/plugins/index_management/public/application/lib/data_streams.test.tsx

@@ -34,5 +34,15 @@ describe('Data stream helpers', () => {
         })
       ).toBe('2 days');
     });
+
+    it('effective_retention should always have precedence over data_retention', () => {
+      expect(
+        getLifecycleValue({
+          enabled: true,
+          data_retention: '2d',
+          effective_retention: '5d',
+        })
+      ).toBe('5 days');
+    });
   });
 });

x-pack/plugins/index_management/public/application/lib/data_streams.tsx

@@ -51,7 +51,7 @@ export const getLifecycleValue = (
     return i18n.translate('xpack.idxMgmt.dataStreamList.dataRetentionDisabled', {
       defaultMessage: 'Disabled',
     });
-  } else if (!lifecycle?.data_retention) {
+  } else if (!lifecycle?.effective_retention && !lifecycle?.data_retention) {
     const infiniteDataRetention = i18n.translate(
       'xpack.idxMgmt.dataStreamList.dataRetentionInfinite',
       {
@@ -75,7 +75,8 @@ export const getLifecycleValue = (
   }
 
   // Extract size and unit, in order to correctly map the unit to the correct text
-  const { size, unit } = splitSizeAndUnits(lifecycle?.data_retention as string);
+  const activeRetention = lifecycle?.effective_retention || lifecycle?.data_retention;
+  const { size, unit } = splitSizeAndUnits(activeRetention as string);
   const availableTimeUnits = [...timeUnits, ...extraTimeUnits];
   const match = availableTimeUnits.find((timeUnit) => timeUnit.value === unit);
 

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx

@@ -8,6 +8,7 @@
 import React, { useState, Fragment } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
+import { omit } from 'lodash';
 import {
   EuiButton,
   EuiButtonEmpty,
@@ -310,7 +311,7 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
       },
       {
         name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.dataRetentionTitle', {
-          defaultMessage: 'Data retention',
+          defaultMessage: 'Effective data retention',
         }),
         toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.dataRetentionToolTip', {
           defaultMessage: `Data is kept at least this long before being automatically deleted. The data retention value only applies to the data managed directly by the data stream. If some data is subject to an index lifecycle management policy, then the data retention value set for the data stream doesn't apply to that data.`,
@@ -327,6 +328,34 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
       },
     ];
 
+    // If both rentention types are available, we wanna surface to the user both
+    if (lifecycle?.effective_retention && lifecycle?.data_retention) {
+      defaultDetails.push({
+        name: i18n.translate(
+          'xpack.idxMgmt.dataStreamDetailPanel.customerDefinedDataRetentionTitle',
+          {
+            defaultMessage: 'Data retention',
+          }
+        ),
+        toolTip: i18n.translate(
+          'xpack.idxMgmt.dataStreamDetailPanel.customerDefinedDataRetentionTooltip',
+          {
+            defaultMessage:
+              "This is the data retention that you defined. Because of other system constraints or settings, the data retention that is effectively applied may be different from the value you set. You can find the value retained and applied by the system under 'Effective data retention'.",
+          }
+        ),
+        content: (
+          <ConditionalWrap
+            condition={isDataStreamFullyManagedByILM(dataStream)}
+            wrap={(children) => <EuiTextColor color="subdued">{children}</EuiTextColor>}
+          >
+            <>{getLifecycleValue(omit(lifecycle, ['effective_retention']))}</>
+          </ConditionalWrap>
+        ),
+        dataTestSubj: 'dataRetentionDetail',
+      });
+    }
+
     const managementDetails = getManagementDetails();
     const details = [...defaultDetails, ...managementDetails];
 

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx

@@ -19,6 +19,7 @@ import {
 } from '@elastic/eui';
 import { ScopedHistory } from '@kbn/core/public';
 
+import { MAX_DATA_RETENTION } from '../../../../../../common/constants';
 import { useAppContext } from '../../../../app_context';
 import { DataStream } from '../../../../../../common/types';
 import { getLifecycleValue } from '../../../../lib/data_streams';
@@ -169,7 +170,33 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
         condition={dataStream.isDataStreamFullyManagedByILM}
         wrap={(children) => <EuiTextColor color="subdued">{children}</EuiTextColor>}
       >
-        <>{getLifecycleValue(lifecycle, INFINITE_AS_ICON)}</>
+        <>
+          {getLifecycleValue(lifecycle, INFINITE_AS_ICON)}
+
+          {lifecycle?.retention_determined_by === MAX_DATA_RETENTION && (
+            <>
+              {' '}
+              <EuiToolTip
+                content={i18n.translate(
+                  'xpack.idxMgmt.dataStreamList.table.usingEffectiveRetentionTooltip',
+                  {
+                    defaultMessage: `This data stream is using the maximum allowed data retention: [{effectiveRetention}].`,
+                    values: {
+                      effectiveRetention: lifecycle?.effective_retention,
+                    },
+                  }
+                )}
+              >
+                <EuiIcon
+                  size="s"
+                  color="subdued"
+                  type="iInCircle"
+                  data-test-subj="usingMaxRetention"
+                />
+              </EuiToolTip>
+            </>
+          )}
+        </>
       </ConditionalWrap>
     ),
   });

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/edit_data_retention_modal/edit_data_retention_modal.tsx

@@ -232,6 +232,13 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
 
     return updateDataRetention(dataStreamName, data).then(({ data: responseData, error }) => {
       if (responseData) {
+        // If the response came back with a warning from ES, rely on that for the
+        // toast message.
+        if (responseData.warning) {
+          notificationService.showWarningToast(responseData.warning);
+          return onClose({ hasUpdatedDataRetention: true });
+        }
+
         const successMessage = i18n.translate(
           'xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.successDataRetentionNotification',
           {

x-pack/plugins/index_management/server/routes/api/data_streams/data_streams.test.ts

@@ -9,6 +9,7 @@ import { addBasePath } from '..';
 import { RouterMock, routeDependencies, RequestMock } from '../../../test/helpers';
 
 import { registerDataStreamRoutes } from '.';
+import { getEsWarningText } from './register_put_route';
 
 describe('Data streams API', () => {
   const router = new RouterMock();
@@ -57,5 +58,11 @@ describe('Data streams API', () => {
 
       await expect(router.runRequest(mockRequest)).rejects.toThrowError(error);
     });
+
+    it('knows how to extract the es warning header from the response', () => {
+      expect(getEsWarningText('Elasticsearch-asdqwe123 "Test string"')).toBeNull();
+      expect(getEsWarningText('299 Easdqwe123 "Test string"')).toBeNull();
+      expect(getEsWarningText('299 Elasticsearch-asdqwe123 "Test string"')).toBe('Test string');
+    });
   });
 });

x-pack/plugins/index_management/server/routes/api/data_streams/register_put_route.ts

@@ -10,6 +10,15 @@ import { schema, TypeOf } from '@kbn/config-schema';
 import { RouteDependencies } from '../../../types';
 import { addBasePath } from '..';
 
+/** HTTP Warning headers have the following syntax:
+ * <warn-code> <warn-agent> <warn-text> (where warn-code is a three-digit number)
+ * This function only returns the warn-text if it exists.
+ * */
+export const getEsWarningText = (warning: string): string | null => {
+  const match = warning.match(/\d{3} Elasticsearch-\w+ "(.*)"/);
+  return match ? match[1] : null;
+};
+
 export function registerPutDataRetention({ router, lib: { handleEsError } }: RouteDependencies) {
   const paramsSchema = schema.object({
     name: schema.string(),
@@ -36,9 +45,25 @@ export function registerPutDataRetention({ router, lib: { handleEsError } }: Rou
           await client.asCurrentUser.indices.deleteDataLifecycle({ name });
         } else {
           // Otherwise, we create or update the data retention policy.
-          await client.asCurrentUser.indices.putDataLifecycle({
-            name,
-            data_retention: dataRetention,
+          //
+          // Be aware that in serverless it could happen that the user defined
+          // data retention wont be the effective retention as there might be a
+          // global data retention limit set.
+          const { headers } = await client.asCurrentUser.indices.putDataLifecycle(
+            {
+              name,
+              data_retention: dataRetention,
+            },
+            { meta: true }
+          );
+
+          return response.ok({
+            body: {
+              success: true,
+              ...(headers?.warning
+                ? { warning: getEsWarningText(headers.warning) ?? headers.warning }
+                : {}),
+            },
           });
         }