Allow for the data lifecycle to be explicitly nullified

docs/changelog/95979.yaml

@@ -0,0 +1,5 @@
+pr: 95979
+summary: Allow for the data lifecycle and the retention to be explicitly nullified
+area: DLM
+type: enhancement
+issues: []

modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataLifecycle;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
@@ -32,11 +33,13 @@
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateTsdbMapping;
+import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.composeDataLifecycles;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.datastreams.MetadataDataStreamRolloverServiceTests.createSettingsProvider;
 import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -140,6 +143,47 @@ public void testRequireRoutingPath() throws Exception {
         }
     }
 
+    public void testLifecycleComposition() {
+        // No lifecycles result to null
+        {
+            List<DataLifecycle> lifecycles = List.of();
+            assertThat(composeDataLifecycles(lifecycles), nullValue());
+        }
+        // One lifecycle results to this lifecycle as the final
+        {
+            DataLifecycle lifecycle = switch (randomInt(2)) {
+                case 0 -> new DataLifecycle();
+                case 1 -> new DataLifecycle(DataLifecycle.Retention.NULL);
+                default -> new DataLifecycle(randomMillisUpToYear9999());
+            };
+            List<DataLifecycle> lifecycles = List.of(lifecycle);
+            assertThat(composeDataLifecycles(lifecycles), equalTo(lifecycle));
+        }
+        // If the last lifecycle is missing a property we keep the latest from the previous ones
+        {
+            DataLifecycle lifecycleWithRetention = new DataLifecycle(randomMillisUpToYear9999());
+            List<DataLifecycle> lifecycles = List.of(lifecycleWithRetention, new DataLifecycle());
+            assertThat(
+                composeDataLifecycles(lifecycles).getEffectiveDataRetention(),
+                equalTo(lifecycleWithRetention.getEffectiveDataRetention())
+            );
+        }
+        // If both lifecycle have all properties, then the latest one overwrites all the others
+        {
+            DataLifecycle lifecycle1 = new DataLifecycle(randomMillisUpToYear9999());
+            DataLifecycle lifecycle2 = new DataLifecycle(randomMillisUpToYear9999());
+            List<DataLifecycle> lifecycles = List.of(lifecycle1, lifecycle2);
+            assertThat(composeDataLifecycles(lifecycles), equalTo(lifecycle2));
+        }
+        // If the last lifecycle is explicitly null, the result is also null
+        {
+            DataLifecycle lifecycle1 = new DataLifecycle(randomMillisUpToYear9999());
+            DataLifecycle lifecycle2 = new DataLifecycle(randomMillisUpToYear9999());
+            List<DataLifecycle> lifecycles = List.of(lifecycle1, lifecycle2, Template.NO_LIFECYCLE);
+            assertThat(composeDataLifecycles(lifecycles), nullValue());
+        }
+    }
+
     private MetadataIndexTemplateService getMetadataIndexTemplateService() {
         var indicesService = getInstanceFromNode(IndicesService.class);
         var clusterService = getInstanceFromNode(ClusterService.class);

modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/CrudDataLifecycleIT.java

@@ -10,6 +10,8 @@
 
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
 import org.elasticsearch.cluster.metadata.DataLifecycle;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.dlm.action.DeleteDataLifecycleAction;
 import org.elasticsearch.dlm.action.GetDataLifecycleAction;
@@ -65,11 +67,11 @@ public void testGetLifecycle() throws Exception {
             GetDataLifecycleAction.Response response = client().execute(GetDataLifecycleAction.INSTANCE, getDataLifecycleRequest).get();
             assertThat(response.getDataStreamLifecycles().size(), equalTo(3));
             assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
-            assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+            assertDataLifecycle(response.getDataStreamLifecycles().get(0).lifecycle(), lifecycle);
             assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
-            assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
+            assertDataLifecycle(response.getDataStreamLifecycles().get(1).lifecycle(), lifecycle);
             assertThat(response.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
-            assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
+            assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), nullValue());
             assertThat(response.getRolloverConfiguration(), nullValue());
         }
 
@@ -79,9 +81,9 @@ public void testGetLifecycle() throws Exception {
             GetDataLifecycleAction.Response response = client().execute(GetDataLifecycleAction.INSTANCE, getDataLifecycleRequest).get();
             assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
             assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
-            assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+            assertDataLifecycle(response.getDataStreamLifecycles().get(0).lifecycle(), lifecycle);
             assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
-            assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
+            assertDataLifecycle(response.getDataStreamLifecycles().get(1).lifecycle(), lifecycle);
             assertThat(response.getRolloverConfiguration(), nullValue());
         }
 
@@ -93,7 +95,7 @@ public void testGetLifecycle() throws Exception {
             GetDataLifecycleAction.Response response = client().execute(GetDataLifecycleAction.INSTANCE, getDataLifecycleRequest).get();
             assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
             assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
-            assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+            assertDataLifecycle(response.getDataStreamLifecycles().get(0).lifecycle(), lifecycle);
             assertThat(response.getRolloverConfiguration(), nullValue());
         }
 
@@ -106,14 +108,22 @@ public void testGetLifecycle() throws Exception {
         ).get();
         assertThat(responseWithRollover.getDataStreamLifecycles().size(), equalTo(3));
         assertThat(responseWithRollover.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+        assertDataLifecycle(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), lifecycle);
         assertThat(responseWithRollover.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
-        assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
+        assertDataLifecycle(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), lifecycle);
         assertThat(responseWithRollover.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
         assertThat(responseWithRollover.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
         assertThat(responseWithRollover.getRolloverConfiguration(), notNullValue());
     }
 
+    private void assertDataLifecycle(DataLifecycle dataLifecycle, DataLifecycle expected) {
+        if (expected.equals(Template.NO_LIFECYCLE)) {
+            assertThat(dataLifecycle, nullValue());
+        } else {
+            assertThat(dataLifecycle, equalTo(expected));
+        }
+    }
+
     public void testPutLifecycle() throws Exception {
         putComposableIndexTemplate("id1", null, List.of("my-data-stream*"), null, null, null);
         // Create index without a lifecycle
@@ -132,17 +142,17 @@ public void testPutLifecycle() throws Exception {
 
         // Set lifecycle
         {
-            DataLifecycle lifecycle = randomDataLifecycle();
+            TimeValue dataRetention = randomBoolean() ? null : TimeValue.timeValueMillis(randomMillisUpToYear9999());
             PutDataLifecycleAction.Request putDataLifecycleRequest = new PutDataLifecycleAction.Request(
                 new String[] { "*" },
-                lifecycle.getDataRetention()
+                dataRetention
             );
             assertThat(client().execute(PutDataLifecycleAction.INSTANCE, putDataLifecycleRequest).get().isAcknowledged(), equalTo(true));
             GetDataLifecycleAction.Request getDataLifecycleRequest = new GetDataLifecycleAction.Request(new String[] { "my-data-stream" });
             GetDataLifecycleAction.Response response = client().execute(GetDataLifecycleAction.INSTANCE, getDataLifecycleRequest).get();
             assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
             assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
-            assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
+            assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getEffectiveDataRetention(), equalTo(dataRetention));
         }
     }
 

modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/ExplainDataLifecycleIT.java

@@ -119,7 +119,7 @@ public void testExplainLifecycle() throws Exception {
                 assertThat(explainIndex.isManagedByDLM(), is(true));
                 assertThat(explainIndex.getIndexCreationDate(), notNullValue());
                 assertThat(explainIndex.getLifecycle(), notNullValue());
-                assertThat(explainIndex.getLifecycle().getDataRetention(), nullValue());
+                assertThat(explainIndex.getLifecycle().getEffectiveDataRetention(), nullValue());
                 if (internalCluster().numDataNodes() > 1) {
                     // If the number of nodes is 1 then the cluster will be yellow so forcemerge will report an error if it has run
                     assertThat(explainIndex.getError(), nullValue());
@@ -208,7 +208,7 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception {
                 assertThat(explainIndex.isManagedByDLM(), is(true));
                 assertThat(explainIndex.getIndexCreationDate(), notNullValue());
                 assertThat(explainIndex.getLifecycle(), notNullValue());
-                assertThat(explainIndex.getLifecycle().getDataRetention(), nullValue());
+                assertThat(explainIndex.getLifecycle().getEffectiveDataRetention(), nullValue());
                 assertThat(explainIndex.getRolloverDate(), nullValue());
                 assertThat(explainIndex.getTimeSinceRollover(System::currentTimeMillis), nullValue());
                 // index has not been rolled over yet

modules/dlm/src/main/java/org/elasticsearch/dlm/DataLifecycleService.java

@@ -290,7 +290,7 @@ private void maybeExecuteRollover(ClusterState state, DataStream dataStream) {
             RolloverRequest rolloverRequest = getDefaultRolloverRequest(
                 rolloverConfiguration,
                 dataStream.getName(),
-                dataStream.getLifecycle().getDataRetention()
+                dataStream.getLifecycle().getEffectiveDataRetention()
             );
             transportActionsDeduplicator.executeOnce(
                 rolloverRequest,
@@ -525,7 +525,7 @@ static TimeValue getRetentionConfiguration(DataStream dataStream) {
         if (dataStream.getLifecycle() == null) {
             return null;
         }
-        return dataStream.getLifecycle().getDataRetention();
+        return dataStream.getLifecycle().getEffectiveDataRetention();
     }
 
     /**

modules/dlm/src/test/java/org/elasticsearch/dlm/DLMFixtures.java

@@ -28,9 +28,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.lucene.tests.util.LuceneTestCase.rarely;
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
 import static org.elasticsearch.test.ESIntegTestCase.client;
+import static org.elasticsearch.test.ESTestCase.randomInt;
 import static org.elasticsearch.test.ESTestCase.randomIntBetween;
 import static org.junit.Assert.assertTrue;
 
@@ -94,6 +94,11 @@ static void putComposableIndexTemplate(
     }
 
     static DataLifecycle randomDataLifecycle() {
-        return rarely() ? new DataLifecycle() : new DataLifecycle(TimeValue.timeValueDays(randomIntBetween(1, 365)));
+        return switch (randomInt(3)) {
+            case 0 -> new DataLifecycle();
+            case 1 -> new DataLifecycle(DataLifecycle.Retention.NULL);
+            case 2 -> Template.NO_LIFECYCLE;
+            default -> new DataLifecycle(TimeValue.timeValueDays(randomIntBetween(1, 365)));
+        };
     }
 }

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -127,12 +127,13 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId
     public static final TransportVersion V_8_500_004 = registerTransportVersion(8_500_004, "6a00db6a-fd66-42a9-97ea-f6cc53169110");
     public static final TransportVersion V_8_500_005 = registerTransportVersion(8_500_005, "65370d2a-d936-4383-a2e0-8403f708129b");
     public static final TransportVersion V_8_500_006 = registerTransportVersion(8_500_006, "7BB5621A-80AC-425F-BA88-75543C442F23");
+    public static final TransportVersion V_8_500_007 = registerTransportVersion(8_500_007, "77261d43-4149-40af-89c5-7e71e0454fce");
 
     /**
      * Reference to the most recent transport version.
      * This should be the transport version with the highest id.
      */
-    public static final TransportVersion CURRENT = V_8_500_006;
+    public static final TransportVersion CURRENT = V_8_500_007;
 
     /**
      * Reference to the earliest compatible transport version to this version of the codebase.