Adding data_lifecycle to the _xpack/usage API

docs/changelog/96177.yaml

@@ -0,0 +1,5 @@
+pr: 96177
+summary: Adding `data_lifecycle` to the _xpack/usage API
+area: DLM
+type: enhancement
+issues: []

docs/reference/rest-api/usage.asciidoc

@@ -328,6 +328,19 @@ GET /_xpack/usage
     "data_streams" : 0,
     "indices_count" : 0
   },
+  "data_lifecycle" : {
+    "available": true,
+    "enabled": true,
+    "lifecycle": {
+        "count": 0,
+        "default_rollover_used": true,
+        "retention": {
+            "minimum_millis": 0,
+            "maximum_millis": 0,
+            "average_millis": 0.0
+        }
+    }
+  },
   "data_tiers" : {
     "available" : true,
     "enabled" : true,

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -126,12 +126,13 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId
     public static final TransportVersion V_8_500_003 = registerTransportVersion(8_500_003, "30adbe0c-8614-40dd-81b5-44e9c657bb77");
     public static final TransportVersion V_8_500_004 = registerTransportVersion(8_500_004, "6a00db6a-fd66-42a9-97ea-f6cc53169110");
     public static final TransportVersion V_8_500_005 = registerTransportVersion(8_500_005, "65370d2a-d936-4383-a2e0-8403f708129b");
+    public static final TransportVersion V_8_500_006 = registerTransportVersion(8_500_006, "7BB5621A-80AC-425F-BA88-75543C442F23");
 
     /**
      * Reference to the most recent transport version.
      * This should be the transport version with the highest id.
      */
-    public static final TransportVersion CURRENT = V_8_500_005;
+    public static final TransportVersion CURRENT = V_8_500_006;
 
     /**
      * Reference to the earliest compatible transport version to this version of the codebase.

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataLifecycleUsageTransportActionIT.java

@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.DataLifecycle;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamAlias;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.junit.After;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.core.action.XPackUsageFeatureAction.DATA_LIFECYCLE;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DataLifecycleUsageTransportActionIT extends ESIntegTestCase {
+    /*
+     * The DataLifecycleUsageTransportAction is not exposed in the xpack core plugin, so we have a special test plugin to do this
+     */
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestDateLifecycleUsagePlugin.class);
+    }
+
+    @After
+    private void cleanup() throws Exception {
+        updateClusterState(clusterState -> {
+            ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(clusterState);
+            Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata());
+            metadataBuilder.dataStreams(Map.of(), Map.of());
+            clusterStateBuilder.metadata(metadataBuilder);
+            return clusterStateBuilder.build();
+        });
+        updateClusterSettings(Settings.builder().put(DataLifecycle.CLUSTER_DLM_DEFAULT_ROLLOVER_SETTING.getKey(), (String) null));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testAction() throws Exception {
+        assertUsageResults(0, 0, 0, 0.0, true);
+        AtomicLong count = new AtomicLong(0);
+        AtomicLong totalRetentionTimes = new AtomicLong(0);
+        AtomicLong minRetention = new AtomicLong(Long.MAX_VALUE);
+        AtomicLong maxRetention = new AtomicLong(Long.MIN_VALUE);
+        boolean useDefaultRolloverConfig = randomBoolean();
+        if (useDefaultRolloverConfig == false) {
+            updateClusterSettings(Settings.builder().put(DataLifecycle.CLUSTER_DLM_DEFAULT_ROLLOVER_SETTING.getKey(), "min_docs=33"));
+        }
+        /*
+         * We now add a number of simulated data streams to the cluster state. Some have lifecycles, some don't. The ones with lifecycles
+         * have varying retention periods. After adding them, we make sure the numbers add up.
+         */
+        updateClusterState(clusterState -> {
+            Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata());
+            Map<String, DataStream> dataStreamMap = new HashMap<>();
+            for (int dataStreamCount = 0; dataStreamCount < randomInt(200); dataStreamCount++) {
+                boolean hasLifecycle = randomBoolean();
+                long retentionMillis;
+                if (hasLifecycle) {
+                    retentionMillis = randomLongBetween(1000, 100000);
+                    count.incrementAndGet();
+                    totalRetentionTimes.addAndGet(retentionMillis);
+                    if (retentionMillis < minRetention.get()) {
+                        minRetention.set(retentionMillis);
+                    }
+                    if (retentionMillis > maxRetention.get()) {
+                        maxRetention.set(retentionMillis);
+                    }
+                } else {
+                    retentionMillis = 0;
+                }
+                List<Index> indices = new ArrayList<>();
+                for (int indicesCount = 0; indicesCount < randomIntBetween(1, 10); indicesCount++) {
+                    Index index = new Index(randomAlphaOfLength(60), randomAlphaOfLength(60));
+                    indices.add(index);
+                }
+                boolean systemDataStream = randomBoolean();
+                DataStream dataStream = new DataStream(
+                    randomAlphaOfLength(50),
+                    indices,
+                    randomLongBetween(0, 1000),
+                    Map.of(),
+                    systemDataStream || randomBoolean(),
+                    randomBoolean(),
+                    systemDataStream,
+                    randomBoolean(),
+                    IndexMode.STANDARD,
+                    hasLifecycle ? new DataLifecycle(retentionMillis) : null
+                );
+                dataStreamMap.put(dataStream.getName(), dataStream);
+            }
+            Map<String, DataStreamAlias> dataStreamAliasesMap = Map.of();
+            metadataBuilder.dataStreams(dataStreamMap, dataStreamAliasesMap);
+            ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(clusterState);
+            clusterStateBuilder.metadata(metadataBuilder);
+            return clusterStateBuilder.build();
+        });
+        int expectedMinimumRetention = minRetention.get() == Long.MAX_VALUE ? 0 : minRetention.intValue();
+        int expectedMaximumRetention = maxRetention.get() == Long.MIN_VALUE ? 0 : maxRetention.intValue();
+        double expectedAverageRetention = count.get() == 0 ? 0.0 : totalRetentionTimes.doubleValue() / count.get();
+        assertUsageResults(
+            count.intValue(),
+            expectedMinimumRetention,
+            expectedMaximumRetention,
+            expectedAverageRetention,
+            useDefaultRolloverConfig
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertUsageResults(
+        int count,
+        int minimumRetention,
+        int maximumRetention,
+        double averageRetention,
+        boolean defaultRolloverUsed
+    ) throws Exception {
+        XPackUsageFeatureResponse response = client().execute(DATA_LIFECYCLE, new XPackUsageRequest()).get();
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder = response.getUsage().toXContent(builder, ToXContent.EMPTY_PARAMS);
+        Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(
+            BytesReference.bytes(builder),
+            true,
+            XContentType.JSON
+        );
+
+        Map<String, Object> map = tuple.v2();
+        assertThat(map.get("available"), equalTo(true));
+        assertThat(map.get("enabled"), equalTo(true));
+        assertThat(map.get("count"), equalTo(count));
+        assertThat(map.get("default_rollover_used"), equalTo(defaultRolloverUsed));
+        Map<String, Object> retentionMap = (Map<String, Object>) map.get("retention");
+        assertThat(retentionMap.size(), equalTo(3));
+        assertThat(retentionMap.get("minimum_millis"), equalTo(minimumRetention));
+        assertThat(retentionMap.get("maximum_millis"), equalTo(maximumRetention));
+        assertThat(retentionMap.get("average_millis"), equalTo(averageRetention));
+    }
+
+    /*
+     * Updates the cluster state in the internal cluster using the provided function
+     */
+    protected static void updateClusterState(final Function<ClusterState, ClusterState> updater) throws Exception {
+        final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        final ClusterService clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        clusterService.submitUnbatchedStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                return updater.apply(currentState);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                future.onFailure(e);
+            }
+
+            @Override
+            public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
+                future.onResponse(null);
+            }
+        });
+        future.get();
+    }
+
+    /*
+     * This plugin exposes the DataLifecycleUsageTransportAction.
+     */
+    public static final class TestDateLifecycleUsagePlugin extends XPackClientPlugin {
+        @Override
+        public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+            List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>();
+            actions.add(new ActionPlugin.ActionHandler<>(DATA_LIFECYCLE, DataLifecycleUsageTransportAction.class));
+            return actions;
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.DataLifecycle;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -38,6 +39,7 @@
 import org.elasticsearch.xpack.core.archive.ArchiveFeatureSetUsage;
 import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
+import org.elasticsearch.xpack.core.datastreams.DataLifecycleFeatureSetUsage;
 import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage;
 import org.elasticsearch.xpack.core.downsample.DownsampleIndexerAction;
 import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage;
@@ -234,6 +236,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Stream;
 
 // TODO: merge this into XPackPlugin
 public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPlugin {
@@ -414,7 +418,7 @@ public List<ActionType<? extends ActionResponse>> getClientActions() {
 
     @Override
     public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
-        return Arrays.asList(
+        return Stream.of(
             // graph
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
             // logstash
@@ -545,6 +549,13 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             ),
             // Data Streams
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_STREAMS, DataStreamFeatureSetUsage::new),
+            DataLifecycle.isEnabled()
+                ? new NamedWriteableRegistry.Entry(
+                    XPackFeatureSet.Usage.class,
+                    XPackField.DATA_LIFECYCLE,
+                    DataLifecycleFeatureSetUsage::new
+                )
+                : null,
             // Data Tiers
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_TIERS, DataTiersFeatureSetUsage::new),
             // Archive
@@ -561,7 +572,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 XPackField.ENTERPRISE_SEARCH,
                 EnterpriseSearchFeatureSetUsage::new
             )
-        );
+        ).filter(Objects::nonNull).toList();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -65,6 +65,8 @@ public final class XPackField {
     public static final String SEARCHABLE_SNAPSHOTS = "searchable_snapshots";
     /** Name constant for the data streams feature. */
     public static final String DATA_STREAMS = "data_streams";
+    /** Name constant for the data lifecycle feature. */
+    public static final String DATA_LIFECYCLE = "data_lifecycle";
     /** Name constant for the data tiers feature. */
     public static final String DATA_TIERS = "data_tiers";
     /** Name constant for the aggregate_metric plugin. */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -74,6 +75,7 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
 import org.elasticsearch.xpack.cluster.routing.allocation.mapper.DataTierFieldMapper;
+import org.elasticsearch.xpack.core.action.DataLifecycleUsageTransportAction;
 import org.elasticsearch.xpack.core.action.DataStreamInfoTransportAction;
 import org.elasticsearch.xpack.core.action.DataStreamUsageTransportAction;
 import org.elasticsearch.xpack.core.action.ReloadAnalyzerAction;
@@ -355,6 +357,9 @@ public Collection<Object> createComponents(
         actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_TIERS, DataTiersUsageTransportAction.class));
         actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_STREAMS, DataStreamUsageTransportAction.class));
         actions.add(new ActionHandler<>(XPackInfoFeatureAction.DATA_STREAMS, DataStreamInfoTransportAction.class));
+        if (DataLifecycle.isEnabled()) {
+            actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_LIFECYCLE, DataLifecycleUsageTransportAction.class));
+        }
         actions.add(new ActionHandler<>(XPackUsageFeatureAction.HEALTH, HealthApiUsageTransportAction.class));
         actions.add(new ActionHandler<>(XPackUsageFeatureAction.REMOTE_CLUSTERS, RemoteClusterUsageTransportAction.class));
         return actions;