Add telemetry for data tiers

docs/reference/rest-api/info.asciidoc

@@ -146,6 +146,10 @@ Example response:
       "data_streams" : {
          "available" : true,
          "enabled" : true,
+      },
+      "data_tiers" : {
+         "available" : true,
+         "enabled" : true,
       }
    },
    "tagline" : "You know, for X"

docs/reference/rest-api/usage.asciidoc

@@ -285,6 +285,70 @@ GET /_xpack/usage
     "enabled" : true,
     "data_streams" : 0,
     "indices_count" : 0
+  },
+  "data_tiers" : {
+    "available" : true,
+    "enabled" : true,
+    "data_warm" : {
+      "node_count" : 0,
+      "index_count" : 0,
+      "total_shard_count" : 0,
+      "primary_shard_count" : 0,
+      "doc_count" : 0,
+      "total_size_bytes" : 0,
+      "primary_size_bytes" : 0,
+      "primary_shard_size_avg_bytes" : 0,
+      "primary_shard_size_median_bytes" : 0,
+      "primary_shard_size_mad_bytes" : 0
+    },
+    "data" : {
+      "node_count" : 1,
+      "index_count" : 0,
+      "total_shard_count" : 0,
+      "primary_shard_count" : 0,
+      "doc_count" : 0,
+      "total_size_bytes" : 0,
+      "primary_size_bytes" : 0,
+      "primary_shard_size_avg_bytes" : 0,
+      "primary_shard_size_median_bytes" : 0,
+      "primary_shard_size_mad_bytes" : 0
+    },
+    "data_cold" : {
+      "node_count" : 0,
+      "index_count" : 0,
+      "total_shard_count" : 0,
+      "primary_shard_count" : 0,
+      "doc_count" : 0,
+      "total_size_bytes" : 0,
+      "primary_size_bytes" : 0,
+      "primary_shard_size_avg_bytes" : 0,
+      "primary_shard_size_median_bytes" : 0,
+      "primary_shard_size_mad_bytes" : 0
+    },
+    "data_content" : {
+      "node_count" : 0,
+      "index_count" : 0,
+      "total_shard_count" : 0,
+      "primary_shard_count" : 0,
+      "doc_count" : 0,
+      "total_size_bytes" : 0,
+      "primary_size_bytes" : 0,
+      "primary_shard_size_avg_bytes" : 0,
+      "primary_shard_size_median_bytes" : 0,
+      "primary_shard_size_mad_bytes" : 0
+    },
+    "data_hot" : {
+      "node_count" : 0,
+      "index_count" : 0,
+      "total_shard_count" : 0,
+      "primary_shard_count" : 0,
+      "doc_count" : 0,
+      "total_size_bytes" : 0,
+      "primary_size_bytes" : 0,
+      "primary_shard_size_avg_bytes" : 0,
+      "primary_shard_size_median_bytes" : 0,
+      "primary_shard_size_mad_bytes" : 0
+    }
   }
 }
 ------------------------------------------------------------

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

@@ -16,21 +16,24 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.core.DataTier;
-import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.DataTiersFeatureSetUsage;
+import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
 public class DataTierIT extends ESIntegTestCase {
     private static final String index = "myindex";
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(LocalStateCompositeXPackPlugin.class);
+        return Collections.singleton(DataTierTelemetryPlugin.class);
     }
 
     public void testDefaultIndexAllocateToContent() {
@@ -194,6 +197,61 @@ public void testTemplateOverridesDefaults() {
         ensureYellow(index);
     }
 
+    public void testDataTierTelemetry() {
+        startContentOnlyNode();
+        startContentOnlyNode();
+        startHotOnlyNode();
+
+        client().admin().indices().prepareCreate(index)
+            .setSettings(Settings.builder()
+                .put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, "data_hot")
+                .put("index.number_of_shards", 2)
+                .put("index.number_of_replicas", 0))
+            .setWaitForActiveShards(0)
+            .get();
+
+        client().admin().indices().prepareCreate(index + "2")
+            .setSettings(Settings.builder()
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 1))
+            .setWaitForActiveShards(0)
+            .get();
+
+        ensureGreen();
+        client().prepareIndex(index).setSource("foo", "bar").get();
+        client().prepareIndex(index + "2").setSource("foo", "bar").get();
+        client().prepareIndex(index + "2").setSource("foo", "bar").get();
+        refresh(index, index + "2");
+
+        DataTiersFeatureSetUsage usage = getUsage();
+        // We can't guarantee that internal indices aren't created, so some of these are >= checks
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).nodeCount, equalTo(2));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).indexCount, greaterThanOrEqualTo(1));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).totalShardCount, greaterThanOrEqualTo(2));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryShardCount, greaterThanOrEqualTo(1));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).docCount, greaterThanOrEqualTo(2L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryByteCount, greaterThanOrEqualTo(1L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryByteCountMedian, greaterThanOrEqualTo(1L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryShardBytesMAD, greaterThanOrEqualTo(0L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).nodeCount, equalTo(1));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).indexCount, greaterThanOrEqualTo(1));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).totalShardCount, greaterThanOrEqualTo(2));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryShardCount, greaterThanOrEqualTo(2));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).docCount, greaterThanOrEqualTo(1L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryByteCount, greaterThanOrEqualTo(1L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryByteCountMedian, greaterThanOrEqualTo(1L));
+        assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryShardBytesMAD, greaterThanOrEqualTo(0L));
+    }
+
+    private DataTiersFeatureSetUsage getUsage() {
+        XPackUsageResponse usages = new XPackUsageRequestBuilder(client()).execute().actionGet();
+        return usages.getUsages().stream()
+            .filter(u -> u instanceof DataTiersFeatureSetUsage)
+            .findFirst()
+            .map(u -> (DataTiersFeatureSetUsage) u)
+            .orElseThrow();
+    }
+
     public void startDataNode() {
         Settings nodeSettings = Settings.builder()
             .putList("node.roles", Arrays.asList("master", "data", "ingest"))

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierTelemetryPlugin.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.cluster.routing.allocation;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.LicenseService;
+import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
+import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageResponse;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This plugin extends {@link LocalStateCompositeXPackPlugin} to only make the data tier telemetry
+ * available. This allows telemetry to be retrieved in integration tests where it would otherwise
+ * throw errors trying to retrieve all of the different telemetry types.
+ */
+public class DataTierTelemetryPlugin extends LocalStateCompositeXPackPlugin {
+
+    public static class DataTiersTransportXPackUsageAction extends TransportXPackUsageAction {
+        @Inject
+        public DataTiersTransportXPackUsageAction(ThreadPool threadPool, TransportService transportService,
+                                                  ClusterService clusterService, ActionFilters actionFilters,
+                                                  IndexNameExpressionResolver indexNameExpressionResolver, NodeClient client) {
+            super(threadPool, transportService, clusterService, actionFilters, indexNameExpressionResolver, client);
+        }
+        @Override
+        protected List<XPackUsageFeatureAction> usageActions() {
+            return Collections.singletonList(XPackUsageFeatureAction.DATA_TIERS);
+        }
+    }
+
+    public static class DataTiersTransportXPackInfoAction extends TransportXPackInfoAction {
+        @Inject
+        public DataTiersTransportXPackInfoAction(TransportService transportService, ActionFilters actionFilters,
+                                                 LicenseService licenseService, NodeClient client) {
+            super(transportService, actionFilters, licenseService, client);
+        }
+
+        @Override
+        protected List<XPackInfoFeatureAction> infoActions() {
+            return Collections.singletonList(XPackInfoFeatureAction.DATA_TIERS);
+        }
+    }
+
+    public DataTierTelemetryPlugin(final Settings settings, final Path configPath) {
+        super(settings, configPath);
+    }
+
+    @Override
+    protected Class<? extends TransportAction<XPackUsageRequest, XPackUsageResponse>> getUsageAction() {
+        return DataTiersTransportXPackUsageAction.class;
+    }
+
+    @Override
+    protected Class<? extends TransportAction<XPackInfoRequest, XPackInfoResponse>> getInfoAction() {
+        return DataTiersTransportXPackInfoAction.class;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java

@@ -16,6 +16,8 @@
 import org.elasticsearch.index.shard.IndexSettingProvider;
 import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
 
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Set;
 
 /**
@@ -34,6 +36,8 @@ public class DataTier {
     public static final String DATA_WARM = "data_warm";
     public static final String DATA_COLD = "data_cold";
 
+    public static final Set<String> ALL_DATA_TIERS = new HashSet<>(Arrays.asList(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD));
+
     /**
      * Returns true if the given tier name is a valid tier
      */