From ddc3b37580d135e6cb18e36b544f2bdbcf42189d Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 27 Jul 2021 10:39:53 -0500 Subject: [PATCH] Adding shard count to node stats api (#75433) * Adding shard count to _nodes/stats api Added a shards section to each node returned by the _nodes/stats api. Currently this new section only contains a total count of all shards on the node. --- docs/reference/cluster/nodes-stats.asciidoc | 12 +++ .../rest-api-spec/api/nodes.stats.json | 6 +- .../test/nodes.stats/11_indices_metrics.yml | 73 ++++++++++++++++++ .../indices/stats/IndexStatsIT.java | 8 +- .../admin/indices/stats/CommonStats.java | 28 ++++++- .../admin/indices/stats/CommonStatsFlags.java | 3 +- .../index/shard/ShardCountStats.java | 77 +++++++++++++++++++ .../index/shard/ShardCountStatsTest.java | 39 ++++++++++ 8 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 0a764b6095ff1..033f202169985 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -959,6 +959,18 @@ Time by which recovery operations were delayed due to throttling. Time in milliseconds recovery operations were delayed due to throttling. ======= + +`shards`:: +(object) +Contains statistics about all shards assigned to the node. ++ +.Properties of `shards` +[%collapsible%open] +======= +`total_count`:: +(integer) +The total number of shards assigned to the node. +======= ====== [[cluster-nodes-stats-api-response-body-os]] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json index acea11146d384..7a13a6c1033c5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json @@ -127,7 +127,8 @@ "segments", "store", "warmer", - "bulk" + "bulk", + "shards" ], "description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified." } @@ -175,7 +176,8 @@ "segments", "store", "warmer", - "bulk" + "bulk", + "shards" ], "description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified." }, diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 3b2ffcd272727..3824d2eaa1604 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -109,6 +109,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards --- "Metric - multi": @@ -166,6 +167,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards --- "Metric - _all include_segment_file_sizes": @@ -223,6 +225,7 @@ - is_true: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards - is_true: nodes.$node_id.indices.segments.file_sizes --- @@ -254,6 +257,7 @@ - is_true: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards --- "Metric - _all include_unloaded_segments": @@ -315,3 +319,72 @@ - gte: { nodes.$node_id.http.clients.0.request_size_bytes: 0 } # values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often # null and cannot be tested here + +--- +"Metric - blank for indices shards": + - skip: + features: [arbitrary_key] + version: " - 7.99.99" + reason: "total shard count added in version 8.0" + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: {} + + - is_true: nodes.$node_id.indices.shards + - match: { nodes.$node_id.indices.shards.total_count: 0 } + +--- +"Metric - _all for indices shards": + - skip: + features: [arbitrary_key] + version: " - 7.99.99" + reason: "total shard count added in version 8.0" + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: { metric: _all } + + - is_true: nodes.$node_id.indices.shards + - match: { nodes.$node_id.indices.shards.total_count: 0 } + + +--- +"indices shards total count test": + + - skip: + features: ["allowed_warnings", arbitrary_key] + version: " - 7.99.99" + reason: "total shard count added in version 8.0" + + - do: + indices.create: + index: index1 + body: + settings: + number_of_shards: "5" + number_of_replicas: "0" + + - do: + indices.create: + index: index2 + body: + settings: + number_of_shards: "3" + number_of_replicas: "1" + + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: { metric: _all } + + - gte: { nodes.$node_id.indices.shards.total_count: 1 } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index a05cd5e6fd48e..fc2594752645d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -735,7 +735,7 @@ public void testEncodeDecodeCommonStats() throws IOException { public void testFlagOrdinalOrder() { Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh, Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments, - Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk}; + Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk, Flag.Shards}; assertThat(flags.length, equalTo(Flag.values().length)); for (int i = 0; i < flags.length; i++) { @@ -914,6 +914,10 @@ private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean s case Bulk: builder.setBulk(set); break; + case Shards: + // We don't actually expose shards in IndexStats, but this test fails if it isn't handled + builder.request().flags().set(Flag.Shards, set); + break; default: fail("new flag? " + flag); break; @@ -956,6 +960,8 @@ private static boolean isSet(Flag flag, CommonStats response) { return response.getRecoveryStats() != null; case Bulk: return response.getBulk() != null; + case Shards: + return response.getShards() != null; default: fail("new flag? " + flag); return false; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 772cb100a534c..6d7cb1c0c919a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexingStats; +import org.elasticsearch.index.shard.ShardCountStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; @@ -96,6 +97,9 @@ public class CommonStats implements Writeable, ToXContentFragment { @Nullable public BulkStats bulk; + @Nullable + public ShardCountStats shards; + public CommonStats() { this(CommonStatsFlags.NONE); } @@ -156,6 +160,9 @@ public CommonStats(CommonStatsFlags flags) { case Bulk: bulk = new BulkStats(); break; + case Shards: + shards = new ShardCountStats(); + break; default: throw new IllegalStateException("Unknown Flag: " + flag); } @@ -218,6 +225,10 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C case Bulk: bulk = indexShard.bulkStats(); break; + case Shards: + // Setting to 1 because the single IndexShard passed to this method implies 1 shard + shards = new ShardCountStats(1); + break; default: throw new IllegalStateException("Unknown Flag: " + flag); } @@ -246,6 +257,7 @@ public CommonStats(StreamInput in) throws IOException { recoveryStats = in.readOptionalWriteable(RecoveryStats::new); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { bulk = in.readOptionalWriteable(BulkStats::new); + shards = in.readOptionalWriteable(ShardCountStats::new); } } @@ -269,6 +281,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(recoveryStats); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeOptionalWriteable(bulk); + out.writeOptionalWriteable(shards); } } @@ -410,6 +423,14 @@ public void add(CommonStats stats) { } else { bulk.add(stats.getBulk()); } + if (stats.shards != null) { + if (shards == null) { + shards = stats.shards; + } + else { + shards = shards.add(stats.shards); + } + } } @Nullable @@ -497,6 +518,11 @@ public BulkStats getBulk() { return bulk; } + @Nullable + public ShardCountStats getShards() { + return shards; + } + /** * Utility method which computes total memory by adding * FieldData, PercolatorCache, Segments (memory, index writer, version map) @@ -522,7 +548,7 @@ public ByteSizeValue getTotalMemory() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { final Stream stream = Arrays.stream(new ToXContent[] { - docs, store, indexing, get, search, merge, refresh, flush, warmer, queryCache, + docs, shards, store, indexing, get, search, merge, refresh, flush, warmer, queryCache, fieldData, completion, segments, translog, requestCache, recoveryStats, bulk}) .filter(Objects::nonNull); for (ToXContent toXContent : ((Iterable)stream::iterator)) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index 70e677cb6e055..dd89fa3323d02 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -217,7 +217,8 @@ public enum Flag { // 14 was previously used for Suggest RequestCache("request_cache", 15), Recovery("recovery", 16), - Bulk("bulk", 17); + Bulk("bulk", 17), + Shards("shards", 18); private final String restName; private final int index; diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java new file mode 100644 index 0000000000000..5ea65c35691ea --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; + +public class ShardCountStats implements Writeable, ToXContentFragment { + + private final long totalCount; + + public ShardCountStats() { + totalCount = 0; + } + + public ShardCountStats(StreamInput in) throws IOException { + totalCount = in.readVLong(); + } + + public ShardCountStats(long totalCount) { + this.totalCount = totalCount; + } + + public long getTotalCount() { + return this.totalCount; + } + + public ShardCountStats add(@Nullable ShardCountStats other) { + return new ShardCountStats(this.totalCount + (other == null ? 0 : other.totalCount)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.SHARDS); + builder.field(Fields.TOTAL_COUNT, totalCount); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(totalCount); + } + + static final class Fields { + static final String SHARDS = "shards"; + static final String TOTAL_COUNT = "total_count"; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public boolean equals(Object o) { + return (o instanceof ShardCountStats) && totalCount == ((ShardCountStats) o).totalCount; + } + + @Override + public int hashCode() { + return Long.hashCode(totalCount); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java new file mode 100644 index 0000000000000..9e7d130182032 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.junit.Assert; + +class ShardCountStatsTest extends AbstractWireSerializingTestCase { + + public void testAdd() { + ShardCountStats shardStats1 = new ShardCountStats(5); + ShardCountStats shardStats2 = new ShardCountStats(8); + ShardCountStats combinedShardStats = shardStats1.add(shardStats2); + Assert.assertEquals(13, combinedShardStats.getTotalCount()); + Assert.assertEquals(5, shardStats1.getTotalCount()); + Assert.assertEquals(8, shardStats2.getTotalCount()); + ShardCountStats noCountGiven = new ShardCountStats(); + Assert.assertEquals(0, noCountGiven.getTotalCount()); + Assert.assertEquals(8, shardStats2.add(null).getTotalCount()); + Assert.assertEquals(8, shardStats2.getTotalCount()); + } + + @Override + protected Writeable.Reader instanceReader() { + return ShardCountStats::new; + } + + @Override + protected ShardCountStats createTestInstance() { + return new ShardCountStats(randomIntBetween(0, Integer.MAX_VALUE)); + } +}