Skip to content

Commit

Permalink
Adding shard count to node stats api (#75433)
Browse files Browse the repository at this point in the history
* Adding shard count to _nodes/stats api

Added a shards section to each node returned by the _nodes/stats api. Currently this new section only contains a total count of all shards on the node.
  • Loading branch information
masseyke authored Jul 27, 2021
1 parent b5016d9 commit ddc3b37
Show file tree
Hide file tree
Showing 8 changed files with 241 additions and 5 deletions.
12 changes: 12 additions & 0 deletions docs/reference/cluster/nodes-stats.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -959,6 +959,18 @@ Time by which recovery operations were delayed due to throttling.
Time in milliseconds
recovery operations were delayed due to throttling.
=======
`shards`::
(object)
Contains statistics about all shards assigned to the node.
+
.Properties of `shards`
[%collapsible%open]
=======
`total_count`::
(integer)
The total number of shards assigned to the node.
=======
======

[[cluster-nodes-stats-api-response-body-os]]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@
"segments",
"store",
"warmer",
"bulk"
"bulk",
"shards"
],
"description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified."
}
Expand Down Expand Up @@ -175,7 +176,8 @@
"segments",
"store",
"warmer",
"bulk"
"bulk",
"shards"
],
"description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified."
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@
- is_false: nodes.$node_id.indices.segments
- is_false: nodes.$node_id.indices.translog
- is_false: nodes.$node_id.indices.recovery
- is_false: nodes.$node_id.indices.shards

---
"Metric - multi":
Expand Down Expand Up @@ -166,6 +167,7 @@
- is_false: nodes.$node_id.indices.segments
- is_false: nodes.$node_id.indices.translog
- is_true: nodes.$node_id.indices.recovery
- is_false: nodes.$node_id.indices.shards

---
"Metric - _all include_segment_file_sizes":
Expand Down Expand Up @@ -223,6 +225,7 @@
- is_true: nodes.$node_id.indices.segments
- is_false: nodes.$node_id.indices.translog
- is_false: nodes.$node_id.indices.recovery
- is_false: nodes.$node_id.indices.shards
- is_true: nodes.$node_id.indices.segments.file_sizes

---
Expand Down Expand Up @@ -254,6 +257,7 @@
- is_true: nodes.$node_id.indices.segments
- is_false: nodes.$node_id.indices.translog
- is_false: nodes.$node_id.indices.recovery
- is_false: nodes.$node_id.indices.shards

---
"Metric - _all include_unloaded_segments":
Expand Down Expand Up @@ -315,3 +319,72 @@
- gte: { nodes.$node_id.http.clients.0.request_size_bytes: 0 }
# values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often
# null and cannot be tested here

---
"Metric - blank for indices shards":
- skip:
features: [arbitrary_key]
version: " - 7.99.99"
reason: "total shard count added in version 8.0"
- do:
nodes.info: {}
- set:
nodes._arbitrary_key_: node_id

- do:
nodes.stats: {}

- is_true: nodes.$node_id.indices.shards
- match: { nodes.$node_id.indices.shards.total_count: 0 }

---
"Metric - _all for indices shards":
- skip:
features: [arbitrary_key]
version: " - 7.99.99"
reason: "total shard count added in version 8.0"
- do:
nodes.info: {}
- set:
nodes._arbitrary_key_: node_id

- do:
nodes.stats: { metric: _all }

- is_true: nodes.$node_id.indices.shards
- match: { nodes.$node_id.indices.shards.total_count: 0 }


---
"indices shards total count test":

- skip:
features: ["allowed_warnings", arbitrary_key]
version: " - 7.99.99"
reason: "total shard count added in version 8.0"

- do:
indices.create:
index: index1
body:
settings:
number_of_shards: "5"
number_of_replicas: "0"

- do:
indices.create:
index: index2
body:
settings:
number_of_shards: "3"
number_of_replicas: "1"

- do:
nodes.info: {}
- set:
nodes._arbitrary_key_: node_id

- do:
nodes.stats: { metric: _all }

- gte: { nodes.$node_id.indices.shards.total_count: 1 }
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,7 @@ public void testEncodeDecodeCommonStats() throws IOException {
public void testFlagOrdinalOrder() {
Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh,
Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments,
Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk};
Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk, Flag.Shards};

assertThat(flags.length, equalTo(Flag.values().length));
for (int i = 0; i < flags.length; i++) {
Expand Down Expand Up @@ -914,6 +914,10 @@ private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean s
case Bulk:
builder.setBulk(set);
break;
case Shards:
// We don't actually expose shards in IndexStats, but this test fails if it isn't handled
builder.request().flags().set(Flag.Shards, set);
break;
default:
fail("new flag? " + flag);
break;
Expand Down Expand Up @@ -956,6 +960,8 @@ private static boolean isSet(Flag flag, CommonStats response) {
return response.getRecoveryStats() != null;
case Bulk:
return response.getBulk() != null;
case Shards:
return response.getShards() != null;
default:
fail("new flag? " + flag);
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.IndexingStats;
import org.elasticsearch.index.shard.ShardCountStats;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.index.translog.TranslogStats;
import org.elasticsearch.index.warmer.WarmerStats;
Expand Down Expand Up @@ -96,6 +97,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
@Nullable
public BulkStats bulk;

@Nullable
public ShardCountStats shards;

public CommonStats() {
this(CommonStatsFlags.NONE);
}
Expand Down Expand Up @@ -156,6 +160,9 @@ public CommonStats(CommonStatsFlags flags) {
case Bulk:
bulk = new BulkStats();
break;
case Shards:
shards = new ShardCountStats();
break;
default:
throw new IllegalStateException("Unknown Flag: " + flag);
}
Expand Down Expand Up @@ -218,6 +225,10 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
case Bulk:
bulk = indexShard.bulkStats();
break;
case Shards:
// Setting to 1 because the single IndexShard passed to this method implies 1 shard
shards = new ShardCountStats(1);
break;
default:
throw new IllegalStateException("Unknown Flag: " + flag);
}
Expand Down Expand Up @@ -246,6 +257,7 @@ public CommonStats(StreamInput in) throws IOException {
recoveryStats = in.readOptionalWriteable(RecoveryStats::new);
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
bulk = in.readOptionalWriteable(BulkStats::new);
shards = in.readOptionalWriteable(ShardCountStats::new);
}
}

Expand All @@ -269,6 +281,7 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalWriteable(recoveryStats);
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
out.writeOptionalWriteable(bulk);
out.writeOptionalWriteable(shards);
}
}

Expand Down Expand Up @@ -410,6 +423,14 @@ public void add(CommonStats stats) {
} else {
bulk.add(stats.getBulk());
}
if (stats.shards != null) {
if (shards == null) {
shards = stats.shards;
}
else {
shards = shards.add(stats.shards);
}
}
}

@Nullable
Expand Down Expand Up @@ -497,6 +518,11 @@ public BulkStats getBulk() {
return bulk;
}

@Nullable
public ShardCountStats getShards() {
return shards;
}

/**
* Utility method which computes total memory by adding
* FieldData, PercolatorCache, Segments (memory, index writer, version map)
Expand All @@ -522,7 +548,7 @@ public ByteSizeValue getTotalMemory() {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
final Stream<ToXContent> stream = Arrays.stream(new ToXContent[] {
docs, store, indexing, get, search, merge, refresh, flush, warmer, queryCache,
docs, shards, store, indexing, get, search, merge, refresh, flush, warmer, queryCache,
fieldData, completion, segments, translog, requestCache, recoveryStats, bulk})
.filter(Objects::nonNull);
for (ToXContent toXContent : ((Iterable<ToXContent>)stream::iterator)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,8 @@ public enum Flag {
// 14 was previously used for Suggest
RequestCache("request_cache", 15),
Recovery("recovery", 16),
Bulk("bulk", 17);
Bulk("bulk", 17),
Shards("shards", 18);

private final String restName;
private final int index;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.index.shard;

import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.core.Nullable;

import java.io.IOException;

public class ShardCountStats implements Writeable, ToXContentFragment {

private final long totalCount;

public ShardCountStats() {
totalCount = 0;
}

public ShardCountStats(StreamInput in) throws IOException {
totalCount = in.readVLong();
}

public ShardCountStats(long totalCount) {
this.totalCount = totalCount;
}

public long getTotalCount() {
return this.totalCount;
}

public ShardCountStats add(@Nullable ShardCountStats other) {
return new ShardCountStats(this.totalCount + (other == null ? 0 : other.totalCount));
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.SHARDS);
builder.field(Fields.TOTAL_COUNT, totalCount);
builder.endObject();
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(totalCount);
}

static final class Fields {
static final String SHARDS = "shards";
static final String TOTAL_COUNT = "total_count";
}

@Override
public String toString() {
return Strings.toString(this);
}

@Override
public boolean equals(Object o) {
return (o instanceof ShardCountStats) && totalCount == ((ShardCountStats) o).totalCount;
}

@Override
public int hashCode() {
return Long.hashCode(totalCount);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.index.shard;

import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.junit.Assert;

class ShardCountStatsTest extends AbstractWireSerializingTestCase<ShardCountStats> {

public void testAdd() {
ShardCountStats shardStats1 = new ShardCountStats(5);
ShardCountStats shardStats2 = new ShardCountStats(8);
ShardCountStats combinedShardStats = shardStats1.add(shardStats2);
Assert.assertEquals(13, combinedShardStats.getTotalCount());
Assert.assertEquals(5, shardStats1.getTotalCount());
Assert.assertEquals(8, shardStats2.getTotalCount());
ShardCountStats noCountGiven = new ShardCountStats();
Assert.assertEquals(0, noCountGiven.getTotalCount());
Assert.assertEquals(8, shardStats2.add(null).getTotalCount());
Assert.assertEquals(8, shardStats2.getTotalCount());
}

@Override
protected Writeable.Reader<ShardCountStats> instanceReader() {
return ShardCountStats::new;
}

@Override
protected ShardCountStats createTestInstance() {
return new ShardCountStats(randomIntBetween(0, Integer.MAX_VALUE));
}
}

0 comments on commit ddc3b37

Please sign in to comment.