Skip to content

Commit

Permalink
Showing 4 changed files with 289 additions and 23 deletions.
Original file line number Diff line number Diff line change
@@ -225,6 +225,36 @@
- is_false: nodes.$node_id.indices.recovery
- is_true: nodes.$node_id.indices.segments.file_sizes

---
"Metric - segments include_unloaded_segments":
- skip:
features: [arbitrary_key]
version: " - 7.12.99"
reason: "support for include_unloaded_segments only added in 7.13"
- do:
nodes.info: {}
- set:
nodes._arbitrary_key_: node_id

- do:
nodes.stats: { metric: indices, index_metric: segments, include_unloaded_segments: true }

- is_false: nodes.$node_id.indices.docs
- is_false: nodes.$node_id.indices.store
- is_false: nodes.$node_id.indices.indexing
- is_false: nodes.$node_id.indices.get
- is_false: nodes.$node_id.indices.search
- is_false: nodes.$node_id.indices.merges
- is_false: nodes.$node_id.indices.refresh
- is_false: nodes.$node_id.indices.flush
- is_false: nodes.$node_id.indices.warmer
- is_false: nodes.$node_id.indices.query_cache
- is_false: nodes.$node_id.indices.fielddata
- is_false: nodes.$node_id.indices.completion
- is_true: nodes.$node_id.indices.segments
- is_false: nodes.$node_id.indices.translog
- is_false: nodes.$node_id.indices.recovery

---
"Metric - _all include_unloaded_segments":
- skip:
@@ -256,31 +286,31 @@
- is_true: nodes.$node_id.indices.recovery

---
"Metric - segments include_unloaded_segments":
"Metric - http":
- skip:
features: [arbitrary_key]
version: " - 7.12.99"
reason: "support for include_unloaded_segments only added in 7.13"
version: " - 7.99.99"
reason: "change after backporting"
- do:
nodes.info: {}
- set:
nodes._arbitrary_key_: node_id

- do:
nodes.stats: { metric: indices, index_metric: segments, include_unloaded_segments: true }
nodes.stats: { metric: http }

- is_false: nodes.$node_id.indices.docs
- is_false: nodes.$node_id.indices.store
- is_false: nodes.$node_id.indices.indexing
- is_false: nodes.$node_id.indices.get
- is_false: nodes.$node_id.indices.search
- is_false: nodes.$node_id.indices.merges
- is_false: nodes.$node_id.indices.refresh
- is_false: nodes.$node_id.indices.flush
- is_false: nodes.$node_id.indices.warmer
- is_false: nodes.$node_id.indices.query_cache
- is_false: nodes.$node_id.indices.fielddata
- is_false: nodes.$node_id.indices.completion
- is_true: nodes.$node_id.indices.segments
- is_false: nodes.$node_id.indices.translog
- is_false: nodes.$node_id.indices.recovery
- is_true: nodes.$node_id
- gte: { nodes.$node_id.http.current_open: 1 }
- gte: { nodes.$node_id.http.total_opened: 1 }
- is_true: nodes.$node_id.http.clients
- gte: { nodes.$node_id.http.clients.0.id: 1 }
- match: { nodes.$node_id.http.clients.0.agent: "/.*/" }
- match: { nodes.$node_id.http.clients.0.local_address: "/.*/" }
- match: { nodes.$node_id.http.clients.0.remote_address: "/.*/" }
- is_true: nodes.$node_id.http.clients.0.last_uri
- gte: { nodes.$node_id.http.clients.0.opened_time_millis: 1614028911317 }
- gte: { nodes.$node_id.http.clients.0.last_request_time_millis: 1614028911317 }
- gte: { nodes.$node_id.http.clients.0.request_count: 1 }
- gte: { nodes.$node_id.http.clients.0.request_size_bytes: 0 }
# values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often
# null and cannot be tested here
Original file line number Diff line number Diff line change
@@ -45,8 +45,10 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;

@@ -60,6 +62,9 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
private static final Logger logger = LogManager.getLogger(AbstractHttpServerTransport.class);
private static final ActionListener<Void> NO_OP = ActionListener.wrap(() -> {});

private static final long PRUNE_THROTTLE_INTERVAL = TimeUnit.SECONDS.toMillis(60);
private static final long MAX_CLIENT_STATS_AGE = TimeUnit.MINUTES.toMillis(5);

protected final Settings settings;
public final HttpHandlingSettings handlingSettings;
protected final NetworkService networkService;
@@ -78,10 +83,12 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
private final AtomicLong totalChannelsAccepted = new AtomicLong();
private final Set<HttpChannel> httpChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
private final Set<HttpServerChannel> httpServerChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
private final Map<Integer, HttpStats.ClientStats> httpChannelStats = new ConcurrentHashMap<>();

private final HttpTracer tracer;

private volatile long slowLogThresholdMs;
protected volatile long lastClientStatsPruneTime;

protected AbstractHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, ClusterSettings clusterSettings) {
@@ -128,7 +135,26 @@ public HttpInfo info() {

@Override
public HttpStats stats() {
return new HttpStats(httpChannels.size(), totalChannelsAccepted.get());
pruneClientStats(false);
return new HttpStats(new ArrayList<>(httpChannelStats.values()), httpChannels.size(), totalChannelsAccepted.get());
}

/**
* Prunes client stats of entries that have been disconnected for more than five minutes.
*
* @param throttled When true, executes the prune process only if more than 60 seconds has elapsed since the last execution.
*/
void pruneClientStats(boolean throttled) {
if (throttled == false || (threadPool.relativeTimeInMillis() - lastClientStatsPruneTime > PRUNE_THROTTLE_INTERVAL)) {
long nowMillis = threadPool.absoluteTimeInMillis();
for (var statsEntry : httpChannelStats.entrySet()) {
long closedTimeMillis = statsEntry.getValue().closedTimeMillis;
if (closedTimeMillis > 0 && (nowMillis - closedTimeMillis > MAX_CLIENT_STATS_AGE)) {
httpChannelStats.remove(statsEntry.getKey());
}
}
lastClientStatsPruneTime = threadPool.relativeTimeInMillis();
}
}

protected void bindServer() {
@@ -291,7 +317,23 @@ protected void serverAcceptedChannel(HttpChannel httpChannel) {
boolean addedOnThisCall = httpChannels.add(httpChannel);
assert addedOnThisCall : "Channel should only be added to http channel set once";
totalChannelsAccepted.incrementAndGet();
httpChannel.addCloseListener(ActionListener.wrap(() -> httpChannels.remove(httpChannel)));
httpChannelStats.put(
HttpStats.ClientStats.getChannelKey(httpChannel),
new HttpStats.ClientStats(threadPool.absoluteTimeInMillis())
);
httpChannel.addCloseListener(ActionListener.wrap(() -> {
try {
httpChannels.remove(httpChannel);
HttpStats.ClientStats clientStats = httpChannelStats.get(HttpStats.ClientStats.getChannelKey(httpChannel));
if (clientStats != null) {
clientStats.closedTimeMillis = threadPool.absoluteTimeInMillis();
}
} catch (Exception e) {
// the listener code about should never throw
logger.trace("error removing HTTP channel listener", e);
}
}));
pruneClientStats(true);
logger.trace(() -> new ParameterizedMessage("Http channel accepted: {}", httpChannel));
}

@@ -302,6 +344,7 @@ protected void serverAcceptedChannel(HttpChannel httpChannel) {
* @param httpChannel that received the http request
*/
public void incomingRequest(final HttpRequest httpRequest, final HttpChannel httpChannel) {
updateClientStats(httpRequest, httpChannel);
final long startTime = threadPool.relativeTimeInMillis();
try {
handleIncomingRequest(httpRequest, httpChannel, httpRequest.getInboundException());
@@ -315,6 +358,41 @@ public void incomingRequest(final HttpRequest httpRequest, final HttpChannel htt
}
}

void updateClientStats(final HttpRequest httpRequest, final HttpChannel httpChannel) {
HttpStats.ClientStats clientStats = httpChannelStats.get(HttpStats.ClientStats.getChannelKey(httpChannel));
if (clientStats != null) {
if (clientStats.agent == null) {
if (hasAtLeastOneHeaderValue(httpRequest, "x-elastic-product-origin")) {
clientStats.agent = httpRequest.getHeaders().get("x-elastic-product-origin").get(0);
} else if (hasAtLeastOneHeaderValue(httpRequest, "User-Agent")) {
clientStats.agent = httpRequest.getHeaders().get("User-Agent").get(0);
}
}
if (clientStats.localAddress == null) {
clientStats.localAddress = NetworkAddress.format(httpChannel.getLocalAddress());
clientStats.remoteAddress = NetworkAddress.format(httpChannel.getRemoteAddress());
}
if (clientStats.forwardedFor == null) {
if (hasAtLeastOneHeaderValue(httpRequest, "x-forwarded-for")) {
clientStats.forwardedFor = httpRequest.getHeaders().get("x-forwarded-for").get(0);
}
}
if (clientStats.opaqueId == null) {
if (hasAtLeastOneHeaderValue(httpRequest, "x-opaque-id")) {
clientStats.opaqueId = httpRequest.getHeaders().get("x-opaque-id").get(0);
}
}
clientStats.lastRequestTimeMillis = threadPool.absoluteTimeInMillis();
clientStats.lastUri = httpRequest.uri();
clientStats.requestCount.increment();
clientStats.requestSizeBytes.add(httpRequest.content().length());
}
}

private static boolean hasAtLeastOneHeaderValue(final HttpRequest request, final String header) {
return request.getHeaders().containsKey(header) && request.getHeaders().get(header).size() > 0;
}

// Visible for testing
void dispatchRequest(final RestRequest restRequest, final RestChannel channel, final Throwable badRequestCause) {
final ThreadContext threadContext = threadPool.getThreadContext();
139 changes: 138 additions & 1 deletion server/src/main/java/org/elasticsearch/http/HttpStats.java
Original file line number Diff line number Diff line change
@@ -8,33 +8,50 @@

package org.elasticsearch.http;

import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.LongAdder;

public class HttpStats implements Writeable, ToXContentFragment {

private final long serverOpen;
private final long totalOpen;
private final List<ClientStats> clientStats;

public HttpStats(long serverOpen, long totalOpened) {
public HttpStats(List<ClientStats> clientStats, long serverOpen, long totalOpened) {
this.clientStats = clientStats;
this.serverOpen = serverOpen;
this.totalOpen = totalOpened;
}

public HttpStats(long serverOpen, long totalOpened) {
this(List.of(), serverOpen, totalOpened);
}

public HttpStats(StreamInput in) throws IOException {
serverOpen = in.readVLong();
totalOpen = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
clientStats = in.readList(ClientStats::new);
} else {
clientStats = List.of();
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(serverOpen);
out.writeVLong(totalOpen);
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
out.writeList(clientStats);
}
}

public long getServerOpen() {
@@ -49,14 +66,134 @@ static final class Fields {
static final String HTTP = "http";
static final String CURRENT_OPEN = "current_open";
static final String TOTAL_OPENED = "total_opened";
static final String CLIENTS = "clients";
static final String CLIENT_ID = "id";
static final String CLIENT_AGENT = "agent";
static final String CLIENT_LOCAL_ADDRESS = "local_address";
static final String CLIENT_REMOTE_ADDRESS = "remote_address";
static final String CLIENT_LAST_URI = "last_uri";
static final String CLIENT_OPENED_TIME_MILLIS = "opened_time_millis";
static final String CLIENT_CLOSED_TIME_MILLIS = "closed_time_millis";
static final String CLIENT_LAST_REQUEST_TIME_MILLIS = "last_request_time_millis";
static final String CLIENT_REQUEST_COUNT = "request_count";
static final String CLIENT_REQUEST_SIZE_BYTES = "request_size_bytes";
static final String CLIENT_FORWARDED_FOR = "x_forwarded_for";
static final String CLIENT_OPAQUE_ID = "x_opaque_id";
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.HTTP);
builder.field(Fields.CURRENT_OPEN, serverOpen);
builder.field(Fields.TOTAL_OPENED, totalOpen);
builder.startArray(Fields.CLIENTS);
for (ClientStats clientStats : this.clientStats) {
clientStats.toXContent(builder, params);
}
builder.endArray();
builder.endObject();
return builder;
}

public static class ClientStats implements Writeable, ToXContentFragment {
final int id;
String agent;
String localAddress;
String remoteAddress;
String lastUri;
String forwardedFor;
String opaqueId;
long openedTimeMillis;
long closedTimeMillis = -1;
volatile long lastRequestTimeMillis = -1;
final LongAdder requestCount = new LongAdder();
final LongAdder requestSizeBytes = new LongAdder();

ClientStats(long openedTimeMillis) {
this.id = System.identityHashCode(this);
this.openedTimeMillis = openedTimeMillis;
}

// visible for testing
public ClientStats(String agent, String localAddress, String remoteAddress, String lastUri, String forwardedFor, String opaqueId,
long openedTimeMillis, long closedTimeMillis, long lastRequestTimeMillis, long requestCount, long requestSizeBytes) {
this.id = System.identityHashCode(this);
this.agent = agent;
this.localAddress = localAddress;
this.remoteAddress = remoteAddress;
this.lastUri = lastUri;
this.forwardedFor = forwardedFor;
this.opaqueId = opaqueId;
this.openedTimeMillis = openedTimeMillis;
this.closedTimeMillis = closedTimeMillis;
this.lastRequestTimeMillis = lastRequestTimeMillis;
this.requestCount.add(requestCount);
this.requestSizeBytes.add(requestSizeBytes);
}

ClientStats(StreamInput in) throws IOException {
this.id = in.readInt();
this.agent = in.readOptionalString();
this.localAddress = in.readString();
this.remoteAddress = in.readString();
this.lastUri = in.readString();
this.forwardedFor = in.readOptionalString();
this.opaqueId = in.readOptionalString();
this.openedTimeMillis = in.readLong();
this.closedTimeMillis = in.readLong();
this.lastRequestTimeMillis = in.readLong();
this.requestCount.add(in.readLong());
this.requestSizeBytes.add(in.readLong());
}

@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(Fields.CLIENT_ID, id);
if (agent != null) {
builder.field(Fields.CLIENT_AGENT, agent);
}
builder.field(Fields.CLIENT_LOCAL_ADDRESS, localAddress);
builder.field(Fields.CLIENT_REMOTE_ADDRESS, remoteAddress);
builder.field(Fields.CLIENT_LAST_URI, lastUri);
if (forwardedFor != null) {
builder.field(Fields.CLIENT_FORWARDED_FOR, forwardedFor);
}
if (opaqueId != null) {
builder.field(Fields.CLIENT_OPAQUE_ID, opaqueId);
}
builder.field(Fields.CLIENT_OPENED_TIME_MILLIS, openedTimeMillis);
if (closedTimeMillis != -1) {
builder.field(Fields.CLIENT_CLOSED_TIME_MILLIS, closedTimeMillis);
}
builder.field(Fields.CLIENT_LAST_REQUEST_TIME_MILLIS, lastRequestTimeMillis);
builder.field(Fields.CLIENT_REQUEST_COUNT, requestCount.longValue());
builder.field(Fields.CLIENT_REQUEST_SIZE_BYTES, requestSizeBytes.longValue());
builder.endObject();
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(id);
out.writeOptionalString(agent);
out.writeString(localAddress);
out.writeString(remoteAddress);
out.writeString(lastUri);
out.writeOptionalString(forwardedFor);
out.writeOptionalString(opaqueId);
out.writeLong(openedTimeMillis);
out.writeLong(closedTimeMillis);
out.writeLong(lastRequestTimeMillis);
out.writeLong(requestCount.longValue());
out.writeLong(requestSizeBytes.longValue());
}

/**
* Returns a key suitable for use in a hash table for the specified HttpChannel
*/
public static int getChannelKey(HttpChannel channel) {
// always use an identity-based hash code rather than one based on object state
return System.identityHashCode(channel);
}
}
}
Original file line number Diff line number Diff line change
@@ -430,7 +430,28 @@ public static NodeStats createNodeStats() {
}
TransportStats transportStats = frequently() ? new TransportStats(randomNonNegativeLong(), randomNonNegativeLong(),
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null;
HttpStats httpStats = frequently() ? new HttpStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
HttpStats httpStats = null;
if (frequently()) {
int numClients = randomIntBetween(0, 50);
List<HttpStats.ClientStats> clientStats = new ArrayList<>(numClients);
for (int k = 0; k < numClients; k++) {
var cs = new HttpStats.ClientStats(
randomAlphaOfLength(6),
randomAlphaOfLength(6),
randomAlphaOfLength(6),
randomAlphaOfLength(6),
randomAlphaOfLength(6),
randomAlphaOfLength(6),
randomNonNegativeLong(),
randomBoolean() ? -1 : randomNonNegativeLong(),
randomBoolean() ? -1 : randomNonNegativeLong(),
randomLongBetween(0, 100),
randomLongBetween(0, 99999999)
);
clientStats.add(cs);
}
httpStats = new HttpStats(clientStats, randomNonNegativeLong(), randomNonNegativeLong());
}
AllCircuitBreakerStats allCircuitBreakerStats = null;
if (frequently()) {
int numCircuitBreakerStats = randomIntBetween(0, 10);

0 comments on commit 14cee55

Please sign in to comment.