Skip to content

Commit

Permalink
Optimize parse and collections
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielmitterdorfer
danielmitterdorfer committed Jan 23, 2024
1 parent f1a2003 commit bbc03b4
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 20 deletions.
28 changes: 16 additions & 12 deletions server/src/main/java/org/elasticsearch/telemetry/tracing/TraceLogger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,35 +19,39 @@ public final class TraceLogger {
private static final Logger logger = LogManager.getLogger("tracing");

static {
//Loggers.setLevel(logger, Level.TRACE);
// Loggers.setLevel(logger, Level.TRACE);
}

private TraceLogger() {
// no instances intended
}

public static void start(ThreadContext traceContext, String message) {
log(traceContext, "start " + message);
if (logger.isInfoEnabled()) {
log(traceContext, "start " + message);
}
}

public static void stop(ThreadContext traceContext, String message) {
log(traceContext, "stop " + message);
if (logger.isInfoEnabled()) {
log(traceContext, "stop " + message);
}
}

public static void schedule(ThreadContext traceContext, String message) {
log(traceContext, "schedule " + message);
}

public static void log(ThreadContext traceContext, String message) {
if (logger.isInfoEnabled()) {
// log only traces initiated by end-user requests
//if (traceContext != null && traceContext.getRequestHeadersOnly().isEmpty() == false) {
// TODO: Add thread context (which request id?)
logger.info((Message) Trace.of(message));
//}
log(traceContext, "schedule " + message);
}
}

private static void log(ThreadContext traceContext, String message) {
// log only traces initiated by end-user requests
// if (traceContext != null && traceContext.getRequestHeadersOnly().isEmpty() == false) {
// TODO: Add thread context (which request id?)
logger.info((Message) Trace.of(message));
// }
}

private static class Trace extends SimpleMessage implements TimestampMessage {
private final long timestamp;

Expand Down
2 changes: 0 additions & 2 deletions ...rc/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@

package org.elasticsearch.xpack.profiling;

import org.apache.lucene.tests.util.LuceneTestCase;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;

import java.util.List;

@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103809")
public class GetStackTracesActionIT extends ProfilingTestCase {
public void testGetStackTracesUnfiltered() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, null, null, null, null, null, null, null, null);
Expand Down
41 changes: 41 additions & 0 deletions x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackTrace.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.xpack.profiling;

import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.xcontent.ObjectPath;
import org.elasticsearch.xcontent.ToXContentObject;
import org.elasticsearch.xcontent.XContentBuilder;
Expand Down Expand Up @@ -190,6 +191,46 @@ static String getFileIDFromStackFrameID(String frameID) {
return frameID.substring(0, 21) + SAFE_BASE64_ENCODER.charAt(frameID.charAt(21) & 0x30);
}

public static StackTrace fromBytes(BytesReference source) {
// assume valid source
String sourceAsString = source.utf8ToString();
int startFrames = "{\"Stacktrace\":{\"frame\":{\"ids\":\"".length();
int endFrames = sourceAsString.indexOf('"', startFrames);

String inputFrameIDs = sourceAsString.substring(startFrames, endFrames);

int startTypes = endFrames + "\",\"types\":\"".length();
int endTypes = sourceAsString.indexOf('"', startTypes);

String inputFrameTypes = sourceAsString.substring(startTypes, endTypes);
int countsFrameIDs = inputFrameIDs.length() / BASE64_FRAME_ID_LENGTH;

List<String> fileIDs = new ArrayList<>(countsFrameIDs);
List<String> frameIDs = new ArrayList<>(countsFrameIDs);
List<Integer> addressOrLines = new ArrayList<>(countsFrameIDs);

// Step 1: Convert the base64-encoded frameID list into two separate
// lists (frame IDs and file IDs), both of which are also base64-encoded.
//
// To get the frame ID, we grab the next 32 bytes.
//
// To get the file ID, we grab the first 22 bytes of the frame ID.
// However, since the file ID is base64-encoded using 21.33 bytes
// (16 * 4 / 3), then the 22 bytes have an extra 4 bits from the
// address (see diagram in definition of EncodedStackTrace).
for (int i = 0, pos = 0; i < countsFrameIDs; i++, pos += BASE64_FRAME_ID_LENGTH) {
String frameID = inputFrameIDs.substring(pos, pos + BASE64_FRAME_ID_LENGTH);
frameIDs.add(frameID);
fileIDs.add(getFileIDFromStackFrameID(frameID));
addressOrLines.add(getAddressFromStackFrameID(frameID));
}

// Step 2: Convert the run-length byte encoding into a list of uint8s.
List<Integer> typeIDs = runLengthDecodeBase64Url(inputFrameTypes, inputFrameTypes.length(), countsFrameIDs);

return new StackTrace(addressOrLines, fileIDs, frameIDs, typeIDs, 0, 0, 0);
}

public static StackTrace fromSource(Map<String, Object> source) {
String inputFrameIDs = ObjectPath.eval(PATH_FRAME_IDS, source);
String inputFrameTypes = ObjectPath.eval(PATH_FRAME_TYPES, source);
Expand Down
13 changes: 7 additions & 6 deletions ...filing/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
Expand Down Expand Up @@ -430,7 +429,7 @@ private void retrieveStackTraces(
// We need to expect a set of slices for each resolved index, plus one for the host metadata.
slicedEventIds.size() * indices.size() + (uniqueHostIDs.isEmpty() ? 0 : 1),
uniqueHostIDs.size(),
(s, e) -> retrieveStackTraceDetails(submitTask, clusterState, client, responseBuilder, s, e, submitListener)
(s, e) -> retrieveStackTraceDetails(submitTask, clusterState, client, responseBuilder, s, e, submitListener)
);
for (List<String> slice : slicedEventIds) {
mget(client, indices, slice, ActionListener.wrap(handler::onStackTraceResponse, submitListener::onFailure));
Expand Down Expand Up @@ -489,8 +488,9 @@ private static final class StackTraceHandler {
// sort items lexicographically to access Lucene's term dictionary more efficiently when issuing an mget request.
// The term dictionary is lexicographically sorted and using the same order reduces the number of page faults
// needed to load it.
private final Set<String> stackFrameIds = new ConcurrentSkipListSet<>();
private final Set<String> executableIds = new ConcurrentSkipListSet<>();
// TODO: Use a concurrent version for production code - for the prototype we don't need that as it is only one thread
private final Set<String> stackFrameIds = new HashSet<>();
private final Set<String> executableIds = new HashSet<>();
private final AtomicInteger totalFrames = new AtomicInteger();
private final StopWatch watch = new StopWatch("retrieveStackTraces");
private final StopWatch hostsWatch = new StopWatch("retrieveHostMetadata");
Expand Down Expand Up @@ -529,7 +529,8 @@ public void onStackTraceResponse(MultiGetResponse multiGetItemResponses) {
String id = trace.getId();
// Duplicates are expected as we query multiple indices - do a quick pre-check before we deserialize a response
if (stackTracePerId.containsKey(id) == false) {
StackTrace stacktrace = StackTrace.fromSource(trace.getResponse().getSource());
// StackTrace stacktrace = StackTrace.fromSource(trace.getResponse().getSource());
StackTrace stacktrace = StackTrace.fromBytes(trace.getResponse().getSourceAsBytesRef());
// Guard against concurrent access and ensure we only handle each item once
if (stackTracePerId.putIfAbsent(id, stacktrace) == null) {
totalFrames.addAndGet(stacktrace.frameIds.size());
Expand Down Expand Up @@ -615,7 +616,7 @@ public void mayFinish() {
TraceLogger.stop(null, "process stacktrace response finish");
TraceLogger.stop(null, "process stacktrace response");
TraceLogger.stop(null, "retrieveStackTraces");
detailsHandler.accept(new ArrayList<>(stackFrameIds),new ArrayList<>(executableIds));
detailsHandler.accept(new ArrayList<>(stackFrameIds), new ArrayList<>(executableIds));
/*
retrieveStackTraceDetails(new ArrayList<>(stackFrameIds),
new ArrayList<>(executableIds));
Expand Down
11 changes: 11 additions & 0 deletions x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/StackTraceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.xpack.profiling;

import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.EqualsHashCodeTestUtils;
Expand Down Expand Up @@ -79,6 +80,16 @@ public void testCreateFromSource() {
assertEquals(List.of(2), stackTrace.typeIds);
}

public void testCreateFromBytes() {
//BytesReference src = new BytesArray("{\"Stacktrace\":{\"frame\":{\"ids\":\"634wiWh6F21tPpXr0Zz3mgAAAAAAEfFi8NlMClggx8jaziUTJXlmWAAAAAAAAIYIZSkKN3zNxr0HYuO2pqe5hQAAAAAAwcBwZSkKN3zNxr0HYuO2pqe5hQAAAAAA5ECvZSkKN3zNxr0HYuO2pqe5hQAAAAAA4_9_ZSkKN3zNxr0HYuO2pqe5hQAAAAAAj7b5ZSkKN3zNxr0HYuO2pqe5hQAAAAAAgwXKZSkKN3zNxr0HYuO2pqe5hQAAAAAAgu3UAAAAAAAAV4sAAAAAAAAAHezOBBlhpr8qZgY89pr05YIxi0DTL7hyTAAAAAAAAAALzZZ6VCjFYAFVAKtY0XlyPwAAAAAAAAAFySPx-89oJ6TfXYn-uir7mQAAAAAAAABcBtfAnw52Zh-pzF1hN6odGgAAAAAAAAABDJLJRrJNARIRyzxCnxa4WQAAAAAAAAAYv6rQl1rwEGF72NGhRc1i-wAAAAAAAAAMZJExOMsbBfy8WqRgcylCLwAAAAAAAABpLSgILL0Q8I6llgRQX49QDQAAAAAAAADHbUctjbWcypk0wJUtGkKmFgAAAAAAAADJJwuJ_YrJlPmIg66vmsQiFAAAAAAAAAAz6KK4nudrIWi-u7NWSV5S7QAAAAAAAABnEBqXZsoPGK2F0IF3gcVlzwAAAAAAAAAaIunvE7sWPSz4YCgk0UVsFAAAAAAAAAAFObVu4EglQ3fhG0H69eyyrwAAAAAAAAAYW80Dopg4GIca7JXLWHFwVAAAAAAAAAA6PKIQdknzONDqr2S2uRbbjQAAAAAAAACCRJOpyLKTKlv2enKt_xrNSwAAAAAAAAA73-s2ANt1vNwpWiAFGYs2eAAAAAAAAAAFdWyxTJ01gHAXEF1z9sDOKwAAAAAAAAAOBtfAnw52Zh-pzF1hN6odGgAAAAAAAAABJG_5wUb6Q1WN6hfRJFcqEwAAAAAAAAATAo_nvps-se9i_u9kVJi0LQAAAAAAAAAKXrSW2Ro9Ws10jZmj20O0hQAAAAAAAAADHSftxLHu5xLDzJX5F8kBXAAAAAAAAAGVBASYhiEtRRk5wvHC9yWtCgAAAAAAAABV2bJMveL01ajlzm-wdxWvUAAAAAAAAAAIy6xtuPPjpVFeK4CXqSN6fQAAAAAAAAHc6TUXIWC2WcF-_43xKp-wtQAAAAAAAAETuaLlXo_jail-f0Vd8f8fCwAAAAAAAACDIir7eGdVZsjNItLfykxs3gAAAAAAAABaNWk7GQCtDStIuwl83iGqTwAAAAAAAAAv2CwcPQcRkPoyzD89B8CtBQAAAAAAAAB-OoUyVxHN2rYPjLirZK4ihQAAAAAAAAAGf_uYAlUc4Mk4nBdvFkWUPgAAAAAAAAMEGASd-GxTI3CnNP2dmtJ0gQAAAAAAAAHeLeaqrhFv22rb1wzUlEMwAgAAAAAAAAAMkbApFhDbueqDTdsjbbOMUQAAAAAAAAAg7z0vK5_lN4w4BvVehfj4wQAAAAAAAAALTrTEyZn7B30gBbM8MLULSQAAAAAAAAA5ZQ0AxrpV-kvXnFTqilFT7wAAAAAAAAAfK8SfjEwLlz1-kQVCzD5OrAAAAAAAAAAOp_xoiXGCsXgpBNHrp24JKQAAAAAAAAAV5jDiO_Lgv2e10LojWCULCwAAAAAAAAAFOX6pd1GZ1KHmZAikJOZD-wAAAAAAAABoh6nIgR7I9-N28cV-Wv3K3AAAAAAAAAFBCqi1_91FMYO-otI3ESGKwAAAAAAAAAALVo9H_8YeBmc0_i3CIsMkrwAAAAAAAAAAJsaw4aRwbsrSC4mkOwhwQQAAAAAAANJa8NlMClggx8jaziUTJXlmWAAAAAAAATrTlHp5_WAgpLy2alrUVab6HAAAAAAAwACLlHp5_WAgpLy2alrUVab6HAAAAAAAAEIGlHp5_WAgpLy2alrUVab6HAAAAAAALSp_lHp5_WAgpLy2alrUVab6HAAAAAAALSjZlHp5_WAgpLy2alrUVab6HAAAAAAALpP6lHp5_WAgpLy2alrUVab6HAAAAAAALmmhlHp5_WAgpLy2alrUVab6HAAAAAAALkVrlHp5_WAgpLy2alrUVab6HAAAAAAALkColHp5_WAgpLy2alrUVab6HAAAAAAALjanlHp5_WAgpLy2alrUVab6HAAAAAAAL0cjlHp5_WAgpLy2alrUVab6HAAAAAAAD8cD\",\"types\":\"CAMwBQIDCwQ\"}},\"ecs\":{\"version\":\"1.12.0\"}}");
BytesReference src = new BytesArray("{\"Stacktrace\":{\"frame\":{\"ids\":\"AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u\",\"types\":\"AQI\"}},\"ecs\":{\"version\":\"1.12.0\"}}");
StackTrace stackTrace = StackTrace.fromBytes(src);
assertEquals(List.of("AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u"), stackTrace.frameIds);
assertEquals(List.of("AAAAAAAAAAUAAAAAAAAB3g"), stackTrace.fileIds);
assertEquals(List.of(1027822), stackTrace.addressOrLines);
assertEquals(List.of(2), stackTrace.typeIds);
}

public void testToXContent() throws IOException {
XContentType contentType = randomFrom(XContentType.values());
XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
Expand Down

0 comments on commit bbc03b4

Please sign in to comment.