From 1b96e64dcf35fc9facea9e398b25c90ab7fb74fd Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 11 Jul 2022 12:05:24 +0800 Subject: [PATCH 1/4] improve logger, reduce logger cost --- .../xpack/rollup/v2/RollupShardIndexer.java | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 68e09e470cdce..299ae4b504bfe 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -180,7 +180,13 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon ) ); numFailed.addAndGet(failures.size()); - logger.error("Shard [{}] failed to populate rollup index. Failures: [{}]", indexShard.shardId(), failures); + logger.info( + "Shard [{}] failed to populate rollup index, failures summery: failed count: [{}], failed sample: [{}]", + indexShard.shardId(), + failures.size(), + failures.values().iterator().next() + ); + logger.trace("Shard [{}] failed to populate rollup index. Failures: [{}]", indexShard.shardId(), failures); } } @@ -189,7 +195,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) if (failure != null) { long items = request.numberOfActions(); numFailed.addAndGet(items); - logger.error(() -> format("Shard [%s] failed to populate rollup index.", indexShard.shardId()), failure); + logger.info(() -> format("Shard [%s] failed to populate rollup index.", indexShard.shardId()), failure); } } }; @@ -234,13 +240,15 @@ public void collect(int docId, long owningBucketOrd) throws IOException { final long timestamp = aggCtx.getTimestamp(); final long histoTimestamp = rounding.round(timestamp); - logger.trace( - "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", - docId, - DocValueFormat.TIME_SERIES_ID.format(tsid), - timestampFormat.format(timestamp), - timestampFormat.format(histoTimestamp) - ); + if (logger.isTraceEnabled()) { + logger.trace( + "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", + docId, + DocValueFormat.TIME_SERIES_ID.format(tsid), + timestampFormat.format(timestamp), + timestampFormat.format(histoTimestamp) + ); + } /* * Sanity checks to ensure that we receive documents in the correct order @@ -344,11 +352,14 @@ public RollupBucketBuilder init(BytesRef tsid, long timestamp) { this.timestamp = timestamp; this.docCount = 0; this.metricFieldProducers.values().stream().forEach(p -> p.reset()); - logger.trace( - "New bucket for _tsid: [{}], @timestamp: [{}]", - DocValueFormat.TIME_SERIES_ID.format(tsid), - timestampFormat.format(timestamp) - ); + if (logger.isTraceEnabled()) { + logger.trace( + "New bucket for _tsid: [{}], @timestamp: [{}]", + DocValueFormat.TIME_SERIES_ID.format(tsid), + timestampFormat.format(timestamp) + ); + } + return this; } From d2e3862345fa384a3f59852c9bbbb41c97a3e835 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 11 Jul 2022 13:03:25 +0800 Subject: [PATCH 2/4] add shard rollup took --- .../elasticsearch/xpack/rollup/v2/RollupShardIndexer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 299ae4b504bfe..a2aaf53690bd7 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -127,6 +127,7 @@ class RollupShardIndexer { } public RollupIndexerAction.ShardRollupResponse execute() throws IOException { + long startTime = System.currentTimeMillis(); BulkProcessor bulkProcessor = createBulkProcessor(); try (searcher; bulkProcessor) { // TODO: add cancellations @@ -137,12 +138,14 @@ public RollupIndexerAction.ShardRollupResponse execute() throws IOException { bucketCollector.postCollection(); } + logger.info( - "Shard {} successfully sent [{}], indexed [{}], failed [{}]", + "Shard [{}] successfully sent [{}], indexed [{}], failed [{}], took [{}]", indexShard.shardId(), numSent.get(), numIndexed.get(), - numFailed.get() + numFailed.get(), + TimeValue.timeValueMillis(System.currentTimeMillis() - startTime) ); if (numIndexed.get() != numSent.get()) { From b3c0f68cf41e0178c9033048a8814f5705fc99b2 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 11 Jul 2022 13:04:23 +0800 Subject: [PATCH 3/4] add shard rollup took --- .../org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index a2aaf53690bd7..e2417e525b6c8 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -138,7 +138,6 @@ public RollupIndexerAction.ShardRollupResponse execute() throws IOException { bucketCollector.postCollection(); } - logger.info( "Shard [{}] successfully sent [{}], indexed [{}], failed [{}], took [{}]", indexShard.shardId(), From f6bd4fb4a9f598b7679efc034b135d6a2d6ef260 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 12 Jul 2022 16:15:31 +0800 Subject: [PATCH 4/4] revert bulk failed log --- .../xpack/rollup/v2/RollupShardIndexer.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index e2417e525b6c8..6b3836d49228c 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -182,13 +182,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon ) ); numFailed.addAndGet(failures.size()); - logger.info( - "Shard [{}] failed to populate rollup index, failures summery: failed count: [{}], failed sample: [{}]", - indexShard.shardId(), - failures.size(), - failures.values().iterator().next() - ); - logger.trace("Shard [{}] failed to populate rollup index. Failures: [{}]", indexShard.shardId(), failures); + logger.error("Shard [{}] failed to populate rollup index. Failures: [{}]", indexShard.shardId(), failures); } } @@ -197,7 +191,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) if (failure != null) { long items = request.numberOfActions(); numFailed.addAndGet(items); - logger.info(() -> format("Shard [%s] failed to populate rollup index.", indexShard.shardId()), failure); + logger.error(() -> format("Shard [%s] failed to populate rollup index.", indexShard.shardId()), failure); } } };