Skip to content

Commit

Permalink
Enhance console output so that result from smaller segments get print…
Browse files Browse the repository at this point in the history
…ed faster
  • Loading branch information
zacharymorn committed Jun 1, 2021
1 parent bae7e7a commit 70dc71c
Showing 1 changed file with 28 additions and 16 deletions.
44 changes: 28 additions & 16 deletions lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -708,8 +709,29 @@ public Status checkIndex(List<String> onlySegments, ExecutorService executorServ
CompletableFuture<Status.SegmentInfoStatus>[] futures = new CompletableFuture[numSegments];

// checks segments concurrently
List<SegmentCommitInfo> segmentCommitInfos = new ArrayList<>();
for (SegmentCommitInfo sci : sis) {
segmentCommitInfos.add(sci);
}

// sort segmentCommitInfos by segment size, as smaller segment tends to finish faster, and
// hence its output can be printed out faster
Collections.sort(
segmentCommitInfos,
(info1, info2) -> {
try {
return Long.compare(info1.sizeInBytes(), info2.sizeInBytes());
} catch (IOException e) {
msg(
infoStream,
"ERROR: IOException occurred when comparing SegmentCommitInfo file sizes");
if (infoStream != null) e.printStackTrace(infoStream);
return 0;
}
});

for (int i = 0; i < numSegments; i++) {
final SegmentCommitInfo info = sis.info(i);
final SegmentCommitInfo info = segmentCommitInfos.get(i);
updateMaxSegmentName(result, info);
if (onlySegments != null && !onlySegments.contains(info.info.name)) {
continue;
Expand All @@ -718,15 +740,7 @@ public Status checkIndex(List<String> onlySegments, ExecutorService executorServ
SegmentInfos finalSis = sis;

ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream stream;
if (i > 0) {
// buffer the messages for segment starting from the 2nd one so that they can later be
// printed in order
stream = new PrintStream(output, true, IOUtils.UTF_8);
} else {
// optimize for first segment to print real-time
stream = infoStream;
}
PrintStream stream = new PrintStream(output, true, IOUtils.UTF_8);
msg(
stream,
(1 + i)
Expand All @@ -743,7 +757,7 @@ public Status checkIndex(List<String> onlySegments, ExecutorService executorServ
}

for (int i = 0; i < numSegments; i++) {
SegmentCommitInfo info = sis.info(i);
SegmentCommitInfo info = segmentCommitInfos.get(i);
if (onlySegments != null && !onlySegments.contains(info.info.name)) {
continue;
}
Expand All @@ -767,13 +781,11 @@ public Status checkIndex(List<String> onlySegments, ExecutorService executorServ
infoStream.println(output.toString(StandardCharsets.UTF_8));

assert failFast;
throw new CheckIndexException("Segment " + info.info.name + " check failed.", e);
throw new CheckIndexException(
"Segment " + info.info.name + " check failed.", e.getCause());
}

if (i > 0) {
// first segment output already printed by infoStream
infoStream.print(output.toString(StandardCharsets.UTF_8));
}
infoStream.print(output.toString(StandardCharsets.UTF_8));

processSegmentInfoStatusResult(result, info, segmentInfoStatus);
}
Expand Down

0 comments on commit 70dc71c

Please sign in to comment.