Skip to content

Commit

Permalink
Use correct abbreviations for data quantities
Browse files Browse the repository at this point in the history 
* Add option for decimal data size/rates, change KB etc to KiB etc for binary
* Use correct abbreviations for data quantities. Fixes #13054
  • Loading branch information
@metadaddy
metadaddy authored Sep 19, 2024
1 parent bb83d31 commit 08ff9ca
Show file tree
Hide file tree
Showing 9 changed files with 558 additions and 51 deletions.
3 changes: 3 additions & 0 deletions client/trino-cli/src/main/java/io/trino/cli/ClientOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,9 @@ public class ClientOptions
@Option(names = "--disable-auto-suggestion", description = "Disable auto suggestion")
public boolean disableAutoSuggestion;

@Option(names = "--decimal-data-size", description = "Show data size and rate in base 10 rather than base 2")
public boolean decimalDataSize;

public enum OutputFormat
{
AUTO,
Expand Down
19 changes: 12 additions & 7 deletions client/trino-cli/src/main/java/io/trino/cli/Console.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ public boolean run()
query,
clientOptions.outputFormat,
clientOptions.ignoreErrors,
clientOptions.progress.orElse(false));
clientOptions.progress.orElse(false),
clientOptions.decimalDataSize);
}

Optional<String> pager = clientOptions.pager;
Expand All @@ -196,7 +197,8 @@ public boolean run()
getHistoryFile(clientOptions.historyFile),
pager,
clientOptions.progress.orElse(true),
clientOptions.disableAutoSuggestion);
clientOptions.disableAutoSuggestion,
clientOptions.decimalDataSize);
return true;
}
finally {
Expand All @@ -223,7 +225,8 @@ private static void runConsole(
Optional<Path> historyFile,
Optional<String> pager,
boolean progress,
boolean disableAutoSuggestion)
boolean disableAutoSuggestion,
boolean decimalDataSize)
{
try (TableNameCompleter tableNameCompleter = new TableNameCompleter(queryRunner);
InputReader reader = new InputReader(editingMode, historyFile, disableAutoSuggestion, commandCompleter(), tableNameCompleter)) {
Expand Down Expand Up @@ -290,7 +293,7 @@ private static void runConsole(
currentOutputFormat = OutputFormat.VERTICAL;
}

process(queryRunner, split.statement(), currentOutputFormat, tableNameCompleter::populateCache, pager, progress, reader.getTerminal(), System.out, System.out);
process(queryRunner, split.statement(), currentOutputFormat, tableNameCompleter::populateCache, pager, progress, decimalDataSize, reader.getTerminal(), System.out, System.out);
}

// replace remaining with trailing partial statement
Expand All @@ -308,13 +311,14 @@ private static boolean executeCommand(
String query,
OutputFormat outputFormat,
boolean ignoreErrors,
boolean showProgress)
boolean showProgress,
boolean decimalDataSize)
{
boolean success = true;
StatementSplitter splitter = new StatementSplitter(query);
for (Statement split : splitter.getCompleteStatements()) {
if (!isEmptyStatement(split.statement())) {
if (!process(queryRunner, split.statement(), outputFormat, () -> {}, Optional.of(""), showProgress, getTerminal(), System.out, System.err)) {
if (!process(queryRunner, split.statement(), outputFormat, () -> {}, Optional.of(""), showProgress, decimalDataSize, getTerminal(), System.out, System.err)) {
if (!ignoreErrors) {
return false;
}
Expand All @@ -339,6 +343,7 @@ private static boolean process(
Runnable schemaChanged,
Optional<String> pager,
boolean showProgress,
boolean decimalDataSize,
Terminal terminal,
PrintStream out,
PrintStream errorChannel)
Expand All @@ -360,7 +365,7 @@ private static boolean process(
}

try (Query query = queryRunner.startQuery(finalSql)) {
boolean success = query.renderOutput(terminal, out, errorChannel, outputFormat, pager, showProgress);
boolean success = query.renderOutput(terminal, out, errorChannel, outputFormat, pager, showProgress, decimalDataSize);

ClientSession session = queryRunner.getSession();

Expand Down
36 changes: 21 additions & 15 deletions client/trino-cli/src/main/java/io/trino/cli/FormatUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,49 +90,55 @@ public static String formatCountRate(double count, Duration duration, boolean lo
return rateString;
}

public static String formatDataSize(DataSize size, boolean longForm)
public static String formatDataSize(DataSize size, boolean longForm, boolean decimalDataSize)
{
int divisor = decimalDataSize ? 1000 : 1024;
double fractional = size.toBytes();
String unit = null;
if (fractional >= 1024) {
fractional /= 1024;
if (fractional >= divisor) {
fractional /= divisor;
unit = "K";
}
if (fractional >= 1024) {
fractional /= 1024;
if (fractional >= divisor) {
fractional /= divisor;
unit = "M";
}
if (fractional >= 1024) {
fractional /= 1024;
if (fractional >= divisor) {
fractional /= divisor;
unit = "G";
}
if (fractional >= 1024) {
fractional /= 1024;
if (fractional >= divisor) {
fractional /= divisor;
unit = "T";
}
if (fractional >= 1024) {
fractional /= 1024;
if (fractional >= divisor) {
fractional /= divisor;
unit = "P";
}

if (unit == null) {
unit = "B";
}
else if (longForm) {
unit += "B";
else {
if (!decimalDataSize) {
unit += "i";
}
if (longForm) {
unit += "B";
}
}

return format("%s%s", getFormat(fractional).format(fractional), unit);
}

public static String formatDataRate(DataSize dataSize, Duration duration, boolean longForm)
public static String formatDataRate(DataSize dataSize, Duration duration, boolean longForm, boolean decimalDataSize)
{
long rate = Math.round(dataSize.toBytes() / duration.getValue(SECONDS));
if (Double.isNaN(rate) || Double.isInfinite(rate)) {
rate = 0;
}

String rateString = formatDataSize(DataSize.ofBytes(rate), false);
String rateString = formatDataSize(DataSize.ofBytes(rate), false, decimalDataSize);
if (longForm) {
if (!rateString.endsWith("B")) {
rateString += "B";
Expand Down
8 changes: 4 additions & 4 deletions client/trino-cli/src/main/java/io/trino/cli/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public boolean isClearTransactionId()
return client.isClearTransactionId();
}

public boolean renderOutput(Terminal terminal, PrintStream out, PrintStream errorChannel, OutputFormat outputFormat, Optional<String> pager, boolean showProgress)
public boolean renderOutput(Terminal terminal, PrintStream out, PrintStream errorChannel, OutputFormat outputFormat, Optional<String> pager, boolean showProgress, boolean decimalDataSize)
{
Thread clientThread = Thread.currentThread();
SignalHandler oldHandler = terminal.handle(Signal.INT, signal -> {
Expand All @@ -147,21 +147,21 @@ public boolean renderOutput(Terminal terminal, PrintStream out, PrintStream erro
clientThread.interrupt();
});
try {
return renderQueryOutput(terminal, out, errorChannel, outputFormat, pager, showProgress);
return renderQueryOutput(terminal, out, errorChannel, outputFormat, pager, showProgress, decimalDataSize);
}
finally {
terminal.handle(Signal.INT, oldHandler);
Thread.interrupted(); // clear interrupt status
}
}

private boolean renderQueryOutput(Terminal terminal, PrintStream out, PrintStream errorChannel, OutputFormat outputFormat, Optional<String> pager, boolean showProgress)
private boolean renderQueryOutput(Terminal terminal, PrintStream out, PrintStream errorChannel, OutputFormat outputFormat, Optional<String> pager, boolean showProgress, boolean decimalDataSize)
{
StatusPrinter statusPrinter = null;
WarningsPrinter warningsPrinter = new PrintStreamWarningsPrinter(errorChannel);

if (showProgress) {
statusPrinter = new StatusPrinter(client, errorChannel, debug, isInteractive(pager));
statusPrinter = new StatusPrinter(client, errorChannel, debug, isInteractive(pager), decimalDataSize);
statusPrinter.printInitialStatusUpdates(terminal);
}
else {
Expand Down
46 changes: 24 additions & 22 deletions client/trino-cli/src/main/java/io/trino/cli/StatusPrinter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,16 @@ public class StatusPrinter
private final boolean checkInput;

private boolean debug;
private boolean decimalDataSize;

public StatusPrinter(StatementClient client, PrintStream out, boolean debug, boolean checkInput)
public StatusPrinter(StatementClient client, PrintStream out, boolean debug, boolean checkInput, boolean decimalDataSize)
{
this.client = client;
this.out = out;
this.console = new ConsolePrinter(out);
this.debug = debug;
this.checkInput = checkInput;
this.decimalDataSize = decimalDataSize;
}

/*
Expand Down Expand Up @@ -200,7 +202,7 @@ public void printFinalInfo()
String cpuTimeSummary = format("CPU Time: %.1fs total, %5s rows/s, %8s, %d%% active",
cpuTime.getValue(SECONDS),
formatCountRate(stats.getProcessedRows(), cpuTime, false),
formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true),
formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true, decimalDataSize),
(int) percentage(stats.getCpuTimeMillis(), stats.getWallTimeMillis()));
out.println(cpuTimeSummary);

Expand All @@ -210,28 +212,28 @@ public void printFinalInfo()
String perNodeSummary = format("Per Node: %.1f parallelism, %5s rows/s, %8s",
parallelism / nodes,
formatCountRate((double) stats.getProcessedRows() / nodes, wallTime, false),
formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true));
formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true, decimalDataSize));
reprintLine(perNodeSummary);

// Parallelism: 5.3
out.println(format("Parallelism: %.1f", parallelism));

// Peak Memory: 1.97GB
reprintLine("Peak Memory: " + formatDataSize(bytes(stats.getPeakMemoryBytes()), true));
reprintLine("Peak Memory: " + formatDataSize(bytes(stats.getPeakMemoryBytes()), true, decimalDataSize));

// Spilled Data: 20GB
if (stats.getSpilledBytes() > 0) {
reprintLine("Spilled: " + formatDataSize(bytes(stats.getSpilledBytes()), true));
reprintLine("Spilled: " + formatDataSize(bytes(stats.getSpilledBytes()), true, decimalDataSize));
}
}

// 0:32 [2.12GB, 15M rows] [67MB/s, 463K rows/s]
String statsLine = format("%s [%s rows, %s] [%s rows/s, %s]",
formatFinalTime(wallTime),
formatCount(stats.getProcessedRows()),
formatDataSize(bytes(stats.getProcessedBytes()), true),
formatDataSize(bytes(stats.getProcessedBytes()), true, decimalDataSize),
formatCountRate(stats.getProcessedRows(), wallTime, false),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true));
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true, decimalDataSize));

out.println(statsLine);

Expand Down Expand Up @@ -296,7 +298,7 @@ private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPri
String cpuTimeSummary = format("CPU Time: %.1fs total, %5s rows/s, %8s, %d%% active",
cpuTime.getValue(SECONDS),
formatCountRate(stats.getProcessedRows(), cpuTime, false),
formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true),
formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true, decimalDataSize),
(int) percentage(stats.getCpuTimeMillis(), stats.getWallTimeMillis()));
reprintLine(cpuTimeSummary);

Expand All @@ -306,18 +308,18 @@ private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPri
String perNodeSummary = format("Per Node: %.1f parallelism, %5s rows/s, %8s",
parallelism / nodes,
formatCountRate((double) stats.getProcessedRows() / nodes, wallTime, false),
formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true));
formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true, decimalDataSize));
reprintLine(perNodeSummary);

// Parallelism: 5.3
reprintLine(format("Parallelism: %.1f", parallelism));

// Peak Memory: 1.97GB
reprintLine("Peak Memory: " + formatDataSize(bytes(stats.getPeakMemoryBytes()), true));
reprintLine("Peak Memory: " + formatDataSize(bytes(stats.getPeakMemoryBytes()), true, decimalDataSize));

// Spilled Data: 20GB
if (stats.getSpilledBytes() > 0) {
reprintLine("Spilled: " + formatDataSize(bytes(stats.getSpilledBytes()), true));
reprintLine("Spilled: " + formatDataSize(bytes(stats.getSpilledBytes()), true, decimalDataSize));
}
}

Expand All @@ -334,9 +336,9 @@ private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPri
String progressLine = format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s] %d%%",
formatTime(wallTime),
formatCount(stats.getProcessedRows()),
formatDataSize(bytes(stats.getProcessedBytes()), true),
formatDataSize(bytes(stats.getProcessedBytes()), true, decimalDataSize),
formatCountRate(stats.getProcessedRows(), wallTime, false),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true, decimalDataSize),
progressBar,
progressPercentage);

Expand All @@ -349,9 +351,9 @@ private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPri
String progressLine = format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s]",
formatTime(wallTime),
formatCount(stats.getProcessedRows()),
formatDataSize(bytes(stats.getProcessedBytes()), true),
formatDataSize(bytes(stats.getProcessedBytes()), true, decimalDataSize),
formatCountRate(stats.getProcessedRows(), wallTime, false),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true, decimalDataSize),
progressBar);

reprintLine(progressLine);
Expand Down Expand Up @@ -382,12 +384,12 @@ private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPri
stats.getState(),

formatCount(stats.getProcessedRows()),
formatDataSize(bytes(stats.getProcessedBytes()), false),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false),
formatDataSize(bytes(stats.getProcessedBytes()), false, decimalDataSize),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false, decimalDataSize),

formatCount(stats.getProcessedRows()),
formatDataSize(bytes(stats.getProcessedBytes()), false),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false),
formatDataSize(bytes(stats.getProcessedBytes()), false, decimalDataSize),
formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false, decimalDataSize),

stats.getQueuedSplits(),
stats.getRunningSplits(),
Expand Down Expand Up @@ -415,11 +417,11 @@ private void printStageTree(StageStats stage, String indent, AtomicInteger stage
String bytesPerSecond;
String rowsPerSecond;
if (stage.isDone()) {
bytesPerSecond = formatDataRate(DataSize.ofBytes(0), new Duration(0, SECONDS), false);
bytesPerSecond = formatDataRate(DataSize.ofBytes(0), new Duration(0, SECONDS), false, decimalDataSize);
rowsPerSecond = formatCountRate(0, new Duration(0, SECONDS), false);
}
else {
bytesPerSecond = formatDataRate(bytes(stage.getProcessedBytes()), elapsedTime, false);
bytesPerSecond = formatDataRate(bytes(stage.getProcessedBytes()), elapsedTime, false, decimalDataSize);
rowsPerSecond = formatCountRate(stage.getProcessedRows(), elapsedTime, false);
}

Expand All @@ -430,7 +432,7 @@ private void printStageTree(StageStats stage, String indent, AtomicInteger stage
formatCount(stage.getProcessedRows()),
rowsPerSecond,

formatDataSize(bytes(stage.getProcessedBytes()), false),
formatDataSize(bytes(stage.getProcessedBytes()), false, decimalDataSize),
bytesPerSecond,

stage.getQueuedSplits(),
Expand Down
Loading

0 comments on commit 08ff9ca

Please sign in to comment.