From 15fa45c68d50a8db30dfb2e5eca8659d610aabaf Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Sat, 10 Mar 2018 11:55:28 +0100 Subject: [PATCH] ZOOKEEPER-2994. Added padding, tool renamed to TxnLogToolkit, interactive mode, etc. --- .../{TxnLogTool.java => TxnLogToolkit.java} | 101 +++++++++++------- ...ogToolTest.java => TxnLogToolkitTest.java} | 75 ++++++++----- 2 files changed, 109 insertions(+), 67 deletions(-) rename src/java/main/org/apache/zookeeper/server/persistence/{TxnLogTool.java => TxnLogToolkit.java} (72%) rename src/java/test/org/apache/zookeeper/server/persistence/{TxnLogToolTest.java => TxnLogToolkitTest.java} (62%) diff --git a/src/java/main/org/apache/zookeeper/server/persistence/TxnLogTool.java b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java similarity index 72% rename from src/java/main/org/apache/zookeeper/server/persistence/TxnLogTool.java rename to src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java index d653cf469ae..c6fe0924232 100644 --- a/src/java/main/org/apache/zookeeper/server/persistence/TxnLogTool.java +++ b/src/java/main/org/apache/zookeeper/server/persistence/TxnLogToolkit.java @@ -31,8 +31,6 @@ import org.apache.zookeeper.server.TraceFormatter; import org.apache.zookeeper.server.util.SerializeUtils; import org.apache.zookeeper.txn.TxnHeader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.EOFException; @@ -41,21 +39,23 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.text.DateFormat; import java.util.Date; +import java.util.Scanner; import java.util.zip.Adler32; import java.util.zip.Checksum; import static org.apache.zookeeper.server.persistence.FileTxnLog.TXNLOG_MAGIC; -public class TxnLogTool implements Closeable { +public class TxnLogToolkit implements Closeable { - public class TxnLogToolException extends Exception { + static class TxnLogToolkitException extends Exception { private static final long serialVersionUID = 1L; private int exitCode; - TxnLogToolException(int exitCode, String message, Object... params) { + TxnLogToolkitException(int exitCode, String message, Object... params) { super(String.format(message, params)); this.exitCode = exitCode; } @@ -65,11 +65,11 @@ int getExitCode() { } } - public class TxnLogToolParseException extends TxnLogToolException { + static class TxnLogToolkitParseException extends TxnLogToolkitException { private static final long serialVersionUID = 1L; private Options options; - TxnLogToolParseException(Options options, int exitCode, String message, Object... params) { + TxnLogToolkitParseException(Options options, int exitCode, String message, Object... params) { super(exitCode, message, params); this.options = options; } @@ -84,47 +84,44 @@ Options getOptions() { private boolean verbose = false; private FileInputStream txnFis; private BinaryInputArchive logStream; - private boolean initialized = false; // Recovery mode private int crcFixed = 0; private FileOutputStream recoveryFos; private BinaryOutputArchive recoveryOa; private File recoveryLogFile; + private FilePadding filePadding = new FilePadding(); + private boolean force = false; /** * @param args Command line arguments */ public static void main(String[] args) throws Exception { - try (final TxnLogTool logf = new TxnLogTool()) { - logf.run(args); - } catch (TxnLogToolParseException e) { + try (final TxnLogToolkit lt = parseCommandLine(args)) { + lt.dump(new InputStreamReader(System.in)); + lt.printStat(); + } catch (TxnLogToolkitParseException e) { System.err.println(e.getMessage() + "\n"); printHelpAndExit(e.getExitCode(), e.getOptions()); - } catch (TxnLogToolException e) { + } catch (TxnLogToolkitException e) { System.err.println(e.getMessage()); System.exit(e.getExitCode()); } } - public void run(String[] args) throws Exception { - parseCommandLine(args); - dump(); - printStat(); - } - - public void init(boolean recoveryMode, boolean verbose, String txnLogFileName) - throws FileNotFoundException, TxnLogToolException { + public TxnLogToolkit(boolean recoveryMode, boolean verbose, String txnLogFileName, boolean force) + throws FileNotFoundException, TxnLogToolkitException { this.recoveryMode = recoveryMode; this.verbose = verbose; + this.force = force; txnLogFile = new File(txnLogFileName); if (!txnLogFile.exists() || !txnLogFile.canRead()) { - throw new TxnLogToolException(1, "File doesn't exist or not readable: %s", txnLogFile); + throw new TxnLogToolkitException(1, "File doesn't exist or not readable: %s", txnLogFile); } if (recoveryMode) { recoveryLogFile = new File(txnLogFile.toString() + ".fixed"); if (recoveryLogFile.exists()) { - throw new TxnLogToolException(1, "Recovery file %s already exists or not writable", recoveryLogFile); + throw new TxnLogToolkitException(1, "Recovery file %s already exists or not writable", recoveryLogFile); } } @@ -132,20 +129,15 @@ public void init(boolean recoveryMode, boolean verbose, String txnLogFileName) if (recoveryMode) { openRecoveryFile(); } - - initialized = true; } - public void dump() throws Exception { - if (!initialized) { - throw new TxnLogToolException(1, "TxnLogTool is not yet initialized"); - } + public void dump(Reader input) throws Exception { crcFixed = 0; FileHeader fhdr = new FileHeader(); fhdr.deserialize(logStream, "fileheader"); if (fhdr.getMagic() != TXNLOG_MAGIC) { - throw new TxnLogToolException(2, "Invalid magic number for %s", txnLogFile.getName()); + throw new TxnLogToolkitException(2, "Invalid magic number for %s", txnLogFile.getName()); } System.out.println("ZooKeeper Transactional Log File with dbid " + fhdr.getDbid() + " txnlog format version " @@ -153,6 +145,8 @@ public void dump() throws Exception { if (recoveryMode) { fhdr.serialize(recoveryOa, "fileheader"); + recoveryFos.flush(); + filePadding.setCurrentSize(recoveryFos.getChannel().position()); } int count = 0; @@ -176,9 +170,17 @@ public void dump() throws Exception { crc.update(bytes, 0, bytes.length); if (crcValue != crc.getValue()) { if (recoveryMode) { - crcValue = crc.getValue(); - printTxn(bytes, "CRC FIXED"); - ++crcFixed; + if (!force) { + printTxn(bytes, "CRC ERROR"); + if (askForFix(input)) { + crcValue = crc.getValue(); + ++crcFixed; + } + } else { + crcValue = crc.getValue(); + printTxn(bytes, "CRC FIXED"); + ++crcFixed; + } } else { printTxn(bytes, "CRC ERROR"); } @@ -187,19 +189,35 @@ public void dump() throws Exception { printTxn(bytes); } if (logStream.readByte("EOR") != 'B') { - throw new TxnLogToolException(1, "Last transaction was partial."); + throw new TxnLogToolkitException(1, "Last transaction was partial."); } if (recoveryMode) { + filePadding.padFile(recoveryFos.getChannel()); recoveryOa.writeLong(crcValue, "crcvalue"); recoveryOa.writeBuffer(bytes, "txnEntry"); recoveryOa.writeByte((byte)'B', "EOR"); - // add padding - } count++; } } + private boolean askForFix(Reader input) throws TxnLogToolkitException { + try (Scanner scanner = new Scanner(input)) { + while (true) { + System.out.print("Would you like to fix it (Yes/No/Abort) ? "); + char answer = Character.toUpperCase(scanner.next().charAt(0)); + switch (answer) { + case 'Y': + return true; + case 'N': + return false; + case 'A': + throw new TxnLogToolkitException(0, "Recovery aborted."); + } + } + } + } + private void printTxn(byte[] bytes) throws IOException { printTxn(bytes, ""); } @@ -246,7 +264,7 @@ private void closeRecoveryFile() throws IOException { } } - private void parseCommandLine(String[] args) throws TxnLogToolException, FileNotFoundException { + private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolkitException, FileNotFoundException { CommandLineParser parser = new PosixParser(); Options options = new Options(); @@ -262,6 +280,9 @@ private void parseCommandLine(String[] args) throws TxnLogToolException, FileNot Option dumpOpt = new Option("d", "dump", false, "Dump mode. Dump all entries of the log file. (this is the default)"); options.addOption(dumpOpt); + Option forceOpt = new Option("y", "yes", false, "Non-interactive mode: repair all CRC errors without asking"); + options.addOption(forceOpt); + try { CommandLine cli = parser.parse(options, args); if (cli.hasOption("help")) { @@ -270,15 +291,15 @@ private void parseCommandLine(String[] args) throws TxnLogToolException, FileNot if (cli.getArgs().length < 1) { printHelpAndExit(1, options); } - init(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0]); + return new TxnLogToolkit(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0], cli.hasOption("yes")); } catch (ParseException e) { - throw new TxnLogToolParseException(options, 1, e.getMessage()); + throw new TxnLogToolkitParseException(options, 1, e.getMessage()); } } private static void printHelpAndExit(int exitCode, Options options) { HelpFormatter help = new HelpFormatter(); - help.printHelp(120,"TxnLogTool [-dhrv] ", "", options, ""); + help.printHelp(120,"TxnLogToolkit [-dhrv] ", "", options, ""); System.exit(exitCode); } diff --git a/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolTest.java b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java similarity index 62% rename from src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolTest.java rename to src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java index f90cfbc9c54..e55a2b2fffb 100644 --- a/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolTest.java +++ b/src/java/test/org/apache/zookeeper/server/persistence/TxnLogToolkitTest.java @@ -29,12 +29,14 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; +import java.io.StringReader; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.core.IsNot.not; import static org.junit.Assert.assertThat; -public class TxnLogToolTest { +public class TxnLogToolkitTest { private static final File testData = new File( System.getProperty("test.data.dir", "build/test/data")); @@ -63,47 +65,39 @@ public void tearDown() throws IOException { public void testDumpMode() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); - TxnLogTool lt = new TxnLogTool(); - lt.init(false, false, logfile.toString()); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); // Act - lt.dump(); + lt.dump(null); // Assert // no exception thrown } - @Test(expected = TxnLogTool.TxnLogToolException.class) - public void testInitMissingFile() throws FileNotFoundException, TxnLogTool.TxnLogToolException { - // Arrange + @Test(expected = TxnLogToolkit.TxnLogToolkitException.class) + public void testInitMissingFile() throws FileNotFoundException, TxnLogToolkit.TxnLogToolkitException { + // Arrange & Act File logfile = new File("this_file_should_not_exists"); - TxnLogTool lt = new TxnLogTool(); - - // Act - lt.init(false, false, logfile.toString()); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); } - @Test(expected = TxnLogTool.TxnLogToolException.class) - public void testInitWithRecoveryFileExists() throws IOException, TxnLogTool.TxnLogToolException { - // Arrange + @Test(expected = TxnLogToolkit.TxnLogToolkitException.class) + public void testInitWithRecoveryFileExists() throws IOException, TxnLogToolkit.TxnLogToolkitException { + // Arrange & Act File logfile = new File(new File(mySnapDir, "version-2"), "log.274"); File recoveryFile = new File(new File(mySnapDir, "version-2"), "log.274.fixed"); recoveryFile.createNewFile(); - TxnLogTool lt = new TxnLogTool(); - - // Act - lt.init(true, false, logfile.toString()); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); } @Test public void testDumpWithCrcError() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); - TxnLogTool lt = new TxnLogTool(); - lt.init(false, false, logfile.toString()); + TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true); // Act - lt.dump(); + lt.dump(null); // Assert String output = outContent.toString(); @@ -114,16 +108,43 @@ public void testDumpWithCrcError() throws Exception { public void testRecoveryFixBrokenFile() throws Exception { // Arrange File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); - TxnLogTool lt = new TxnLogTool(); - lt.init(true, false, logfile.toString()); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true); + + // Act + lt.dump(null); + + // Assert + String output = outContent.toString(); + assertThat(output, containsString("CRC FIXED")); + + // Should be able to dump the recovered logfile with no CRC error + outContent.reset(); + logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); + lt = new TxnLogToolkit(false, false, logfile.toString(), true); + lt.dump(null); + output = outContent.toString(); + assertThat(output, not(containsString("CRC ERROR"))); + } + + @Test + public void testRecoveryInteractiveMode() throws Exception { + // Arrange + File logfile = new File(new File(mySnapDir, "version-2"), "log.42"); + TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), false); // Act - lt.dump(); + lt.dump(new StringReader("y\n")); // Assert - // Should be able to dump the recovered logfile + String output = outContent.toString(); + assertThat(output, containsString("CRC ERROR")); + + // Should be able to dump the recovered logfile with no CRC error + outContent.reset(); logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed"); - lt.init(false, false, logfile.toString()); - lt.dump(); + lt = new TxnLogToolkit(false, false, logfile.toString(), true); + lt.dump(null); + output = outContent.toString(); + assertThat(output, not(containsString("CRC ERROR"))); } }