Skip to content

Commit

Permalink
ZOOKEEPER-2994. Added padding, tool renamed to TxnLogToolkit,
Browse files Browse the repository at this point in the history 
interactive mode, etc.
  • Loading branch information
@anmolnar
anmolnar committed Mar 26, 2018
1 parent 6a1ad0e commit 15fa45c
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 67 deletions.
101 changes: 61 additions & 40 deletions ...keeper/server/persistence/TxnLogTool.java → ...per/server/persistence/TxnLogToolkit.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
import org.apache.zookeeper.server.TraceFormatter;
import org.apache.zookeeper.server.util.SerializeUtils;
import org.apache.zookeeper.txn.TxnHeader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.EOFException;
Expand All @@ -41,21 +39,23 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.text.DateFormat;
import java.util.Date;
import java.util.Scanner;
import java.util.zip.Adler32;
import java.util.zip.Checksum;

import static org.apache.zookeeper.server.persistence.FileTxnLog.TXNLOG_MAGIC;

public class TxnLogTool implements Closeable {
public class TxnLogToolkit implements Closeable {

public class TxnLogToolException extends Exception {
static class TxnLogToolkitException extends Exception {
private static final long serialVersionUID = 1L;
private int exitCode;

TxnLogToolException(int exitCode, String message, Object... params) {
TxnLogToolkitException(int exitCode, String message, Object... params) {
super(String.format(message, params));
this.exitCode = exitCode;
}
Expand All @@ -65,11 +65,11 @@ int getExitCode() {
}
}

public class TxnLogToolParseException extends TxnLogToolException {
static class TxnLogToolkitParseException extends TxnLogToolkitException {
private static final long serialVersionUID = 1L;
private Options options;

TxnLogToolParseException(Options options, int exitCode, String message, Object... params) {
TxnLogToolkitParseException(Options options, int exitCode, String message, Object... params) {
super(exitCode, message, params);
this.options = options;
}
Expand All @@ -84,75 +84,69 @@ Options getOptions() {
private boolean verbose = false;
private FileInputStream txnFis;
private BinaryInputArchive logStream;
private boolean initialized = false;

// Recovery mode
private int crcFixed = 0;
private FileOutputStream recoveryFos;
private BinaryOutputArchive recoveryOa;
private File recoveryLogFile;
private FilePadding filePadding = new FilePadding();
private boolean force = false;

/**
* @param args Command line arguments
*/
public static void main(String[] args) throws Exception {
try (final TxnLogTool logf = new TxnLogTool()) {
logf.run(args);
} catch (TxnLogToolParseException e) {
try (final TxnLogToolkit lt = parseCommandLine(args)) {
lt.dump(new InputStreamReader(System.in));
lt.printStat();
} catch (TxnLogToolkitParseException e) {
System.err.println(e.getMessage() + "\n");
printHelpAndExit(e.getExitCode(), e.getOptions());
} catch (TxnLogToolException e) {
} catch (TxnLogToolkitException e) {
System.err.println(e.getMessage());
System.exit(e.getExitCode());
}
}

public void run(String[] args) throws Exception {
parseCommandLine(args);
dump();
printStat();
}

public void init(boolean recoveryMode, boolean verbose, String txnLogFileName)
throws FileNotFoundException, TxnLogToolException {
public TxnLogToolkit(boolean recoveryMode, boolean verbose, String txnLogFileName, boolean force)
throws FileNotFoundException, TxnLogToolkitException {
this.recoveryMode = recoveryMode;
this.verbose = verbose;
this.force = force;
txnLogFile = new File(txnLogFileName);
if (!txnLogFile.exists() || !txnLogFile.canRead()) {
throw new TxnLogToolException(1, "File doesn't exist or not readable: %s", txnLogFile);
throw new TxnLogToolkitException(1, "File doesn't exist or not readable: %s", txnLogFile);
}
if (recoveryMode) {
recoveryLogFile = new File(txnLogFile.toString() + ".fixed");
if (recoveryLogFile.exists()) {
throw new TxnLogToolException(1, "Recovery file %s already exists or not writable", recoveryLogFile);
throw new TxnLogToolkitException(1, "Recovery file %s already exists or not writable", recoveryLogFile);
}
}

openTxnLogFile();
if (recoveryMode) {
openRecoveryFile();
}

initialized = true;
}

public void dump() throws Exception {
if (!initialized) {
throw new TxnLogToolException(1, "TxnLogTool is not yet initialized");
}
public void dump(Reader input) throws Exception {
crcFixed = 0;

FileHeader fhdr = new FileHeader();
fhdr.deserialize(logStream, "fileheader");
if (fhdr.getMagic() != TXNLOG_MAGIC) {
throw new TxnLogToolException(2, "Invalid magic number for %s", txnLogFile.getName());
throw new TxnLogToolkitException(2, "Invalid magic number for %s", txnLogFile.getName());
}
System.out.println("ZooKeeper Transactional Log File with dbid "
+ fhdr.getDbid() + " txnlog format version "
+ fhdr.getVersion());

if (recoveryMode) {
fhdr.serialize(recoveryOa, "fileheader");
recoveryFos.flush();
filePadding.setCurrentSize(recoveryFos.getChannel().position());
}

int count = 0;
Expand All @@ -176,9 +170,17 @@ public void dump() throws Exception {
crc.update(bytes, 0, bytes.length);
if (crcValue != crc.getValue()) {
if (recoveryMode) {
crcValue = crc.getValue();
printTxn(bytes, "CRC FIXED");
++crcFixed;
if (!force) {
printTxn(bytes, "CRC ERROR");
if (askForFix(input)) {
crcValue = crc.getValue();
++crcFixed;
}
} else {
crcValue = crc.getValue();
printTxn(bytes, "CRC FIXED");
++crcFixed;
}
} else {
printTxn(bytes, "CRC ERROR");
}
Expand All @@ -187,19 +189,35 @@ public void dump() throws Exception {
printTxn(bytes);
}
if (logStream.readByte("EOR") != 'B') {
throw new TxnLogToolException(1, "Last transaction was partial.");
throw new TxnLogToolkitException(1, "Last transaction was partial.");
}
if (recoveryMode) {
filePadding.padFile(recoveryFos.getChannel());
recoveryOa.writeLong(crcValue, "crcvalue");
recoveryOa.writeBuffer(bytes, "txnEntry");
recoveryOa.writeByte((byte)'B', "EOR");
// add padding

}
count++;
}
}

private boolean askForFix(Reader input) throws TxnLogToolkitException {
try (Scanner scanner = new Scanner(input)) {
while (true) {
System.out.print("Would you like to fix it (Yes/No/Abort) ? ");
char answer = Character.toUpperCase(scanner.next().charAt(0));
switch (answer) {
case 'Y':
return true;
case 'N':
return false;
case 'A':
throw new TxnLogToolkitException(0, "Recovery aborted.");
}
}
}
}

private void printTxn(byte[] bytes) throws IOException {
printTxn(bytes, "");
}
Expand Down Expand Up @@ -246,7 +264,7 @@ private void closeRecoveryFile() throws IOException {
}
}

private void parseCommandLine(String[] args) throws TxnLogToolException, FileNotFoundException {
private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolkitException, FileNotFoundException {
CommandLineParser parser = new PosixParser();
Options options = new Options();

Expand All @@ -262,6 +280,9 @@ private void parseCommandLine(String[] args) throws TxnLogToolException, FileNot
Option dumpOpt = new Option("d", "dump", false, "Dump mode. Dump all entries of the log file. (this is the default)");
options.addOption(dumpOpt);

Option forceOpt = new Option("y", "yes", false, "Non-interactive mode: repair all CRC errors without asking");
options.addOption(forceOpt);

try {
CommandLine cli = parser.parse(options, args);
if (cli.hasOption("help")) {
Expand All @@ -270,15 +291,15 @@ private void parseCommandLine(String[] args) throws TxnLogToolException, FileNot
if (cli.getArgs().length < 1) {
printHelpAndExit(1, options);
}
init(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0]);
return new TxnLogToolkit(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0], cli.hasOption("yes"));
} catch (ParseException e) {
throw new TxnLogToolParseException(options, 1, e.getMessage());
throw new TxnLogToolkitParseException(options, 1, e.getMessage());
}
}

private static void printHelpAndExit(int exitCode, Options options) {
HelpFormatter help = new HelpFormatter();
help.printHelp(120,"TxnLogTool [-dhrv] <txn_log_file_name>", "", options, "");
help.printHelp(120,"TxnLogToolkit [-dhrv] <txn_log_file_name>", "", options, "");
System.exit(exitCode);
}

Expand Down
75 changes: 48 additions & 27 deletions ...er/server/persistence/TxnLogToolTest.java → ...server/persistence/TxnLogToolkitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.assertThat;


public class TxnLogToolTest {
public class TxnLogToolkitTest {
private static final File testData = new File(
System.getProperty("test.data.dir", "build/test/data"));

Expand Down Expand Up @@ -63,47 +65,39 @@ public void tearDown() throws IOException {
public void testDumpMode() throws Exception {
// Arrange
File logfile = new File(new File(mySnapDir, "version-2"), "log.274");
TxnLogTool lt = new TxnLogTool();
lt.init(false, false, logfile.toString());
TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true);

// Act
lt.dump();
lt.dump(null);

// Assert
// no exception thrown
}

@Test(expected = TxnLogTool.TxnLogToolException.class)
public void testInitMissingFile() throws FileNotFoundException, TxnLogTool.TxnLogToolException {
// Arrange
@Test(expected = TxnLogToolkit.TxnLogToolkitException.class)
public void testInitMissingFile() throws FileNotFoundException, TxnLogToolkit.TxnLogToolkitException {
// Arrange & Act
File logfile = new File("this_file_should_not_exists");
TxnLogTool lt = new TxnLogTool();

// Act
lt.init(false, false, logfile.toString());
TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true);
}

@Test(expected = TxnLogTool.TxnLogToolException.class)
public void testInitWithRecoveryFileExists() throws IOException, TxnLogTool.TxnLogToolException {
// Arrange
@Test(expected = TxnLogToolkit.TxnLogToolkitException.class)
public void testInitWithRecoveryFileExists() throws IOException, TxnLogToolkit.TxnLogToolkitException {
// Arrange & Act
File logfile = new File(new File(mySnapDir, "version-2"), "log.274");
File recoveryFile = new File(new File(mySnapDir, "version-2"), "log.274.fixed");
recoveryFile.createNewFile();
TxnLogTool lt = new TxnLogTool();

// Act
lt.init(true, false, logfile.toString());
TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true);
}

@Test
public void testDumpWithCrcError() throws Exception {
// Arrange
File logfile = new File(new File(mySnapDir, "version-2"), "log.42");
TxnLogTool lt = new TxnLogTool();
lt.init(false, false, logfile.toString());
TxnLogToolkit lt = new TxnLogToolkit(false, false, logfile.toString(), true);

// Act
lt.dump();
lt.dump(null);

// Assert
String output = outContent.toString();
Expand All @@ -114,16 +108,43 @@ public void testDumpWithCrcError() throws Exception {
public void testRecoveryFixBrokenFile() throws Exception {
// Arrange
File logfile = new File(new File(mySnapDir, "version-2"), "log.42");
TxnLogTool lt = new TxnLogTool();
lt.init(true, false, logfile.toString());
TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), true);

// Act
lt.dump(null);

// Assert
String output = outContent.toString();
assertThat(output, containsString("CRC FIXED"));

// Should be able to dump the recovered logfile with no CRC error
outContent.reset();
logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed");
lt = new TxnLogToolkit(false, false, logfile.toString(), true);
lt.dump(null);
output = outContent.toString();
assertThat(output, not(containsString("CRC ERROR")));
}

@Test
public void testRecoveryInteractiveMode() throws Exception {
// Arrange
File logfile = new File(new File(mySnapDir, "version-2"), "log.42");
TxnLogToolkit lt = new TxnLogToolkit(true, false, logfile.toString(), false);

// Act
lt.dump();
lt.dump(new StringReader("y\n"));

// Assert
// Should be able to dump the recovered logfile
String output = outContent.toString();
assertThat(output, containsString("CRC ERROR"));

// Should be able to dump the recovered logfile with no CRC error
outContent.reset();
logfile = new File(new File(mySnapDir, "version-2"), "log.42.fixed");
lt.init(false, false, logfile.toString());
lt.dump();
lt = new TxnLogToolkit(false, false, logfile.toString(), true);
lt.dump(null);
output = outContent.toString();
assertThat(output, not(containsString("CRC ERROR")));
}
}

0 comments on commit 15fa45c

Please sign in to comment.