Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove legacy translog header versions and checks #90395

Merged
merged 3 commits into from
Sep 27, 2022
Merged

Remove legacy translog header versions and checks #90395

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
41ebbce
Remove legacy translog header versions and checks
kingherc Sep 27, 2022
7b658d3
Remove tryReportOldVersionError
kingherc Sep 27, 2022
8be407b
Fix checks
kingherc Sep 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 2 additions & 29 deletions server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.Channels;
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
Expand All @@ -31,9 +30,7 @@
final class TranslogHeader {
public static final String TRANSLOG_CODEC = "translog";

public static final int VERSION_CHECKSUMS = 1; // pre-2.0 - unsupported
public static final int VERSION_CHECKPOINTS = 2; // added checkpoints
public static final int VERSION_PRIMARY_TERM = 3; // added primary term
public static final int VERSION_PRIMARY_TERM = 3; // with: checksums, checkpoints and primary term
public static final int CURRENT_VERSION = VERSION_PRIMARY_TERM;

private final String translogUUID;
Expand Down Expand Up @@ -96,17 +93,10 @@ private static int headerSizeInBytes(int version, int uuidLength) {
static int readHeaderVersion(final Path path, final FileChannel channel, final StreamInput in) throws IOException {
final int version;
try {
version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM);
version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_PRIMARY_TERM, VERSION_PRIMARY_TERM);
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
tryReportOldVersionError(path, channel);
throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e);
}
if (version == VERSION_CHECKSUMS) {
throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
}
if (version == VERSION_CHECKPOINTS) {
throw new IllegalStateException("pre-6.3 translog found [" + path + "]");
}
return version;
}

Expand Down Expand Up @@ -158,23 +148,6 @@ static TranslogHeader read(final String translogUUID, final Path path, final Fil
}
}

private static void tryReportOldVersionError(final Path path, final FileChannel channel) throws IOException {
// Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the header, in binary this looks like:
// binary: 0011 1111 1101 0111 0110 1100 0001 0111
// hex : 3 f d 7 6 c 1 7
//
// With version 0 of the translog, the first byte is the Operation.Type, which will always be between 0-4,
// so we know if we grab the first byte, it can be:
// 0x3f => Lucene's magic number, so we can assume it's version 1 or later
// 0x00 => version 0 of the translog
final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0];
if (b1 == 0x3f) { // LUCENE_CODEC_HEADER_BYTE
throw new TranslogCorruptedException(path.toString(), "translog looks like version 1 or later, but has corrupted header");
} else if (b1 == 0x00) { // UNVERSIONED_TRANSLOG_HEADER_BYTE
throw new IllegalStateException("pre-1.4 translog found [" + path + "]");
}
}

/**
* Writes this header with the latest format into the file channel
*/
Expand Down
35 changes: 13 additions & 22 deletions server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;

import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
Expand Down Expand Up @@ -51,37 +49,30 @@ public void testCurrentHeaderVersion() throws Exception {
final TranslogCorruptedException corruption = expectThrows(TranslogCorruptedException.class, () -> {
try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) {
TranslogHeader.read(randomBoolean() ? outHeader.getTranslogUUID() : UUIDs.randomBase64UUID(), translogFile, channel);
} catch (IllegalStateException e) {
// corruption corrupted the version byte making this look like a v2, v1 or v0 translog
assertThat(
"version " + TranslogHeader.VERSION_CHECKPOINTS + "-or-earlier translog",
e.getMessage(),
anyOf(
containsString("pre-2.0 translog found"),
containsString("pre-1.4 translog found"),
containsString("pre-6.3 translog found")
)
);
throw new TranslogCorruptedException(translogFile.toString(), "adjusted translog version", e);
}
});
assertThat(corruption.getMessage(), not(containsString("this translog file belongs to a different translog")));
}

public void testLegacyTranslogVersions() {
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", IllegalStateException.class, "pre-1.4 translog");
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", IllegalStateException.class, "pre-2.0 translog");
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v2.binary", IllegalStateException.class, "pre-6.3 translog");
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-truncated.binary", IllegalStateException.class, "pre-2.0 translog");
final String expectedMessage = "translog header corrupted";
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", TranslogCorruptedException.class, expectedMessage);
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", TranslogCorruptedException.class, expectedMessage);
checkFailsToOpen("/org/elasticsearch/index/translog/translog-v2.binary", TranslogCorruptedException.class, expectedMessage);
checkFailsToOpen(
"/org/elasticsearch/index/translog/translog-v1-truncated.binary",
TranslogCorruptedException.class,
expectedMessage
);
checkFailsToOpen(
"/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary",
TranslogCorruptedException.class,
"translog looks like version 1 or later, but has corrupted header"
expectedMessage
);
checkFailsToOpen(
"/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary",
IllegalStateException.class,
"pre-2.0 translog"
TranslogCorruptedException.class,
expectedMessage
);
}

Expand All @@ -101,7 +92,7 @@ public void testCorruptTranslogHeader() throws Exception {
TranslogHeader.read(randomValueOtherThan(translogUUID, UUIDs::randomBase64UUID), translogFile, channel);
}
});
assertThat(error, either(instanceOf(IllegalStateException.class)).or(instanceOf(TranslogCorruptedException.class)));
assertThat(error, instanceOf(TranslogCorruptedException.class));
}

private <E extends Exception> void checkFailsToOpen(String file, Class<E> expectedErrorType, String expectedMessage) {
Expand Down