Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate peer recovery from translog to retention lease #49448

Merged
merged 21 commits into from
Dec 13, 2019
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
always disable translog in 7.4
  • Loading branch information
dnhatn committed Dec 9, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 0349f76e7e9ea68b280bdc88e809be8ce18e1731
32 changes: 23 additions & 9 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
Original file line number Diff line number Diff line change
@@ -249,9 +249,7 @@ public final class IndexSettings {
**/
public static final Setting<TimeValue> INDEX_TRANSLOG_RETENTION_AGE_SETTING =
Setting.timeSetting("index.translog.retention.age",
settings -> INDEX_SOFT_DELETES_SETTING.get(settings) &&
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_7_4_0) ?
TimeValue.MINUS_ONE : TimeValue.timeValueHours(12),
settings -> shouldDisableTranslogRetention(settings) ? TimeValue.MINUS_ONE : TimeValue.timeValueHours(12),
TimeValue.MINUS_ONE, Property.Dynamic, Property.IndexScope);

/**
@@ -262,8 +260,7 @@ public final class IndexSettings {
**/
public static final Setting<ByteSizeValue> INDEX_TRANSLOG_RETENTION_SIZE_SETTING =
Setting.byteSizeSetting("index.translog.retention.size",
settings -> INDEX_SOFT_DELETES_SETTING.get(settings) &&
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_7_4_0) ? "-1" : "512MB",
settings -> shouldDisableTranslogRetention(settings) ? "-1" : "512MB",
Property.Dynamic, Property.IndexScope);

/**
@@ -514,8 +511,8 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
this.indexSortConfig = new IndexSortConfig(this);
searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);
translogRetentionAge = scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING);
translogRetentionSize = scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING);
setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING));
setTranslogRetentionSize(scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING));

scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING,
@@ -581,11 +578,21 @@ private void setFlushAfterMergeThresholdSize(ByteSizeValue byteSizeValue) {
}

private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) {
this.translogRetentionSize = byteSizeValue;
if (shouldDisableTranslogRetention(settings) && byteSizeValue.getBytes() >= 0) {
// ignore the translog retention settings if soft-deletes enabled
this.translogRetentionSize = new ByteSizeValue(-1);
} else {
this.translogRetentionSize = byteSizeValue;
}
}

private void setTranslogRetentionAge(TimeValue age) {
this.translogRetentionAge = age;
if (shouldDisableTranslogRetention(settings) && age.millis() >= 0) {
// ignore the translog retention settings if soft-deletes enabled
this.translogRetentionAge = TimeValue.MINUS_ONE;
} else {
this.translogRetentionAge = age;
}
}

private void setGenerationThresholdSize(final ByteSizeValue generationThresholdSize) {
@@ -768,6 +775,7 @@ public TimeValue getRefreshInterval() {
* Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries
*/
public ByteSizeValue getTranslogRetentionSize() {
assert shouldDisableTranslogRetention(settings) == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize;
return translogRetentionSize;
}

@@ -776,6 +784,7 @@ public ByteSizeValue getTranslogRetentionSize() {
* around
*/
public TimeValue getTranslogRetentionAge() {
assert shouldDisableTranslogRetention(settings) == false || translogRetentionAge.millis() == -1L : translogRetentionSize;
return translogRetentionAge;
}

@@ -787,6 +796,11 @@ public int getTranslogRetentionTotalFiles() {
return INDEX_TRANSLOG_RETENTION_TOTAL_FILES_SETTING.get(getSettings());
}

private static boolean shouldDisableTranslogRetention(Settings settings) {
return INDEX_SOFT_DELETES_SETTING.get(settings)
&& IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_7_4_0);
}

/**
* Returns the generation threshold size. As sequence numbers can cause multiple generations to
* be preserved for rollback purposes, we want to keep the size of individual generations from
Original file line number Diff line number Diff line change
@@ -547,23 +547,63 @@ public void testSoftDeletesDefaultSetting() {
assertTrue(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
}

public void testDefaultTranslogRetentionSettings() {
IndexSettings indexSettingsWithPRRL = new IndexSettings(newIndexMeta("index", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_7_4_0, Version.CURRENT))
.build()), Settings.EMPTY);
assertThat(indexSettingsWithPRRL.getTranslogRetentionAge().millis(), equalTo(-1L));
assertThat(indexSettingsWithPRRL.getTranslogRetentionSize().getBytes(), equalTo(-1L));

final Settings.Builder settings = Settings.builder();
public void testIgnoreTranslogRetentionSettingsIfSoftDeletesEnabled() {
Settings.Builder settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_7_4_0, Version.CURRENT));
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
settings.put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()));
} else {
settings.put(IndexMetaData.SETTING_VERSION_CREATED,
VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_3_0));
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
}
IndexSettings indexSettingsWithoutPRRL = new IndexSettings(newIndexMeta("index", settings.build()), Settings.EMPTY);
assertThat(indexSettingsWithoutPRRL.getTranslogRetentionAge(), equalTo(TimeValue.timeValueHours(12)));
assertThat(indexSettingsWithoutPRRL.getTranslogRetentionSize(), equalTo(new ByteSizeValue(512, ByteSizeUnit.MB)));
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
}
IndexMetaData metaData = newIndexMeta("index", settings.build());
IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY);
assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L));
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L));

Settings.Builder newSettings = Settings.builder().put(settings.build());
if (randomBoolean()) {
newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
}
if (randomBoolean()) {
newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
}
indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build()));
assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L));
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L));
}

public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() {
Settings.Builder settings = Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);

TimeValue ageSetting = TimeValue.timeValueHours(12);
if (randomBoolean()) {
ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000));
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting);
}
ByteSizeValue sizeSetting = new ByteSizeValue(512, ByteSizeUnit.MB);
if (randomBoolean()) {
sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024));
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting);
}
IndexMetaData metaData = newIndexMeta("index", settings.build());
IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY);
assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));

Settings.Builder newSettings = Settings.builder().put(settings.build());
if (randomBoolean()) {
ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000));
newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting);
}
if (randomBoolean()) {
sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024));
newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting);
}
indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build()));
assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));
}
}