From 7b494beb5f12f04695414daae4f0dd6e33be73b1 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 23 Jul 2018 18:57:29 +0200 Subject: [PATCH] Make sure _forcemerge respects `max_num_segments`. An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976) will make TieredMergePolicy respect the maximum merged segment size all the time, meaning it will possibly not respect the `max_num_segments` parameter anymore if the shard is larger than the maximum segment size. This change makes sure that `max_num_segments` is respected for now in order to give us time to think about how to integrate this change, and also to delay it until 7.0 as this might be a big-enough change for us to wait for a new major version. --- .../index/EsTieredMergePolicy.java | 120 ++++++++++++++++++ .../index/MergePolicyConfig.java | 2 +- .../index/EsTieredMergePolicyTests.java | 80 ++++++++++++ .../index/MergePolicySettingsTests.java | 43 +++---- 4 files changed, 222 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java create mode 100644 server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java diff --git a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java new file mode 100644 index 0000000000000..27a8396190362 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.apache.lucene.index.FilterMergePolicy; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.TieredMergePolicy; + +import java.io.IOException; +import java.util.Map; + +/** + * Wrapper around {@link TieredMergePolicy} which doesn't respect + * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges. + * See https://issues.apache.org/jira/browse/LUCENE-7976. + */ +final class EsTieredMergePolicy extends FilterMergePolicy { + + final TieredMergePolicy regularMergePolicy; + final TieredMergePolicy forcedMergePolicy; + + EsTieredMergePolicy() { + super(new TieredMergePolicy()); + regularMergePolicy = (TieredMergePolicy) in; + forcedMergePolicy = new TieredMergePolicy(); + forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited + } + + @Override + public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, + Map segmentsToMerge, MergeContext mergeContext) throws IOException { + return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext); + } + + @Override + public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException { + return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext); + } + + public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) { + regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); + forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); + } + + public double getForceMergeDeletesPctAllowed() { + return forcedMergePolicy.getForceMergeDeletesPctAllowed(); + } + + public void setFloorSegmentMB(double mbFrac) { + regularMergePolicy.setFloorSegmentMB(mbFrac); + forcedMergePolicy.setFloorSegmentMB(mbFrac); + } + + public double getFloorSegmentMB() { + return regularMergePolicy.getFloorSegmentMB(); + } + + public void setMaxMergeAtOnce(int maxMergeAtOnce) { + regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); + forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); + } + + public int getMaxMergeAtOnce() { + return regularMergePolicy.getMaxMergeAtOnce(); + } + + public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) { + regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); + forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); + } + + public int getMaxMergeAtOnceExplicit() { + return forcedMergePolicy.getMaxMergeAtOnceExplicit(); + } + + // only setter that must NOT delegate to the forced merge policy + public void setMaxMergedSegmentMB(double mbFrac) { + regularMergePolicy.setMaxMergedSegmentMB(mbFrac); + } + + public double getMaxMergedSegmentMB() { + return regularMergePolicy.getMaxMergedSegmentMB(); + } + + public void setSegmentsPerTier(double segmentsPerTier) { + regularMergePolicy.setSegmentsPerTier(segmentsPerTier); + forcedMergePolicy.setSegmentsPerTier(segmentsPerTier); + } + + public double getSegmentsPerTier() { + return regularMergePolicy.getSegmentsPerTier(); + } + + public void setReclaimDeletesWeight(double reclaimDeletesWeight) { + regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); + forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); + } + + public double getReclaimDeletesWeight() { + return regularMergePolicy.getReclaimDeletesWeight(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java index 0f7305789ecb5..b6c1e124a638e 100644 --- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java @@ -115,7 +115,7 @@ */ public final class MergePolicyConfig { - private final TieredMergePolicy mergePolicy = new TieredMergePolicy(); + private final EsTieredMergePolicy mergePolicy = new EsTieredMergePolicy(); private final Logger logger; private final boolean mergesEnabled; diff --git a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java new file mode 100644 index 0000000000000..fdee707d97a69 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.apache.lucene.index.TieredMergePolicy; +import org.elasticsearch.test.ESTestCase; + +public class EsTieredMergePolicyTests extends ESTestCase { + + public void testDefaults() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + assertEquals( + new TieredMergePolicy().getMaxMergedSegmentMB(), + policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d); + // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double + assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d); + } + + public void testSetMaxMergedSegmentMB() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setMaxMergedSegmentMB(10 * 1024); + assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d); + // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double + assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d); + } + + public void testSetForceMergeDeletesPctAllowed() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setForceMergeDeletesPctAllowed(42); + assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0); + } + + public void testSetFloorSegmentMB() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setFloorSegmentMB(42); + assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0); + assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0); + } + + public void testSetMaxMergeAtOnce() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setMaxMergeAtOnce(42); + assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce()); + } + + public void testSetMaxMergeAtOnceExplicit() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setMaxMergeAtOnceExplicit(42); + assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit()); + } + + public void testSetSegmentsPerTier() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setSegmentsPerTier(42); + assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0); + } + + public void testSetReclaimDeletesWeight() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setReclaimDeletesWeight(42); + assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java index ff60a206229d6..d8b03ffeedbfc 100644 --- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.TieredMergePolicy; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -76,43 +75,43 @@ public void testUpdateSettings() throws IOException { public void testTieredMergePolicySettingsUpdate() throws IOException { IndexSettings indexSettings = indexSettings(Settings.EMPTY); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0); indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); } public Settings build(String value) {