-
Notifications
You must be signed in to change notification settings - Fork 24.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Make sure _forcemerge respects
max_num_segments. (#32291)
An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976) will make TieredMergePolicy respect the maximum merged segment size all the time, meaning it will possibly not respect the `max_num_segments` parameter anymore if the shard is larger than the maximum segment size. This change makes sure that `max_num_segments` is respected for now in order to give us time to think about how to integrate this change, and also to delay it until 7.0 as this might be a big-enough change for us to wait for a new major version.
- Loading branch information
Showing
4 changed files
with
222 additions
and
23 deletions.
There are no files selected for viewing
120 changes: 120 additions & 0 deletions
120
server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index; | ||
|
|
||
| import org.apache.lucene.index.FilterMergePolicy; | ||
| import org.apache.lucene.index.SegmentCommitInfo; | ||
| import org.apache.lucene.index.SegmentInfos; | ||
| import org.apache.lucene.index.TieredMergePolicy; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * Wrapper around {@link TieredMergePolicy} which doesn't respect | ||
| * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges. | ||
| * See https://issues.apache.org/jira/browse/LUCENE-7976. | ||
| */ | ||
| final class EsTieredMergePolicy extends FilterMergePolicy { | ||
|
|
||
| final TieredMergePolicy regularMergePolicy; | ||
| final TieredMergePolicy forcedMergePolicy; | ||
|
|
||
| EsTieredMergePolicy() { | ||
| super(new TieredMergePolicy()); | ||
| regularMergePolicy = (TieredMergePolicy) in; | ||
| forcedMergePolicy = new TieredMergePolicy(); | ||
| forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited | ||
| } | ||
|
|
||
| @Override | ||
| public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, | ||
| Map<SegmentCommitInfo, Boolean> segmentsToMerge, MergeContext mergeContext) throws IOException { | ||
| return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext); | ||
| } | ||
|
|
||
| @Override | ||
| public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException { | ||
| return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext); | ||
| } | ||
|
|
||
| public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) { | ||
| regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); | ||
| forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); | ||
| } | ||
|
|
||
| public double getForceMergeDeletesPctAllowed() { | ||
| return forcedMergePolicy.getForceMergeDeletesPctAllowed(); | ||
| } | ||
|
|
||
| public void setFloorSegmentMB(double mbFrac) { | ||
| regularMergePolicy.setFloorSegmentMB(mbFrac); | ||
| forcedMergePolicy.setFloorSegmentMB(mbFrac); | ||
| } | ||
|
|
||
| public double getFloorSegmentMB() { | ||
| return regularMergePolicy.getFloorSegmentMB(); | ||
| } | ||
|
|
||
| public void setMaxMergeAtOnce(int maxMergeAtOnce) { | ||
| regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); | ||
| forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); | ||
| } | ||
|
|
||
| public int getMaxMergeAtOnce() { | ||
| return regularMergePolicy.getMaxMergeAtOnce(); | ||
| } | ||
|
|
||
| public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) { | ||
| regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); | ||
| forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); | ||
| } | ||
|
|
||
| public int getMaxMergeAtOnceExplicit() { | ||
| return forcedMergePolicy.getMaxMergeAtOnceExplicit(); | ||
| } | ||
|
|
||
| // only setter that must NOT delegate to the forced merge policy | ||
| public void setMaxMergedSegmentMB(double mbFrac) { | ||
| regularMergePolicy.setMaxMergedSegmentMB(mbFrac); | ||
| } | ||
|
|
||
| public double getMaxMergedSegmentMB() { | ||
| return regularMergePolicy.getMaxMergedSegmentMB(); | ||
| } | ||
|
|
||
| public void setSegmentsPerTier(double segmentsPerTier) { | ||
| regularMergePolicy.setSegmentsPerTier(segmentsPerTier); | ||
| forcedMergePolicy.setSegmentsPerTier(segmentsPerTier); | ||
| } | ||
|
|
||
| public double getSegmentsPerTier() { | ||
| return regularMergePolicy.getSegmentsPerTier(); | ||
| } | ||
|
|
||
| public void setReclaimDeletesWeight(double reclaimDeletesWeight) { | ||
| regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); | ||
| forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); | ||
| } | ||
|
|
||
| public double getReclaimDeletesWeight() { | ||
| return regularMergePolicy.getReclaimDeletesWeight(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
80 changes: 80 additions & 0 deletions
80
server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index; | ||
|
|
||
| import org.apache.lucene.index.TieredMergePolicy; | ||
| import org.elasticsearch.test.ESTestCase; | ||
|
|
||
| public class EsTieredMergePolicyTests extends ESTestCase { | ||
|
|
||
| public void testDefaults() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| assertEquals( | ||
| new TieredMergePolicy().getMaxMergedSegmentMB(), | ||
| policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d); | ||
| // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double | ||
| assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d); | ||
| } | ||
|
|
||
| public void testSetMaxMergedSegmentMB() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setMaxMergedSegmentMB(10 * 1024); | ||
| assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d); | ||
| // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double | ||
| assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d); | ||
| } | ||
|
|
||
| public void testSetForceMergeDeletesPctAllowed() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setForceMergeDeletesPctAllowed(42); | ||
| assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0); | ||
| } | ||
|
|
||
| public void testSetFloorSegmentMB() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setFloorSegmentMB(42); | ||
| assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0); | ||
| assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0); | ||
| } | ||
|
|
||
| public void testSetMaxMergeAtOnce() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setMaxMergeAtOnce(42); | ||
| assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce()); | ||
| } | ||
|
|
||
| public void testSetMaxMergeAtOnceExplicit() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setMaxMergeAtOnceExplicit(42); | ||
| assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit()); | ||
| } | ||
|
|
||
| public void testSetSegmentsPerTier() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setSegmentsPerTier(42); | ||
| assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0); | ||
| } | ||
|
|
||
| public void testSetReclaimDeletesWeight() { | ||
| EsTieredMergePolicy policy = new EsTieredMergePolicy(); | ||
| policy.setReclaimDeletesWeight(42); | ||
| assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters