Make sure _forcemerge respects max_num_segments.

server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.apache.lucene.index.FilterMergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.TieredMergePolicy;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Wrapper around {@link TieredMergePolicy} which doesn't respect
+ * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges.
+ * See https://issues.apache.org/jira/browse/LUCENE-7976.
+ */
+final class EsTieredMergePolicy extends FilterMergePolicy {
+
+    final TieredMergePolicy regularMergePolicy;
+    final TieredMergePolicy forcedMergePolicy;
+
+    EsTieredMergePolicy() {
+        super(new TieredMergePolicy());
+        regularMergePolicy = (TieredMergePolicy) in;
+        forcedMergePolicy = new TieredMergePolicy();
+        forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited
+    }
+
+    @Override
+    public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount,
+            Map<SegmentCommitInfo, Boolean> segmentsToMerge, MergeContext mergeContext) throws IOException {
+        return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext);
+    }
+
+    @Override
+    public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
+        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);    
+    }
+
+    public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
+        regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
+        forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
+    }
+
+    public double getForceMergeDeletesPctAllowed() {
+        return forcedMergePolicy.getForceMergeDeletesPctAllowed();
+    }
+
+    public void setFloorSegmentMB(double mbFrac) {
+        regularMergePolicy.setFloorSegmentMB(mbFrac);
+        forcedMergePolicy.setFloorSegmentMB(mbFrac);
+    }
+
+    public double getFloorSegmentMB() {
+        return regularMergePolicy.getFloorSegmentMB();
+    }
+
+    public void setMaxMergeAtOnce(int maxMergeAtOnce) {
+        regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
+        forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
+    }
+
+    public int getMaxMergeAtOnce() {
+        return regularMergePolicy.getMaxMergeAtOnce();
+    }
+
+    public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) {
+        regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
+        forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
+    }
+
+    public int getMaxMergeAtOnceExplicit() {
+        return forcedMergePolicy.getMaxMergeAtOnceExplicit();
+    }
+
+    // only setter that must NOT delegate to the forced merge policy
+    public void setMaxMergedSegmentMB(double mbFrac) {
+        regularMergePolicy.setMaxMergedSegmentMB(mbFrac);
+    }
+
+    public double getMaxMergedSegmentMB() {
+        return regularMergePolicy.getMaxMergedSegmentMB();
+    }
+
+    public void setSegmentsPerTier(double segmentsPerTier) {
+        regularMergePolicy.setSegmentsPerTier(segmentsPerTier);
+        forcedMergePolicy.setSegmentsPerTier(segmentsPerTier);
+    }
+
+    public double getSegmentsPerTier() {
+        return regularMergePolicy.getSegmentsPerTier();
+    }
+
+    public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
+        regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
+        forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
+    }
+
+    public double getReclaimDeletesWeight() {
+        return regularMergePolicy.getReclaimDeletesWeight();
+    }
+}

server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java

@@ -115,7 +115,7 @@
  */
 
 public final class MergePolicyConfig {
-    private final TieredMergePolicy mergePolicy = new TieredMergePolicy();
+    private final EsTieredMergePolicy mergePolicy = new EsTieredMergePolicy();
     private final Logger logger;
     private final boolean mergesEnabled;
 

server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.apache.lucene.index.TieredMergePolicy;
+import org.elasticsearch.test.ESTestCase;
+
+public class EsTieredMergePolicyTests extends ESTestCase {
+
+    public void testDefaults() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        assertEquals(
+                new TieredMergePolicy().getMaxMergedSegmentMB(),
+                policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
+        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
+        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+    }
+
+    public void testSetMaxMergedSegmentMB() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setMaxMergedSegmentMB(10 * 1024);
+        assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
+        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
+        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+    }
+
+    public void testSetForceMergeDeletesPctAllowed() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setForceMergeDeletesPctAllowed(42);
+        assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0);
+    }
+
+    public void testSetFloorSegmentMB() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setFloorSegmentMB(42);
+        assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0);
+        assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0);
+    }
+
+    public void testSetMaxMergeAtOnce() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setMaxMergeAtOnce(42);
+        assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce());
+    }
+
+    public void testSetMaxMergeAtOnceExplicit() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setMaxMergeAtOnceExplicit(42);
+        assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit());
+    }
+
+    public void testSetSegmentsPerTier() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setSegmentsPerTier(42);
+        assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
+    }
+
+    public void testSetReclaimDeletesWeight() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setReclaimDeletesWeight(42);
+        assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
+    }
+}

server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java

@@ -19,7 +19,6 @@
 package org.elasticsearch.index;
 
 import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.TieredMergePolicy;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -76,43 +75,43 @@ public void testUpdateSettings() throws IOException {
 
     public void testTieredMergePolicySettingsUpdate() throws IOException {
         IndexSettings indexSettings = indexSettings(Settings.EMPTY);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
 
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
 
         indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
     }
 
     public Settings build(String value) {