-
Notifications
You must be signed in to change notification settings - Fork 24.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Reduce memory usage of match all bitset (#92777)
By default, Elasticsearch uses up to 1 bit per document to store the set of root documents when the index mapping has nested fields. This PR introduces a special BitSet using less memory for match_all filters. This optimization is only triggered when the index mapping has a nested field, but that field never exists in documents.
- Loading branch information
Showing
10 changed files
with
242 additions
and
132 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| pr: 92777 | ||
| summary: Reduce memory usage of match all bitset | ||
| area: Search | ||
| type: enhancement | ||
| issues: [] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
server/src/main/java/org/elasticsearch/lucene/util/BitSets.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.lucene.util; | ||
|
|
||
| import org.apache.lucene.search.DocIdSetIterator; | ||
| import org.apache.lucene.util.BitSet; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public final class BitSets { | ||
| private BitSets() {} | ||
|
|
||
| /** | ||
| * Build a {@link BitSet} from the content of the provided {@link DocIdSetIterator}. If the iterator matches all documents, | ||
| * then this method will wrap the returned Bitset as {@link MatchAllBitSet} to reduce memory usage. | ||
| */ | ||
| public static BitSet of(DocIdSetIterator iter, int maxDocs) throws IOException { | ||
| final BitSet bitSet = BitSet.of(iter, maxDocs); | ||
| if (bitSet.cardinality() == maxDocs) { | ||
| return new MatchAllBitSet(maxDocs); | ||
| } else { | ||
| return bitSet; | ||
| } | ||
| } | ||
| } |
90 changes: 90 additions & 0 deletions
90
server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.lucene.util; | ||
|
|
||
| import org.apache.lucene.search.DocIdSetIterator; | ||
| import org.apache.lucene.util.BitSet; | ||
| import org.apache.lucene.util.RamUsageEstimator; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * An optimized implementation of {@link BitSet} that matches all documents to reduce memory usage. | ||
| */ | ||
| public final class MatchAllBitSet extends BitSet { | ||
| private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MatchAllBitSet.class); | ||
|
|
||
| private final int numBits; | ||
|
|
||
| public MatchAllBitSet(int numBits) { | ||
| this.numBits = numBits; | ||
| } | ||
|
|
||
| @Override | ||
| public void set(int i) { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public boolean getAndSet(int i) { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public void clear(int i) { | ||
| assert false : "MatchAllBitSet doesn't support clear"; | ||
| throw new UnsupportedOperationException("MatchAllBitSet doesn't support clear"); | ||
| } | ||
|
|
||
| @Override | ||
| public void clear(int startIndex, int endIndex) { | ||
| assert false : "MatchAllBitSet doesn't support clear"; | ||
| throw new UnsupportedOperationException("MatchAllBitSet doesn't support clear"); | ||
| } | ||
|
|
||
| @Override | ||
| public int cardinality() { | ||
| return numBits; | ||
| } | ||
|
|
||
| @Override | ||
| public int approximateCardinality() { | ||
| return numBits; | ||
| } | ||
|
|
||
| @Override | ||
| public int prevSetBit(int index) { | ||
| return index; | ||
| } | ||
|
|
||
| @Override | ||
| public int nextSetBit(int index) { | ||
| return index; | ||
| } | ||
|
|
||
| @Override | ||
| public long ramBytesUsed() { | ||
| return RAM_BYTES_USED; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean get(int index) { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public int length() { | ||
| return numBits; | ||
| } | ||
|
|
||
| @Override | ||
| public void or(DocIdSetIterator iter) throws IOException { | ||
|
|
||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
server/src/test/java/org/elasticsearch/lucene/util/BitSetsTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.lucene.util; | ||
|
|
||
| import org.apache.lucene.search.DocIdSetIterator; | ||
| import org.apache.lucene.util.BitSet; | ||
| import org.apache.lucene.util.BitSetIterator; | ||
| import org.apache.lucene.util.FixedBitSet; | ||
| import org.elasticsearch.test.ESTestCase; | ||
|
|
||
| import static org.hamcrest.Matchers.equalTo; | ||
| import static org.hamcrest.Matchers.instanceOf; | ||
|
|
||
| public class BitSetsTests extends ESTestCase { | ||
|
|
||
| public void testRandomBitset() throws Exception { | ||
| int maxDocs = randomIntBetween(1, 1024); | ||
| int numDocs = 0; | ||
| FixedBitSet matches = new FixedBitSet(maxDocs); | ||
| for (int i = 0; i < maxDocs; i++) { | ||
| if (numDocs < maxDocs && randomBoolean()) { | ||
| numDocs++; | ||
| matches.set(i); | ||
| } | ||
| } | ||
| DocIdSetIterator it = new BitSetIterator(matches, randomIntBetween(0, numDocs)); | ||
| BitSet bitSet = BitSets.of(it, maxDocs); | ||
| assertThat(bitSet.cardinality(), equalTo(numDocs)); | ||
| assertThat(bitSet.length(), equalTo(maxDocs)); | ||
| for (int i = 0; i < maxDocs; i++) { | ||
| assertThat(bitSet.get(i), equalTo(matches.get(i))); | ||
| assertThat(bitSet.nextSetBit(i), equalTo(matches.nextSetBit(i))); | ||
| assertThat(bitSet.prevSetBit(i), equalTo(matches.prevSetBit(i))); | ||
| } | ||
| } | ||
|
|
||
| public void testMatchAllBitSet() throws Exception { | ||
| int maxDocs = randomIntBetween(1, 128); | ||
| FixedBitSet matches = new FixedBitSet(maxDocs); | ||
| for (int i = 0; i < maxDocs; i++) { | ||
| matches.set(i); | ||
| } | ||
| DocIdSetIterator it = new BitSetIterator(matches, randomNonNegativeLong()); | ||
| BitSet bitSet = BitSets.of(it, maxDocs); | ||
| assertThat(bitSet, instanceOf(MatchAllBitSet.class)); | ||
| for (int i = 0; i < maxDocs; i++) { | ||
| assertTrue(bitSet.get(i)); | ||
| assertThat(bitSet.nextSetBit(i), equalTo(matches.nextSetBit(i))); | ||
| assertThat(bitSet.prevSetBit(i), equalTo(matches.prevSetBit(i))); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.