diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index df90fe70497f5..592c1512d60cf 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -261,19 +261,6 @@ - - - - - - - - - - - - - @@ -318,8 +305,6 @@ - - @@ -513,10 +498,6 @@ - - - - diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java index 642270113cf76..bb3388bc943c9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java @@ -54,23 +54,6 @@ */ public interface IndexFieldData extends IndexComponent { - class CommonSettings { - public static final String SETTING_MEMORY_STORAGE_HINT = "memory_storage_hint"; - - public enum MemoryStorageFormat { - ORDINALS, PACKED, PAGED; - - public static MemoryStorageFormat fromString(String string) { - for (MemoryStorageFormat e : MemoryStorageFormat.values()) { - if (e.name().equalsIgnoreCase(string)) { - return e; - } - } - return null; - } - } - } - /** * The field name. */ diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java index 5238f06a7909c..83108ad571190 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java @@ -31,7 +31,8 @@ public interface IndexFieldDataCache { > FD load(LeafReaderContext context, IFD indexFieldData) throws Exception; - > IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception; + > IFD load(DirectoryReader indexReader, IFD indexFieldData) + throws Exception; /** * Clears all the field data stored cached in on this index. @@ -59,13 +60,15 @@ default void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lo class None implements IndexFieldDataCache { @Override - public > FD load(LeafReaderContext context, IFD indexFieldData) throws Exception { + public > FD load(LeafReaderContext context, IFD indexFieldData) + throws Exception { return indexFieldData.loadDirect(context); } @Override @SuppressWarnings("unchecked") - public > IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception { + public > IFD load(DirectoryReader indexReader, + IFD indexFieldData) throws Exception { return (IFD) indexFieldData.localGlobalDirect(indexReader); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 25b03bcb2506f..db5a7b437d35f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -42,15 +42,16 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Closeable { public static final String FIELDDATA_CACHE_VALUE_NODE = "node"; public static final String FIELDDATA_CACHE_KEY = "index.fielddata.cache"; - public static final Setting INDEX_FIELDDATA_CACHE_KEY = new Setting<>(FIELDDATA_CACHE_KEY, (s) -> FIELDDATA_CACHE_VALUE_NODE, (s) -> { - switch (s) { - case "node": - case "none": - return s; - default: - throw new IllegalArgumentException("failed to parse [" + s + "] must be one of [node,none]"); - } - }, Property.IndexScope); + public static final Setting INDEX_FIELDDATA_CACHE_KEY = + new Setting<>(FIELDDATA_CACHE_KEY, (s) -> FIELDDATA_CACHE_VALUE_NODE, (s) -> { + switch (s) { + case "node": + case "none": + return s; + default: + throw new IllegalArgumentException("failed to parse [" + s + "] must be one of [node,none]"); + } + }, Property.IndexScope); private final CircuitBreakerService circuitBreakerService; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index 0a273d88380eb..c414944801f79 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -43,7 +43,8 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato private final IndexNumericFieldData indexFieldData; - public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { + public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, + Nested nested) { super(missingValue, sortMode, nested); this.indexFieldData = indexFieldData; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index beb27644a1b95..4621c7fd287d5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -41,7 +41,8 @@ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparator private final IndexNumericFieldData indexFieldData; - public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { + public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, + Nested nested) { super(missingValue, sortMode, nested); this.indexFieldData = indexFieldData; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index f323709e8f5ee..48b0a1b155f92 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -40,7 +40,8 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS private final IndexNumericFieldData indexFieldData; - public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { + public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, + Nested nested) { super(missingValue, sortMode, nested); this.indexFieldData = indexFieldData; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index 646c7adb40408..3c8f100a17db5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -79,7 +79,8 @@ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexO ); } - public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { + public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, final IndexReader indexReader, + IndexOrdinalsFieldData indexFieldData) throws IOException { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java index 86e2787658c35..47307f27c4014 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java @@ -46,7 +46,8 @@ public class MultiOrdinals extends Ordinals { /** * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ - public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, float acceptableOverheadRatio) { + public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds, + float acceptableOverheadRatio) { int bitsPerOrd = PackedInts.bitsRequired(numOrds); bitsPerOrd = PackedInts.fastestFormatAndBits(numDocsWithValue, bitsPerOrd, acceptableOverheadRatio).bitsPerValue; // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index a2baf1fee6c0b..4b1292591cdd9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -19,14 +19,7 @@ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.packed.GrowableWriter; import org.apache.lucene.util.packed.PackedInts; @@ -43,11 +36,6 @@ */ public final class OrdinalsBuilder implements Closeable { - /** - * Whether to for the use of {@link MultiOrdinals} to store the ordinals for testing purposes. - */ - public static final String FORCE_MULTI_ORDINALS = "force_multi_ordinals"; - /** * Default acceptable overhead ratio. {@link OrdinalsBuilder} memory usage is mostly transient so it is likely a better trade-off to * trade memory for speed in order to resize less often. @@ -159,7 +147,8 @@ private static int numOrdinals(int level, long offset) { this.acceptableOverheadRatio = acceptableOverheadRatio; positions = new PagedGrowableWriter(maxDoc, PAGE_SIZE, startBitsPerValue, acceptableOverheadRatio); firstOrdinals = new GrowableWriter(startBitsPerValue, maxDoc, acceptableOverheadRatio); - // over allocate in order to never worry about the array sizes, 24 entries would allow to store several millions of ordinals per doc... + // over allocate in order to never worry about the array sizes, 24 entries would allow + // to store several millions of ordinals per doc... ordinals = new PagedGrowableWriter[24]; nextLevelSlices = new PagedGrowableWriter[24]; sizes = new int[24]; @@ -200,7 +189,8 @@ private int firstLevel(int docID, long ordinal) { } else { final long newSlice = newSlice(1); if (firstNextLevelSlices == null) { - firstNextLevelSlices = new PagedGrowableWriter(firstOrdinals.size(), PAGE_SIZE, 3, acceptableOverheadRatio); + firstNextLevelSlices = + new PagedGrowableWriter(firstOrdinals.size(), PAGE_SIZE, 3, acceptableOverheadRatio); } firstNextLevelSlices.set(docID, newSlice); final long offset = startOffset(1, newSlice); @@ -282,14 +272,14 @@ public void appendOrdinals(int docID, LongsRef ords) { private OrdinalsStore ordinals; private final LongsRef spare; - public OrdinalsBuilder(int maxDoc, float acceptableOverheadRatio) throws IOException { + public OrdinalsBuilder(int maxDoc, float acceptableOverheadRatio) { this.maxDoc = maxDoc; int startBitsPerValue = 8; ordinals = new OrdinalsStore(maxDoc, startBitsPerValue, acceptableOverheadRatio); spare = new LongsRef(); } - public OrdinalsBuilder(int maxDoc) throws IOException { + public OrdinalsBuilder(int maxDoc) { this(maxDoc, DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); } @@ -303,7 +293,7 @@ public LongsRef docOrds(int docID) { } /** - * Return a {@link org.apache.lucene.util.packed.PackedInts.Reader} instance mapping every doc ID to its first ordinal + 1 if it exists and 0 otherwise. + * Return a {@link PackedInts.Reader} instance mapping every doc ID to its first ordinal + 1 if it exists and 0 otherwise. */ public PackedInts.Reader getFirstOrdinals() { return ordinals.firstOrdinals; @@ -339,27 +329,6 @@ public OrdinalsBuilder addDoc(int doc) { return this; } - /** - * Returns true iff this builder contains a document ID that is associated with more than one ordinal. Otherwise false; - */ - public boolean isMultiValued() { - return numMultiValuedDocs > 0; - } - - /** - * Returns the number distinct of document IDs with one or more values. - */ - public int getNumDocsWithValue() { - return numDocsWithValue; - } - - /** - * Returns the number distinct of document IDs associated with exactly one value. - */ - public int getNumSingleValuedDocs() { - return numDocsWithValue - numMultiValuedDocs; - } - /** * Returns the number distinct of document IDs associated with two or more values. */ @@ -381,29 +350,15 @@ public long getValueCount() { return currentOrd + 1; } - /** - * Builds a {@link BitSet} where each documents bit is that that has one or more ordinals associated with it. - * if every document has an ordinal associated with it this method returns null - */ - public BitSet buildDocsWithValuesSet() { - if (numDocsWithValue == maxDoc) { - return null; - } - final FixedBitSet bitSet = new FixedBitSet(maxDoc); - for (int docID = 0; docID < maxDoc; ++docID) { - if (ordinals.firstOrdinals.get(docID) != 0) { - bitSet.set(docID); - } - } - return bitSet; - } - /** * Builds an {@link Ordinals} instance from the builders current state. */ public Ordinals build() { final float acceptableOverheadRatio = PackedInts.DEFAULT; - if (numMultiValuedDocs > 0 || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals(maxDoc, numDocsWithValue, getValueCount(), acceptableOverheadRatio)) { + if (numMultiValuedDocs > 0 + || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals( + maxDoc, numDocsWithValue, getValueCount(), acceptableOverheadRatio) + ) { // MultiOrdinals can be smaller than SinglePackedOrdinals for sparse fields return new MultiOrdinals(this, acceptableOverheadRatio); } else { @@ -419,33 +374,6 @@ public int maxDoc() { return maxDoc; } - /** - * This method iterates all terms in the given {@link TermsEnum} and - * associates each terms ordinal with the terms documents. The caller must - * exhaust the returned {@link BytesRefIterator} which returns all values - * where the first returned value is associated with the ordinal {@code 1} - * etc. - */ - public BytesRefIterator buildFromTerms(final TermsEnum termsEnum) throws IOException { - return new BytesRefIterator() { - private PostingsEnum docsEnum = null; - - @Override - public BytesRef next() throws IOException { - BytesRef ref; - if ((ref = termsEnum.next()) != null) { - docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE); - nextOrdinal(); - int docId; - while ((docId = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - addDoc(docId); - } - } - return ref; - } - }; - } - /** * Closes this builder and release all resources. */ diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index 27f0aadee8719..9bbdaeb3f7b54 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -42,7 +42,8 @@ public SinglePackedOrdinals(OrdinalsBuilder builder, float acceptableOverheadRat assert builder.getNumMultiValuesDocs() == 0; this.valueCount = (int) builder.getValueCount(); // We don't reuse the builder as-is because it might have been built with a higher overhead ratio - final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(valueCount), acceptableOverheadRatio); + final PackedInts.Mutable reader = + PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(valueCount), acceptableOverheadRatio); PackedInts.copy(builder.getFirstOrdinals(), 0, reader, 0, builder.maxDoc(), 8 * 1024); this.reader = reader; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index d89c6d64d4915..0dc0de838a3e5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -36,7 +36,8 @@ import java.io.IOException; -public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData implements IndexOrdinalsFieldData { +public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData + implements IndexOrdinalsFieldData { private final double minFrequency, maxFrequency; private final int minSegmentSize; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java index 6c92d571196ec..ed77d3d5f8b37 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java @@ -43,7 +43,8 @@ public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndex } @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, + boolean reverse) { throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java index a7e1981766704..13d90ea36e738 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java @@ -46,7 +46,8 @@ public BinaryDVAtomicFieldData loadDirect(LeafReaderContext context) throws Exce } @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, + boolean reverse) { XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); /** * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/NonEstimatingEstimator.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/NonEstimatingEstimator.java deleted file mode 100644 index 481b2b3c84ca0..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/NonEstimatingEstimator.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreaker; - -import java.io.IOException; - -/** - * Estimator that does nothing except for adjust the breaker after the field - * data has been loaded. Useful for field data implementations that do not yet - * have pre-loading estimations. - */ -public class NonEstimatingEstimator implements AbstractIndexFieldData.PerValueEstimator { - - private final CircuitBreaker breaker; - - NonEstimatingEstimator(CircuitBreaker breaker) { - this.breaker = breaker; - } - - @Override - public long bytesPerValue(BytesRef term) { - return 0; - } - - @Override - public TermsEnum beforeLoad(Terms terms) throws IOException { - return null; - } - - @Override - public void afterLoad(@Nullable TermsEnum termsEnum, long actualUsed) { - breaker.addWithoutBreaking(actualUsed); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java index e6100d2f89311..66a44a95afc7f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -65,7 +65,7 @@ public Builder(double minFrequency, double maxFrequency, int minSegmentSize) { @Override public IndexOrdinalsFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, - IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { return new PagedBytesIndexFieldData(indexSettings, fieldType.name(), cache, breakerService, minFrequency, maxFrequency, minSegmentSize); } @@ -78,7 +78,8 @@ public PagedBytesIndexFieldData(IndexSettings indexSettings, String fieldName, } @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, + boolean reverse) { XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); return new SortField(getFieldName(), source, reverse); } @@ -88,7 +89,8 @@ public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exce LeafReader reader = context.reader(); AtomicOrdinalsFieldData data = null; - PagedBytesEstimator estimator = new PagedBytesEstimator(context, breakerService.getBreaker(CircuitBreaker.FIELDDATA), getFieldName()); + PagedBytesEstimator estimator = + new PagedBytesEstimator(context, breakerService.getBreaker(CircuitBreaker.FIELDDATA), getFieldName()); Terms terms = reader.terms(getFieldName()); if (terms == null) { data = AbstractAtomicOrdinalsFieldData.empty(); @@ -250,14 +252,5 @@ public void afterLoad(TermsEnum termsEnum, long actualUsed) { breaker.addWithoutBreaking(-(estimatedBytes - actualUsed)); } - /** - * Adjust the breaker when no terms were actually loaded, but the field - * data takes up space regardless. For instance, when ordinals are - * used. - * @param actualUsed bytes actually used - */ - public void adjustForNoTerms(long actualUsed) { - breaker.addWithoutBreaking(actualUsed); - } } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java index d336c1d5cd7ce..5e8a30ab6e4ea 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java @@ -32,7 +32,7 @@ import java.util.function.Function; /** - * An {@link AtomicFieldData} implementation that uses Lucene {@link org.apache.lucene.index.SortedSetDocValues}. + * An {@link AtomicFieldData} implementation that uses Lucene {@link SortedSetDocValues}. */ public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData { diff --git a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java index e7573ae9f71a0..4a784af6bb3f3 100644 --- a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -90,7 +90,10 @@ public void onRemoval(RemovalNotification notification) { final Accountable value = notification.getValue(); for (IndexFieldDataCache.Listener listener : key.listeners) { try { - listener.onRemoval(key.shardId, indexCache.fieldName, notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED, value.ramBytesUsed()); + listener.onRemoval( + key.shardId, indexCache.fieldName, + notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED, value.ramBytesUsed() + ); } catch (Exception e) { // load anyway since listeners should not throw exceptions logger.error("Failed to call listener on field data cache unloading", e); @@ -125,7 +128,8 @@ static class IndexFieldCache implements IndexFieldDataCache, IndexReader.ClosedL } @Override - public > FD load(final LeafReaderContext context, final IFD indexFieldData) throws Exception { + public > FD load(final LeafReaderContext context, + final IFD indexFieldData) throws Exception { final ShardId shardId = ShardUtils.extractShardId(context.reader()); final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper(); if (cacheHelper == null) { @@ -151,7 +155,8 @@ public > FD load(fina } @Override - public > IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception { + public > IFD load(final DirectoryReader indexReader, + final IFD indexFieldData) throws Exception { final ShardId shardId = ShardUtils.extractShardId(indexReader); final IndexReader.CacheHelper cacheHelper = indexReader.getReaderCacheHelper(); if (cacheHelper == null) { diff --git a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java deleted file mode 100644 index 1995bb2dfb805..0000000000000 --- a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.fielddata.cache; - -import org.apache.lucene.util.Accountable; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.breaker.CircuitBreakerService; - -/** - * A {@link org.elasticsearch.index.fielddata.IndexFieldDataCache.Listener} implementation that updates indices (node) level statistics / service about - * field data entries being loaded and unloaded. - * - * Currently it only decrements the memory used in the {@link CircuitBreakerService}. - */ -public class IndicesFieldDataCacheListener implements IndexFieldDataCache.Listener { - - private final CircuitBreakerService circuitBreakerService; - - @Inject - public IndicesFieldDataCacheListener(CircuitBreakerService circuitBreakerService) { - this.circuitBreakerService = circuitBreakerService; - } - - @Override - public void onCache(ShardId shardId, String fieldName, Accountable fieldData) { - } - - @Override - public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) { - assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or equal to 0 and not [" + sizeInBytes + "]"; - circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes); - } - -} - diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 7decbe9024fdf..6d9ec95af16cd 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -135,7 +135,9 @@ public void setup() throws Exception { mapperService = indexService.mapperService(); indicesFieldDataCache = getInstanceFromNode(IndicesService.class).getIndicesFieldDataCache(); // LogByteSizeMP to preserve doc ID order - writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy())); + writer = new IndexWriter( + new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) + ); shardContext = indexService.newQueryShardContext(0, null, () -> 0, null); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index ef2a9b3873580..21d84203f6df3 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -264,7 +264,8 @@ public void testActualMissingValue(boolean reverse) throws IOException { final String missingValue = values[1]; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer)); SortField sortField = indexFieldData.sortField(missingValue, MultiValueMode.MIN, null, reverse); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField)); + TopFieldDocs topDocs = + searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField)); assertEquals(numDocs, topDocs.totalHits.value); BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); for (int i = 0; i < topDocs.scoreDocs.length; ++i) { @@ -318,7 +319,8 @@ public void testSortMissing(boolean first, boolean reverse) throws IOException { final IndexFieldData indexFieldData = getForField("value"); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer)); SortField sortField = indexFieldData.sortField(first ? "_first" : "_last", MultiValueMode.MIN, null, reverse); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField)); + TopFieldDocs topDocs = + searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField)); assertEquals(numDocs, topDocs.totalHits.value); BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); for (int i = 0; i < topDocs.scoreDocs.length; ++i) { @@ -406,8 +408,10 @@ public void testNestedSorting(MultiValueMode sortMode) throws IOException { Query parentFilter = new TermQuery(new Term("type", "parent")); Query childFilter = Queries.not(parentFilter); Nested nested = createNested(searcher, parentFilter, childFilter); - BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested); - ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); + BytesRefFieldComparatorSource nestedComparatorSource = + new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested); + ToParentBlockJoinQuery query = + new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("text", nestedComparatorSource)); TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort); assertTrue(topDocs.scoreDocs.length > 0); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java index 375b10e262766..2590dc2660333 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java @@ -65,7 +65,8 @@ public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception { } } iw.close(); - DirectoryReader ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId("_index", "_na_", 0)); + DirectoryReader ir = + ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId("_index", "_na_", 0)); DummyAccountingFieldDataCache fieldDataCache = new DummyAccountingFieldDataCache(); // Testing SortedSetDVOrdinalsIndexFieldData: @@ -114,12 +115,14 @@ private class DummyAccountingFieldDataCache implements IndexFieldDataCache { private int cachedGlobally = 0; @Override - public > FD load(LeafReaderContext context, IFD indexFieldData) throws Exception { + public > FD load(LeafReaderContext context, IFD indexFieldData) + throws Exception { return indexFieldData.loadDirect(context); } @Override - public > IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception { + public > IFD load(DirectoryReader indexReader, + IFD indexFieldData) throws Exception { cachedGlobally++; return (IFD) indexFieldData.localGlobalDirect(indexReader); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 2eba60a1a5f83..7b17b727be2b1 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -252,7 +252,8 @@ private void doTestRequireDocValues(MappedFieldType ft) { ThreadPool threadPool = new TestThreadPool("random_threadpool_name"); try { IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null); - IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null); + IndexFieldDataService ifds = + new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null); ft.setName("some_long"); ft.setHasDocValues(true); ifds.getForField(ft); // no exception