From a48dc123b3e65928d5e59a76735cf6c88099915a Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 12 Nov 2020 16:10:00 +0100 Subject: [PATCH] LUCENE-9378: Make it possible to configure how to trade speed for compression on doc values. (#2069) This adds a switch to `Lucene80DocValuesFormat` which allows to configure whether to prioritize retrieval speed over compression ratio or the other way around. When prioritizing retrieval speed, binary doc values are written using the exact same format as before more aggressive compression got introduced. --- lucene/CHANGES.txt | 3 + .../lucene80/Lucene80DocValuesConsumer.java | 80 ++++++++++++++++++- .../lucene80/Lucene80DocValuesFormat.java | 24 +++++- .../lucene80/Lucene80DocValuesProducer.java | 36 ++++++--- .../lucene/codecs/lucene87/Lucene87Codec.java | 27 ++++++- ... BaseLucene80DocValuesFormatTestCase.java} | 12 +-- ...estCompressionLucene80DocValuesFormat.java | 33 ++++++++ .../TestBestSpeedLucene80DocValuesFormat.java | 33 ++++++++ ...ne87StoredFieldsFormatHighCompression.java | 2 +- .../org/apache/lucene/index/RandomCodec.java | 3 +- .../util/TestRuleSetupAndRestoreClassEnv.java | 3 +- .../apache/solr/core/SchemaCodecFactory.java | 2 +- .../apache/solr/core/TestCodecSupport.java | 2 +- 13 files changed, 225 insertions(+), 35 deletions(-) rename lucene/core/src/test/org/apache/lucene/codecs/lucene80/{TestLucene80DocValuesFormat.java => BaseLucene80DocValuesFormatTestCase.java} (98%) create mode 100644 lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java create mode 100644 lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 65cb05899f33..223a0824eaa1 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -19,6 +19,9 @@ New Features * LUCENE-9594: FeatureField supports newLinearQuery that for scoring uses raw indexed values of features without any transformation. (Mayya Sharipova, Adrien Grand) +* LUCENE-9378: Doc values now allow configuring how to trade compression for + retrieval speed. (Adrien Grand) + Improvements --------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesConsumer.java index 33bf261291ad..290812b9c2ff 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesConsumer.java @@ -64,12 +64,14 @@ /** writer for {@link Lucene80DocValuesFormat} */ final class Lucene80DocValuesConsumer extends DocValuesConsumer implements Closeable { + final Lucene80DocValuesFormat.Mode mode; IndexOutput data, meta; final int maxDoc; private final SegmentWriteState state; /** expert: Creates a new writer */ - public Lucene80DocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException { + public Lucene80DocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension, Lucene80DocValuesFormat.Mode mode) throws IOException { + this.mode = mode; boolean success = false; try { this.state = state; @@ -490,13 +492,86 @@ public void close() throws IOException { } } - @Override public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene80DocValuesFormat.BINARY); + switch (mode) { + case BEST_SPEED: + meta.writeByte((byte) 0); + doAddUncompressedBinaryField(field, valuesProducer); + break; + case BEST_COMPRESSION: + meta.writeByte((byte) 1); + doAddCompressedBinaryField(field, valuesProducer); + break; + default: + throw new AssertionError(); + } + } + + private void doAddUncompressedBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + BinaryDocValues values = valuesProducer.getBinary(field); + long start = data.getFilePointer(); + meta.writeLong(start); // dataOffset + int numDocsWithField = 0; + int minLength = Integer.MAX_VALUE; + int maxLength = 0; + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + numDocsWithField++; + BytesRef v = values.binaryValue(); + int length = v.length; + data.writeBytes(v.bytes, v.offset, v.length); + minLength = Math.min(length, minLength); + maxLength = Math.max(length, maxLength); + } + assert numDocsWithField <= maxDoc; + meta.writeLong(data.getFilePointer() - start); // dataLength + + if (numDocsWithField == 0) { + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithField == maxDoc) { + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + values = valuesProducer.getBinary(field); + final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + meta.writeInt(numDocsWithField); + meta.writeInt(minLength); + meta.writeInt(maxLength); + if (maxLength > minLength) { + start = data.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1, DIRECT_MONOTONIC_BLOCK_SHIFT); + long addr = 0; + writer.add(addr); + values = valuesProducer.getBinary(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + addr += values.binaryValue().length; + writer.add(addr); + } + writer.finish(); + meta.writeLong(data.getFilePointer() - start); + } + } + + private void doAddCompressedBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { try (CompressedBinaryBlockWriter blockWriter = new CompressedBinaryBlockWriter()){ BinaryDocValues values = valuesProducer.getBinary(field); long start = data.getFilePointer(); @@ -542,7 +617,6 @@ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) th meta.writeInt(maxLength); blockWriter.writeMetaData(); - } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesFormat.java index 9f0befc74d40..5f930a6c13d9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesFormat.java @@ -18,6 +18,7 @@ import java.io.IOException; +import java.util.Objects; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; @@ -131,14 +132,30 @@ */ public final class Lucene80DocValuesFormat extends DocValuesFormat { - /** Sole Constructor */ + /** Configuration option for doc values. */ + public static enum Mode { + /** Trade compression ratio for retrieval speed. */ + BEST_SPEED, + /** Trade retrieval speed for compression ratio. */ + BEST_COMPRESSION + } + + private final Mode mode; + + /** Default constructor. */ public Lucene80DocValuesFormat() { + this(Mode.BEST_SPEED); + } + + /** Constructor */ + public Lucene80DocValuesFormat(Mode mode) { super("Lucene80"); + this.mode = Objects.requireNonNull(mode); } @Override public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new Lucene80DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION); + return new Lucene80DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION, mode); } @Override @@ -152,7 +169,8 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti static final String META_EXTENSION = "dvm"; static final int VERSION_START = 0; static final int VERSION_BIN_COMPRESSED = 1; - static final int VERSION_CURRENT = VERSION_BIN_COMPRESSED; + static final int VERSION_CONFIGURABLE_COMPRESSION = 2; + static final int VERSION_CURRENT = VERSION_CONFIGURABLE_COMPRESSION; // indicates docvalues type static final byte NUMERIC = 0; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesProducer.java index 5e721e8d3d1b..685090cd86cf 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesProducer.java @@ -174,6 +174,20 @@ private void readNumeric(ChecksumIndexInput meta, NumericEntry entry) throws IOE private BinaryEntry readBinary(ChecksumIndexInput meta) throws IOException { BinaryEntry entry = new BinaryEntry(); + if (version >= Lucene80DocValuesFormat.VERSION_CONFIGURABLE_COMPRESSION) { + int b = meta.readByte(); + switch (b) { + case 0: + case 1: + // valid + break; + default: + throw new CorruptIndexException("Unexpected byte: " + b + ", expected 0 or 1", meta); + } + entry.compressed = b != 0; + } else { + entry.compressed = version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED; + } entry.dataOffset = meta.readLong(); entry.dataLength = meta.readLong(); entry.docsWithFieldOffset = meta.readLong(); @@ -183,19 +197,19 @@ private BinaryEntry readBinary(ChecksumIndexInput meta) throws IOException { entry.numDocsWithField = meta.readInt(); entry.minLength = meta.readInt(); entry.maxLength = meta.readInt(); - if ((version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED && entry.numDocsWithField > 0) || entry.minLength < entry.maxLength) { + if ((entry.compressed && entry.numDocsWithField > 0) || entry.minLength < entry.maxLength) { entry.addressesOffset = meta.readLong(); // Old count of uncompressed addresses long numAddresses = entry.numDocsWithField + 1L; // New count of compressed addresses - the number of compresseed blocks - if (version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED) { + if (entry.compressed) { entry.numCompressedChunks = meta.readVInt(); entry.docsPerChunkShift = meta.readVInt(); entry.maxUncompressedChunkSize = meta.readVInt(); numAddresses = entry.numCompressedChunks; } - + final int blockShift = meta.readVInt(); entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, numAddresses, blockShift); ramBytesUsed += entry.addressesMeta.ramBytesUsed(); @@ -303,6 +317,7 @@ private static class NumericEntry { } private static class BinaryEntry { + boolean compressed; long dataOffset; long dataLength; long docsWithFieldOffset; @@ -680,9 +695,7 @@ public boolean advanceExact(int target) throws IOException { } } - // BWC - old binary format - private BinaryDocValues getUncompressedBinary(FieldInfo field) throws IOException { - BinaryEntry entry = binaries.get(field.name); + private BinaryDocValues getUncompressedBinary(BinaryEntry entry) throws IOException { if (entry.docsWithFieldOffset == -2) { return DocValues.emptyBinary(); } @@ -844,11 +857,16 @@ BytesRef decode(int docNumber) throws IOException { @Override public BinaryDocValues getBinary(FieldInfo field) throws IOException { - if (version < Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED) { - return getUncompressedBinary(field); + BinaryEntry entry = binaries.get(field.name); + if (entry.compressed) { + return getCompressedBinary(entry); + } else { + return getUncompressedBinary(entry); } + } + + private BinaryDocValues getCompressedBinary(BinaryEntry entry) throws IOException { - BinaryEntry entry = binaries.get(field.name); if (entry.docsWithFieldOffset == -2) { return DocValues.emptyBinary(); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87Codec.java index 5ff407384e22..289e37f39d83 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87Codec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87Codec.java @@ -35,6 +35,7 @@ import org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat; import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat; import org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.lucene80.Lucene80NormsFormat; import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.codecs.lucene86.Lucene86PointsFormat; @@ -54,6 +55,23 @@ * @lucene.experimental */ public class Lucene87Codec extends Codec { + + /** Configuration option for the codec. */ + public static enum Mode { + /** Trade compression ratio for retrieval speed. */ + BEST_SPEED(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, Lucene80DocValuesFormat.Mode.BEST_SPEED), + /** Trade retrieval speed for compression ratio. */ + BEST_COMPRESSION(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + + private final Lucene87StoredFieldsFormat.Mode storedMode; + private final Lucene80DocValuesFormat.Mode dvMode; + + private Mode(Lucene87StoredFieldsFormat.Mode storedMode, Lucene80DocValuesFormat.Mode dvMode) { + this.storedMode = Objects.requireNonNull(storedMode); + this.dvMode = Objects.requireNonNull(dvMode); + } + } + private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat(); private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat(); private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat(); @@ -82,7 +100,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { * Instantiates a new codec. */ public Lucene87Codec() { - this(Lucene87StoredFieldsFormat.Mode.BEST_SPEED); + this(Mode.BEST_SPEED); } /** @@ -91,10 +109,11 @@ public Lucene87Codec() { * @param mode stored fields compression mode to use for newly * flushed/merged segments. */ - public Lucene87Codec(Lucene87StoredFieldsFormat.Mode mode) { + public Lucene87Codec(Mode mode) { super("Lucene87"); - this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Objects.requireNonNull(mode)); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Objects.requireNonNull(mode).storedMode); this.defaultFormat = new Lucene84PostingsFormat(); + this.defaultDVFormat = new Lucene80DocValuesFormat(mode.dvMode); } @Override @@ -168,7 +187,7 @@ public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); + private final DocValuesFormat defaultDVFormat; private final NormsFormat normsFormat = new Lucene80NormsFormat(); diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java similarity index 98% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80DocValuesFormat.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java index 626f063f6b7d..4954fb71b2dd 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80DocValuesFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java @@ -28,7 +28,6 @@ import java.util.function.Supplier; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.asserting.AssertingCodec; @@ -71,15 +70,8 @@ /** * Tests Lucene80DocValuesFormat - * Copied directly from the lucene70 package for separation of codec-code */ -public class TestLucene80DocValuesFormat extends BaseCompressingDocValuesFormatTestCase { - private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene80DocValuesFormat()); - - @Override - protected Codec getCodec() { - return codec; - } +public abstract class BaseLucene80DocValuesFormatTestCase extends BaseCompressingDocValuesFormatTestCase { // TODO: these big methods can easily blow up some of the other ram-hungry codecs... // for now just keep them here, as we want to test this for this format. @@ -287,7 +279,7 @@ private void doTestTermsEnumRandom(int numDocs, Supplier valuesProducer) conf.setMergeScheduler(new SerialMergeScheduler()); // set to duel against a codec which has ordinals: final PostingsFormat pf = TestUtil.getPostingsFormatWithOrds(random()); - final DocValuesFormat dv = new Lucene80DocValuesFormat(); + final DocValuesFormat dv = getCodec().docValuesFormat(); conf.setCodec(new AssertingCodec() { @Override public PostingsFormat getPostingsFormatForField(String field) { diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java new file mode 100644 index 000000000000..2b1fa5895a47 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene80; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.util.TestUtil; + +/** + * Tests Lucene80DocValuesFormat + */ +public class TestBestCompressionLucene80DocValuesFormat extends BaseLucene80DocValuesFormatTestCase { + private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION)); + + @Override + protected Codec getCodec() { + return codec; + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java new file mode 100644 index 000000000000..7d3410591e92 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene80; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.util.TestUtil; + +/** + * Tests Lucene80DocValuesFormat + */ +public class TestBestSpeedLucene80DocValuesFormat extends BaseLucene80DocValuesFormatTestCase { + private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED)); + + @Override + protected Codec getCodec() { + return codec; + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java index b6dc5a5a3afd..861edc786c58 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java @@ -18,7 +18,7 @@ import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode; +import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode; import org.apache.lucene.document.Document; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.BaseStoredFieldsFormatTestCase; diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java index 8888d190472d..070b54409bb0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java @@ -40,6 +40,7 @@ import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval; import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat; import org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings; +import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.lucene86.Lucene86PointsReader; import org.apache.lucene.codecs.lucene86.Lucene86PointsWriter; import org.apache.lucene.codecs.memory.DirectDocValuesFormat; @@ -209,9 +210,9 @@ public RandomCodec(Random random, Set avoidCodecs) { new AssertingPostingsFormat()); addDocValues(avoidCodecs, - TestUtil.getDefaultDocValuesFormat(), new DirectDocValuesFormat(), // maybe not a great idea... TestUtil.getDefaultDocValuesFormat(), + new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION), new AssertingDocValuesFormat()); Collections.shuffle(formats, random); diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java index 81cb328aada2..82fbff2cc041 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java @@ -33,7 +33,6 @@ import org.apache.lucene.codecs.asserting.AssertingPostingsFormat; import org.apache.lucene.codecs.cheapbastard.CheapBastardCodec; import org.apache.lucene.codecs.compressing.CompressingCodec; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat; import org.apache.lucene.codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat; import org.apache.lucene.codecs.simpletext.SimpleTextCodec; @@ -188,7 +187,7 @@ public String toString() { } else if ("Compressing".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) { codec = CompressingCodec.randomInstance(random); } else if ("Lucene87".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene87"))) { - codec = new Lucene87Codec(RandomPicks.randomFrom(random, Lucene87StoredFieldsFormat.Mode.values()) + codec = new Lucene87Codec(RandomPicks.randomFrom(random, Lucene87Codec.Mode.values()) ); } else if (!"random".equals(TEST_CODEC)) { codec = Codec.forName(TEST_CODEC); diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java index edad01ebae37..d6124aa9f27f 100644 --- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java @@ -23,8 +23,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode; import org.apache.lucene.codecs.lucene87.Lucene87Codec; +import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.NamedList; diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index dfb4aab04055..c0216150f052 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -20,8 +20,8 @@ import java.util.Map; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode; import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.index.SegmentInfo;