Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to lucene snapshot lucene-9.3.0-snapshot-b8d1fcfd0ec #88706

Merged
merged 3 commits into from
Jul 22, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build-tools-internal/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 8.4.0
lucene = 9.3.0-snapshot-2d05f5c623e
lucene = 9.3.0-snapshot-b8d1fcfd0ec

bundled_jdk_vendor = openjdk
bundled_jdk = 18.0.2+9@f6ad4b4450fd4d298113270ec84f30ee
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f9d91e4de3468b4c513a82a3d20d9d19137c4311

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b146dc1d898b3f638328a4d6a64f68cfede251ec

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f87c4435a856c612a5799fa89397364a7b2d6f7e

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
110e8b2e5bced4b8f482ac58a2cf9cd64591b028

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
501aa4f0028424a994b06627f30ffb36150ffbe2

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a4a84f37391ab5da0697ba6344555b633aa4bacd

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
012f177949d83aa7bdf26c309f5569f67d1c65b5

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
792f50d6cd8b75c277c514f2f6e9914572942dfe

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b1488267195c87749dcc42de6b2f665d24ff8d9e

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c78968b087eaf2a95ed3b67540efc32455bab84d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
978ee14dad7edab6384d04655ce1db219547b6d8

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7a154a194ea505d27b538270ee2db2b5a4a38371

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3ad36d2a32c1dda37040cdfed9dcdf294b8f3b7c

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e41aa9fe38033e61da13fe420aa6e9400f467dd8

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
71cd063e306af5acf1cef0492eebbbf000e6a6ce

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
adc913180fac1b221f57288661f069cb7a240127

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3618de63e62d734ab1892ff446ae4f5ef866bee6

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
07e0de548fc392428545db40192280b4f83daf4f

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ab308291a7dd5ec9988a229dc8e7c27fc2bb5409

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8b5804be2c87d995c5255ff1ad739052fc243661

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a55d8a68cccaaf4af5a973c4332519d3eb477068

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0a6005b6b9b09b1da1c3c74558693824f429e55d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d4c5418c469be74cc5df3427ac07386598c18882
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
import java.util.Map;

public class Lucene {
public static final String LATEST_CODEC = "Lucene93";
public static final String LATEST_CODEC = "Lucene92";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
package org.elasticsearch.index.codec;

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.mapper.MapperService;

Expand All @@ -34,11 +34,11 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService) {
final var codecs = new HashMap<String, Codec>();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene93Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene93Codec(Lucene93Codec.Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Lucene92Codec.Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene93Codec.Mode.BEST_SPEED, mapperService));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene93Codec.Mode.BEST_COMPRESSION, mapperService));
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_SPEED, mapperService));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_COMPRESSION, mapperService));
}
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
for (String codec : Codec.availableCodecs()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperService;
Expand All @@ -27,7 +27,7 @@
* per index in real time via the mapping API. If no specific postings format or vector format is
* configured for a specific field the default postings or vector format is used.
*/
public class PerFieldMapperCodec extends Lucene93Codec {
public class PerFieldMapperCodec extends Lucene92Codec {
private final MapperService mapperService;

private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
package org.elasticsearch.index.mapper.vectors;

import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnVectorField;
Expand Down Expand Up @@ -523,7 +523,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField() {
return null; // use default format
} else {
HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
return new Lucene93HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
return new Lucene92HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
Expand Down Expand Up @@ -297,7 +297,7 @@ public void testTriangle() throws Exception {
public void testCompletionField() throws Exception {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(false)
.setCodec(new Lucene93Codec(Lucene93Codec.Mode.BEST_SPEED) {
.setCodec(new Lucene92Codec(Lucene92Codec.Mode.BEST_SPEED) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if (field.startsWith("suggest_")) {
Expand Down Expand Up @@ -384,25 +384,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
enum CodecMode {
BEST_SPEED {
@Override
Lucene93Codec.Mode mode() {
return Lucene93Codec.Mode.BEST_SPEED;
Lucene92Codec.Mode mode() {
return Lucene92Codec.Mode.BEST_SPEED;
}
},

BEST_COMPRESSION {
@Override
Lucene93Codec.Mode mode() {
return Lucene93Codec.Mode.BEST_COMPRESSION;
Lucene92Codec.Mode mode() {
return Lucene92Codec.Mode.BEST_COMPRESSION;
}
};

abstract Lucene93Codec.Mode mode();
abstract Lucene92Codec.Mode mode();
}

static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(randomBoolean())
.setCodec(new Lucene93Codec(codecMode.mode()));
.setCodec(new Lucene92Codec(codecMode.mode()));
try (IndexWriter writer = new IndexWriter(directory, config)) {
for (int i = 0; i < numDocs; i++) {
final Document doc = new Document();
Expand Down Expand Up @@ -580,7 +580,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire
try (DirectoryReader reader = DirectoryReader.open(source)) {
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setUseCompoundFile(randomBoolean())
.setCodec(new Lucene93Codec(mode.mode()) {
.setCodec(new Lucene92Codec(mode.mode()) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return new Lucene90PostingsFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
Expand Down Expand Up @@ -41,21 +41,21 @@ public class CodecTests extends ESTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene93Codec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
}

public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertStoredFieldsCompressionEquals(Lucene93Codec.Mode.BEST_SPEED, codec);
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
}

public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertStoredFieldsCompressionEquals(Lucene93Codec.Mode.BEST_COMPRESSION, codec);
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
}

// write some docs with it, inspect .si to see this was the used compression
private void assertStoredFieldsCompressionEquals(Lucene93Codec.Mode expected, Codec actual) throws Exception {
private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(null);
iwc.setCodec(actual);
Expand All @@ -67,7 +67,7 @@ private void assertStoredFieldsCompressionEquals(Lucene93Codec.Mode expected, Co
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
assertNotNull(v);
assertEquals(expected, Lucene93Codec.Mode.valueOf(v));
assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
ir.close();
dir.close();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
package org.elasticsearch.index.engine;

import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
Expand Down Expand Up @@ -45,7 +45,7 @@ public void testExceptionsAreNotCached() {
public void testCompletionStatsCache() throws IOException, InterruptedException {
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
indexWriterConfig.setCodec(new Lucene93Codec() {
indexWriterConfig.setCodec(new Lucene92Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormat; // all fields are suggest fields
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.IndexableField;
Expand All @@ -39,8 +39,8 @@
import java.util.Arrays;
import java.util.List;

import static org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
Expand Down Expand Up @@ -454,8 +454,8 @@ public void testKnnVectorsFormat() throws IOException {
Codec codec = codecService.codec("default");
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
assertThat(knnVectorsFormat, instanceOf(Lucene93HnswVectorsFormat.class));
String expectedString = "Lucene93HnswVectorsFormat(name=Lucene93HnswVectorsFormat, maxConn="
assertThat(knnVectorsFormat, instanceOf(Lucene92HnswVectorsFormat.class));
String expectedString = "lucene92HnswVectorsFormat(name = lucene92HnswVectorsFormat, maxConn = "
+ m
+ ", beamWidth="
+ efConstruction
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
501aa4f0028424a994b06627f30ffb36150ffbe2