Update to lucene snapshot e7c625430ed (#57981)

Includes LUCENE-9148 and LUCENE-9398, which splits the BKD metadata, index and data into separate files and keeps the index off-heap.

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.9.0
-lucene            = 8.6.0-snapshot-9d6c738ffce
+lucene            = 8.6.0-snapshot-e7c625430ed
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 14.0.1+7

modules/lang-expression/licenses/lucene-expressions-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+c9dbc427cb1998b500ff07f8af4c13ebd72fa4ba

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+23ef92764508757a04703fa129930774862bce31

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+04432005742de8e9ad8e1248e0ac4ef41319e687

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+c994b8702089de33cbba337d390553707d2e4225

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+65d46f9ef354fd88f7b971d293ed2b90e4e5373b

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+d4903699eb1c5932f8c120a6e71321917a2ea1aa

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+fd67d4aa9730a8a7a7bcba2dc5c7f53838ffd1d8

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+561cb8c4efa2e703246526cd5ee2beac7e28ead8

server/licenses/lucene-analyzers-common-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+188c577102f49cdb097e2487ff6958dccff1a2de

server/licenses/lucene-backward-codecs-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+0cada355cf6f8b210c13602ad39378cbc568a6f9

server/licenses/lucene-core-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+180065e697f361ead5e6ee831582cc2cf37bd5b9

server/licenses/lucene-grouping-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+e5a26420f917e6d5f8bc4888594fc9f156a5f7ba

server/licenses/lucene-highlighter-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+06d0935908b272667a2f5aa3aa7da7f1832d27a8

server/licenses/lucene-join-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+45c68a4d29befe8d1d9aef7a92f933598cba85e5

server/licenses/lucene-memory-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+ac72a12e5725ce030d63bad44bd6a169594ac651

server/licenses/lucene-misc-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+e196e5ef80672e894d61fe0f0ae2568dfb83f63a

server/licenses/lucene-queries-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+15bd0e6b44e0192db9a8c5a5e0f8d392b4610884

server/licenses/lucene-queryparser-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+627e685ec2d1025429abc02441e22f466443be5a

server/licenses/lucene-sandbox-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+1c4a1e4281865c329e1fec53a0e6d5e53d25155f

server/licenses/lucene-spatial-extras-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+e0c9b19c5e9d99d924b1cf9e6b4329b94920c9b8

server/licenses/lucene-spatial3d-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+d780c5bf1374627afcca67d487319507211a5620

server/licenses/lucene-suggest-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+0906b0aca0283153ab00b448a57e755847a659a4

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -103,7 +103,7 @@
 import java.util.Map;
 
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene84";
+    public static final String LATEST_CODEC = "Lucene86";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

server/src/main/java/org/elasticsearch/index/codec/CodecService.java

@@ -22,7 +22,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.codecs.lucene84.Lucene84Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86Codec;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.index.mapper.MapperService;
@@ -47,8 +47,8 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene84Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene84Codec(Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene86Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene86Codec(Mode.BEST_COMPRESSION));
         } else {
             codecs.put(DEFAULT_CODEC,
                     new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));

server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -23,7 +23,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene84.Lucene84Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86Codec;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -37,7 +37,7 @@
  * per index in real time via the mapping API. If no specific postings format is
  * configured for a specific field the default postings format is used.
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene84Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene86Codec {
     private final Logger logger;
     private final MapperService mapperService;
 

server/src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -19,16 +19,11 @@
 
 package org.elasticsearch.index.codec;
 
-import static org.hamcrest.Matchers.instanceOf;
-
-import java.io.IOException;
-import java.util.Collections;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.codecs.lucene84.Lucene84Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -47,14 +42,19 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.instanceOf;
+
 @SuppressCodecs("*") // we test against default codec so never get a random one here!
 public class CodecTests extends ESTestCase {
 
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Lucene84Codec.class));
-        assertThat(codecService.codec("Lucene84"), instanceOf(Lucene84Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Lucene86Codec.class));
+        assertThat(codecService.codec("Lucene86"), instanceOf(Lucene86Codec.class));
     }
 
     public void testDefault() throws Exception {

server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java

@@ -19,7 +19,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene84.Lucene84Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -57,7 +57,7 @@ public void testExceptionsAreNotCached() {
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         final PostingsFormat postingsFormat = new Completion84PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene84Codec() {
+        indexWriterConfig.setCodec(new Lucene86Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields

server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java

@@ -481,7 +481,7 @@ public void testIndexSortingEarlyTermination() throws Exception {
         context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         context.setSize(1);
         context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
-        context.sort(new SortAndFormats(sort, new DocValueFormat[] {DocValueFormat.RAW}));
+        context.sort(new SortAndFormats(sort, new DocValueFormat[]{DocValueFormat.RAW}));
 
 
         QueryPhase.executeInternal(context);
@@ -635,7 +635,7 @@ public void testDisableTopScoreCollection() throws Exception {
 
 
         context.sort(new SortAndFormats(new Sort(new SortField("other", SortField.Type.INT)),
-            new DocValueFormat[] { DocValueFormat.RAW }));
+            new DocValueFormat[]{DocValueFormat.RAW}));
         topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context, false);
         assertEquals(topDocsContext.create(null).scoreMode(), org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES);
         QueryPhase.executeInternal(context);
@@ -757,14 +757,16 @@ public void testIndexHasDuplicateData() throws IOException {
                 LongPoint.encodeDimension(value, longBytes, 0);
                 w.add(longBytes, docId);
             }
-            long indexFP;
-            try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
-                indexFP = w.finish(out);
+            try (IndexOutput metaout = dir.createOutput("bkdmeta", IOContext.DEFAULT);
+                 IndexOutput indexout = dir.createOutput("bkdindex", IOContext.DEFAULT);
+                 IndexOutput dataout = dir.createOutput("bkddata", IOContext.DEFAULT)) {
+                w.finish(metaout, indexout, dataout).run();
             }
-            try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
-                in.seek(indexFP);
-                BKDReader r = new BKDReader(in);
-                assertTrue(pointsHaveDuplicateData(r, r.getDocCount()/2));
+            try (IndexInput metain = dir.openInput("bkdmeta", IOContext.DEFAULT);
+                 IndexInput indexin = dir.openInput("bkdindex", IOContext.DEFAULT);
+                 IndexInput datain = dir.openInput("bkddata", IOContext.DEFAULT)) {
+                BKDReader r = new BKDReader(metain, indexin, datain);
+                assertTrue(pointsHaveDuplicateData(r, r.getDocCount() / 2));
             }
         }
     }
@@ -785,12 +787,14 @@ public void testIndexHasNoDuplicateData() throws IOException {
             }
             long indexFP;
             try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
-                indexFP = w.finish(out);
+                Runnable finalizer = w.finish(out, out, out);
+                indexFP = out.getFilePointer();
+                finalizer.run();;
             }
             try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
                 in.seek(indexFP);
-                BKDReader r = new BKDReader(in);
-                assertFalse(pointsHaveDuplicateData(r, r.getDocCount()/2));
+                BKDReader r = new BKDReader(in, in, in);
+                assertFalse(pointsHaveDuplicateData(r, r.getDocCount() / 2));
             }
         }
     }
@@ -920,7 +924,7 @@ public void testCancellationDuringPreprocess() throws IOException {
 
             try (IndexReader reader = DirectoryReader.open(dir)) {
                 TestSearchContext context = new TestSearchContextWithRewriteAndCancellation(
-                        null, indexShard, newContextSearcher(reader));
+                    null, indexShard, newContextSearcher(reader));
                 PrefixQuery prefixQuery = new PrefixQuery(new Term("foo", "a"));
                 prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
                 context.parsedQuery(new ParsedQuery(prefixQuery));
@@ -979,15 +983,15 @@ private static ContextIndexSearcher newOptimizedContextSearcher(IndexReader read
 
             @Override
             public void search(List<LeafReaderContext> leaves, Weight weight, CollectorManager manager,
-                    QuerySearchResult result, DocValueFormat[] formats, TotalHits totalHits) throws IOException {
+                               QuerySearchResult result, DocValueFormat[] formats, TotalHits totalHits) throws IOException {
                 final Query query = weight.getQuery();
                 assertTrue(query instanceof BooleanQuery);
                 List<BooleanClause> clauses = ((BooleanQuery) query).clauses();
                 assertTrue(clauses.size() == 2);
                 assertTrue(clauses.get(0).getOccur() == Occur.FILTER);
                 assertTrue(clauses.get(1).getOccur() == Occur.SHOULD);
                 if (queryType == 0) {
-                    assertTrue (clauses.get(1).getQuery().getClass() ==
+                    assertTrue(clauses.get(1).getQuery().getClass() ==
                         LongPoint.newDistanceFeatureQuery("random_field", 1, 1, 1).getClass()
                     );
                 }
@@ -997,7 +1001,7 @@ public void search(List<LeafReaderContext> leaves, Weight weight, CollectorManag
 
             @Override
             public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) {
-                assert(false);  // should not be there, expected to search with CollectorManager
+                assert (false);  // should not be there, expected to search with CollectorManager
             }
         };
     }
@@ -1019,7 +1023,7 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept
                 @Override
                 public void collect(int doc) throws IOException {
                     assert collected <= size : "should not collect more than " + size + " doc per segment, got " + collected;
-                    ++ collected;
+                    ++collected;
                     super.collect(doc);
                 }
             };

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.6.0-snapshot-e7c625430ed.jar.sha1

@@ -0,0 +1 @@
+180065e697f361ead5e6ee831582cc2cf37bd5b9