Upgrade to lucene 8.0.0-snapshot-ff9509a8df (#39350)

Contains the following: * LUCENE-8635: Move terms dictionary off-heap for non-primary-key fields in `MMapDirectory` * LUCENE-8292: `TermsEnum` is fully abstract * LUCENE-8679: Return WITHIN in `EdgeTree#relateTriangle` only when polygon and triangle share one edge * LUCENE-8676: Nori tokenizer deals correctly with large buffers * LUCENE-8697: `GraphTokenStreamFiniteStrings` better handles side paths with gaps * LUCENE-8664: Add `equals` and `hashCode` to `TotalHits` * LUCENE-8660: `TopDocsCollector` returns accurate hit counts if the total equals the threshold * LUCENE-8654: `Polygon2D#relateTriangle` fix for when the polygon is inside the triangle * LUCENE-8645: `Intervals#fixField` can merge intervals from different fields * LUCENE-8585: Create jump-tables for DocValues at index time
elastic · Feb 27, 2019 · 68c57ad · 68c57ad
1 parent 66c0906
commit 68c57ad
Show file tree

Hide file tree

Showing 54 changed files with 70 additions and 29 deletions.
diff --git a/build.gradle b/build.gradle
@@ -160,8 +160,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/39350" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.0.0-snapshot-83f9835
+lucene            = 8.0.0-snapshot-ff9509a8df
 
 # optional dependencies
 spatial4j         = 0.7

diff --git a/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-83f9835.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-ff9509a8df.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+6b487ff516c053fae3fa270cd51964534f46b628
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-83f9835.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-ff9509a8df.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+116fed6827a68b78b46a1eadfda1e27936b171a1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-83f9835.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/...s/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-ff9509a8df.jar.sha1 b/...s/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+c39659fa1684742370dfabf37641e41394ac4799
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-83f9835.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-ff9509a8df.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+51e35640ceac2e4801b9be1263c7bea8637c7ad7
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-83f9835.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/...s/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-ff9509a8df.jar.sha1 b/...s/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+bd8f2fe37a3e2f1c4d052071ceba618e6d16f119
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-83f9835.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/...ins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-ff9509a8df.jar.sha1 b/...ins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+022de0054e9597d16f366bb83d6404d2006dcecf
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-83f9835.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/...ins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-ff9509a8df.jar.sha1 b/...ins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+6cff98d0fdbb6d2d65ce9051005954f5b565986c
diff --git a/...s/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-83f9835.jar.sha1 b/...s/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/...nalysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-ff9509a8df.jar.sha1 b/...nalysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+438537046ef60d75f17358bf09bb339aa84191e5
diff --git a/server/licenses/lucene-analyzers-common-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-analyzers-common-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-analyzers-common-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-analyzers-common-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+d96364f1b6a041f797e3d5f55b45284d8ab5cf95
diff --git a/server/licenses/lucene-backward-codecs-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-backward-codecs-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-backward-codecs-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+a39f5683e6e6ce17a191dd5fcbf90fcc43d25109
diff --git a/server/licenses/lucene-core-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-core-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-core-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-core-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+283cbab4d58f18fd4eed169448bd0114e5d7c971
diff --git a/server/licenses/lucene-grouping-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-grouping-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-grouping-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-grouping-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+e63de6e4690f844b571edef03e4029a63bf2458b
diff --git a/server/licenses/lucene-highlighter-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-highlighter-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-highlighter-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-highlighter-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+097cfc9ed8575c845c32f80745af11c04aaa6541
diff --git a/server/licenses/lucene-join-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-join-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-join-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-join-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+a1b20c5408c9a1d2fec4604dc66b3d54bf91f84c
diff --git a/server/licenses/lucene-memory-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-memory-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-memory-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-memory-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+14bf9224dd2cd13d81ff90a43af713bda9c5b699
diff --git a/server/licenses/lucene-misc-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-misc-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-misc-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-misc-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+79be1830102a764d79da6c81d40f5f6bf85ad032
diff --git a/server/licenses/lucene-queries-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-queries-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-queries-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-queries-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+3f7281351f03ac34c8a950e667e826053b82f31e
diff --git a/server/licenses/lucene-queryparser-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-queryparser-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-queryparser-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-queryparser-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+039229d20cb500de435b458998aa35370c616f69
diff --git a/server/licenses/lucene-sandbox-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-sandbox-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-sandbox-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-sandbox-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+00cee2f9ff5fcaa3ac3838f2d8b0c8b59c55e4a6
diff --git a/server/licenses/lucene-spatial-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-spatial-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-spatial-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-spatial-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+4246d1cc124eb7b2d18ccee1db96b5999ae45b95
diff --git a/server/licenses/lucene-spatial-extras-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-spatial-extras-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-spatial-extras-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+caa0c22ce9a985f4a7a76c315ac8841d37083c2b
diff --git a/server/licenses/lucene-spatial3d-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-spatial3d-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-spatial3d-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+7d01e71c2e2e9fd5a2ad1f397bf75de12b860fab
diff --git a/server/licenses/lucene-suggest-8.0.0-snapshot-83f9835.jar.sha1 b/server/licenses/lucene-suggest-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/server/licenses/lucene-suggest-8.0.0-snapshot-ff9509a8df.jar.sha1 b/server/licenses/lucene-suggest-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+fd72490e235dffbf455886d4a4339dd29f4ccd5b
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.ObjectLongHashMap;
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import org.apache.lucene.index.BaseTermsEnum;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.PostingsEnum;
@@ -229,7 +230,7 @@ public TermsEnum iterator() throws IOException {
             // reset before asking for an iterator
             reset();
             // convert bytes ref for the terms to actual data
-            return new TermsEnum() {
+            return new BaseTermsEnum() {
                 int currentTerm = 0;
                 int freq = 0;
                 int docFreq = -1;

diff --git a/server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java b/server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -32,6 +33,7 @@
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -126,6 +128,11 @@ public BytesRef term() throws IOException {
         return current;
     }
 
+    @Override
+    public AttributeSource attributes() {
+        throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
+    }
+
     @Override
     public boolean seekExact(BytesRef text) throws IOException {
         int docFreq = 0;
@@ -195,6 +202,16 @@ public void seekExact(long ord) throws IOException {
         throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
     }
 
+    @Override
+    public void seekExact(BytesRef term, TermState state) throws IOException {
+        throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
+    }
+
+    @Override
+    public TermState termState() throws IOException {
+        throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
+    }
+
     @Override
     public SeekStatus seekCeil(BytesRef text) throws IOException {
         throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);

diff --git a/...ain/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/...ain/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
@@ -19,6 +19,7 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
@@ -441,6 +442,14 @@ public boolean seekExact(BytesRef term) throws IOException {
             return accept(term) && in.seekExact(term);
         }
 
+        @Override
+        public void seekExact(BytesRef term, TermState state) throws IOException {
+            if (accept(term) == false) {
+                throw new IllegalStateException("Tried to seek using a TermState from a different reader!");
+            }
+            in.seekExact(term, state);
+        }
+
         @Override
         public SeekStatus seekCeil(BytesRef term) throws IOException {
             SeekStatus status = in.seekCeil(term);

diff --git a/...ava/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/...ava/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
@@ -35,12 +36,12 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
@@ -55,6 +56,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.test.ESTestCase;
@@ -768,6 +770,18 @@ public void testFieldNames() throws Exception {
         assertTrue(termsEnum.seekExact(new BytesRef("fieldA")));
         assertFalse(termsEnum.seekExact(new BytesRef("fieldB")));
 
+        // seekExact with TermState
+        // first, collect TermState from underlying reader
+        LeafReader unwrappedReader = FilterDirectoryReader.unwrap(ir).leaves().get(0).reader();
+        Terms unwrappedTerms = unwrappedReader.terms(FieldNamesFieldMapper.NAME);
+        TermsEnum unwrappedTE = unwrappedTerms.iterator();
+        assertTrue(unwrappedTE.seekExact(new BytesRef("fieldB")));
+        TermState termState = unwrappedTE.termState();
+
+        // now try and seekExact with it
+        TermsEnum badEnum = terms.iterator();
+        expectThrows(IllegalStateException.class, () -> badEnum.seekExact(new BytesRef("fieldB"), termState));
+
         // seekCeil
         termsEnum = terms.iterator();
         assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("fieldA")));

diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-83f9835.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-83f9835.jar.sha1
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-ff9509a8df.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-ff9509a8df.jar.sha1
@@ -0,0 +1 @@
+283cbab4d58f18fd4eed169448bd0114e5d7c971