Merge branch 'main' into two-phase-vector

gradle/generation/forUtil.gradle

@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
     description "Regenerate gen_ForUtil.py"
     group "generation"
 
-    def genDir = file("src/java/org/apache/lucene/codecs/lucene912")
+    def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
     def genScript = file("${genDir}/gen_ForUtil.py")
     def genOutput = file("${genDir}/ForUtil.java")
 
@@ -48,7 +48,7 @@ configure(project(":lucene:core")) {
     description "Regenerate gen_ForDeltaUtil.py"
     group "generation"
 
-    def genDir = file("src/java/org/apache/lucene/codecs/lucene912")
+    def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
     def genScript = file("${genDir}/gen_ForDeltaUtil.py")
     def genOutput = file("${genDir}/ForDeltaUtil.java")
 
@@ -68,6 +68,7 @@ configure(project(":lucene:core")) {
           andThenTasks: ["spotlessJava", "spotlessJavaApply"],
           mustRunBefore: [ "compileJava" ]
   ])
+
 }
 
 configure(project(":lucene:backward-codecs")) {
@@ -146,5 +147,55 @@ configure(project(":lucene:backward-codecs")) {
           andThenTasks: ["spotlessJava", "spotlessJavaApply"],
           mustRunBefore: [ "compileJava" ]
   ])
+
+  task generateForUtil912Internal() {
+    description "Regenerate gen_ForUtil.py"
+    group "generation"
+
+    def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene912")
+    def genScript = file("${genDir}/gen_ForUtil.py")
+    def genOutput = file("${genDir}/ForUtil.java")
+
+    inputs.file genScript
+    outputs.file genOutput
+
+    doLast {
+      quietExec {
+        workingDir genDir
+        executable project.externalTool("python3")
+        args = [ '-B', genScript ]
+      }
+    }
+  }
+
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil912Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
+
+  task generateForDeltaUtil912Internal() {
+    description "Regenerate gen_ForDeltaUtil.py"
+    group "generation"
+
+    def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene912")
+    def genScript = file("${genDir}/gen_ForDeltaUtil.py")
+    def genOutput = file("${genDir}/ForDeltaUtil.java")
+
+    inputs.file genScript
+    outputs.file genOutput
+
+    doLast {
+      quietExec {
+        workingDir genDir
+        executable project.externalTool("python3")
+        args = [ '-B', genScript ]
+      }
+    }
+  }
+
+  regenerate.dependsOn wrapWithPersistentChecksums(generateForDeltaUtil912Internal, [
+          andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+          mustRunBefore: [ "compileJava" ]
+  ])
 }
 

gradle/testing/defaults-tests.gradle

@@ -128,8 +128,14 @@ allprojects {
       jvmArgs '--add-modules', 'jdk.management'
 
       // Enable the vector incubator module on supported Java versions:
-      if (rootProject.vectorIncubatorJavaVersions.contains(rootProject.runtimeJavaVersion)) {
+      def prop = propertyOrDefault("org.apache.lucene.vectorization.upperJavaFeatureVersion", "1") as String
+      def v = JavaVersion.toVersion(Integer.parseInt(prop)).majorVersion
+      if (rootProject.vectorIncubatorJavaVersions.contains(rootProject.runtimeJavaVersion) ||
+          rootProject.runtimeJavaVersion.majorVersion <= v) {
         jvmArgs '--add-modules', 'jdk.incubator.vector'
+        if (rootProject.runtimeJavaVersion.majorVersion <= v) {
+          systemProperty 'org.apache.lucene.vectorization.upperJavaFeatureVersion', v
+        }
       }
 
       jvmArgs '--enable-native-access=' + (project.path in [

lucene/CHANGES.txt

@@ -47,7 +47,11 @@ New Features
 
 Improvements
 ---------------------
-(No changes)
+
+* GITHUB#13986: Allow easier configuration of the Panama Vectorization provider with
+  newer Java versions. Set the `org.apache.lucene.vectorization.upperJavaFeatureVersion`
+  system property to increase the set of Java versions that Panama Vectorization will
+  provide optimized implementations for. (Chris Hegarty)
 
 Optimizations
 ---------------------
@@ -84,6 +88,14 @@ Optimizations
 
 * GITHUB#13763: Replace Map<String,Object> with IntObjectHashMap for KnnVectorsReader (Pan Guixin)
 
+* GITHUB#13968: Switch postings from storing doc IDs in a long[] to an int[].
+  Lucene 8.4 had moved to a long[] to help speed up block decoding by using
+  longs that would pack two integers. We are now moving back to integers to be
+  able to take advantage of 2x more lanes with the vector API. (Adrien Grand)
+
+* GITHUB#13994: Speed up top-k retrieval of filtered conjunctions.
+  (Adrien Grand)
+
 Bug Fixes
 ---------------------
 * GITHUB#13832: Fixed an issue where the DefaultPassageFormatter.format method did not format passages as intended
@@ -92,6 +104,10 @@ Bug Fixes
 * GITHUB#12686: Added support for highlighting IndexOrDocValuesQuery. (Prudhvi Godithi)
 * GITHUB#13927: Fix StoredFieldsConsumer finish. (linfn)
 * GITHUB#13944: Ensure deterministic order of clauses for `DisjunctionMaxQuery#toString`. (Laurent Jakubina)
+* GITHUB#13841: Improve Tessellatorlogic when two holes share the same vertex with the polygon which was failing
+  in valid polygons. (Ignacio Vera)
+* GITHUB#13990: Added filter to the toString() method of Knn[Float|Byte]VectorQuery
+  and DiversifyingChildren[Float|Byte]KnnVectorQuery. (Viswanath Kuchibhotla)
 
 Build
 ---------------------
@@ -100,7 +116,7 @@ Build
 
 Other
 ---------------------
-(No changes)
+* GITHUB#13982: Remove duplicate test code. (Lu Xugang)
 
 ======================== Lucene 10.0.1 =======================
 
@@ -328,6 +344,8 @@ Bug Fixes
 * GITHUB#12878: Fix the declared Exceptions of Expression#evaluate() to match those
   of DoubleValues#doubleValue(). (Uwe Schindler)
 
+* GITHUB#13498: Avoid performance regression by constructing lazily the PointTree in NumericComparator, (Ignacio Vera)
+
 Changes in Runtime Behavior
 ---------------------
 

lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil912.json

@@ -0,0 +1,4 @@
+{
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/ForDeltaUtil.java": "b81961f0b277b1458ca259e0d23ccc4eeeb47fe7",
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/gen_ForDeltaUtil.py": "3191d7591309b7876c5c709fb9375af5b87c2ef8"
+}

lucene/backward-codecs/src/generated/checksums/generateForUtil912.json

@@ -0,0 +1,4 @@
+{
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/ForUtil.java": "e6db3c665dfebca8b93eb6b4651d2eb3af637b02",
+    "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/gen_ForUtil.py": "993ecc9cf7ea821963384070669695257b16e040"
+}

lucene/backward-codecs/src/java/module-info.java

@@ -37,6 +37,7 @@
   exports org.apache.lucene.backward_codecs.lucene95;
   exports org.apache.lucene.backward_codecs.lucene99;
   exports org.apache.lucene.backward_codecs.lucene912;
+  exports org.apache.lucene.backward_codecs.lucene100;
   exports org.apache.lucene.backward_codecs.packed;
   exports org.apache.lucene.backward_codecs.store;
 
@@ -46,7 +47,8 @@
       org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,
       org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
       org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
-      org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
+      org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
+      org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
   provides org.apache.lucene.codecs.KnnVectorsFormat with
       org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
       org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
@@ -64,5 +66,6 @@
       org.apache.lucene.backward_codecs.lucene94.Lucene94Codec,
       org.apache.lucene.backward_codecs.lucene95.Lucene95Codec,
       org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,
-      org.apache.lucene.backward_codecs.lucene912.Lucene912Codec;
+      org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,
+      org.apache.lucene.backward_codecs.lucene100.Lucene100Codec;
 }

lucene/core/src/java/org/apache/lucene/codecs/lucene100/Lucene100Codec.java → lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene100/Lucene100Codec.java

@@ -14,9 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.lucene.codecs.lucene100;
+package org.apache.lucene.backward_codecs.lucene100;
 
 import java.util.Objects;
+import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.CompoundFormat;
 import org.apache.lucene.codecs.DocValuesFormat;
@@ -37,7 +38,6 @@
 import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat;
@@ -50,7 +50,7 @@
  *
  * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
  *
- * @see org.apache.lucene.codecs.lucene100 package documentation for file format details.
+ * @see org.apache.lucene.backward_codecs.lucene100 package documentation for file format details.
  * @lucene.experimental
  */
 public class Lucene100Codec extends Codec {

lucene/core/src/java/org/apache/lucene/codecs/lucene912/package-info.java → lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene100/package-info.java

@@ -15,5 +15,5 @@
  * limitations under the License.
  */
 
-/** Lucene 9.12 file format. */
-package org.apache.lucene.codecs.lucene912;
+/** Lucene 10.0 file format. */
+package org.apache.lucene.backward_codecs.lucene100;