Merge branch 'main' into fst-lucene90

.github/actions/prepare-for-build/action.yml

@@ -6,7 +6,7 @@ name: Prepare Lucene build
 inputs:
    java-version:
      required: false
-     default: 17
+     default: 21
      description: "The default JDK version to set up."
 
    java-distribution:

.github/workflows/run-checks-all.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        java: [ '17' ]
+        java: [ '21' ]
 
     runs-on: ${{ matrix.os }}
 
@@ -52,7 +52,7 @@ jobs:
         # windows-latest: fairly slow to build and results in odd errors (see LUCENE-10167)
         # macos-latest: a tad slower than ubuntu and pretty much the same (?) so leaving out.
         os: [ ubuntu-latest ]
-        java: [ '17' ]
+        java: [ '21' ]
 
     runs-on: ${{ matrix.os }}
 

.github/workflows/run-nightly-smoketester.yml

@@ -0,0 +1,74 @@
+name: "Run nightly: buildAndPushRelease and smokeTestRelease.py"
+
+on:
+  # Allow manual dispatch.
+  workflow_dispatch:
+
+  # run nightly at 2:04am.
+  schedule:
+    - cron: '4 2 * * *'
+
+jobs:
+  smokeTestRelease:
+    name: "Smoke test release on jdk ${{ matrix.java-version }}, ${{ matrix.os }}"
+
+    # only run on schedule in the main Lucene repo (not in forks).
+    if: (github.event_name == 'schedule' && github.repository == 'apache/lucene') || (github.event_name != 'schedule')
+
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        java-version: [ '21', '22-ea' ]
+
+    runs-on: ${{ matrix.os }}
+
+    env:
+      LUCENE_RELEASE_DIR: /tmp/lucene-release-dir
+      TMP_DIR: /tmp/lucene-tmp-dir
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-package: jdk
+          check-latest: true
+          # This intentionally lists two versions, the last one is used as the system default (for running gradle).
+          java-version: |
+            ${{ matrix.java-version }}
+            21
+
+      - name: Echo diagnostic paths and locations
+        run: |
+          echo "All installed JDKs:"
+          set | grep "JAVA"
+
+          echo "Gradle's 'RUNTIME_JAVA_HOME' JDK:"
+          RUNTIME_JAVA_HOME_VAR=JAVA_HOME_`echo ${{ matrix.java-version }} | egrep --only "[0-9]+"`_X64
+          echo ${RUNTIME_JAVA_HOME_VAR} points at ${!RUNTIME_JAVA_HOME_VAR}
+
+          # This sets the environment variable and makes it available for subsequent job steps.
+          # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable
+          echo "RUNTIME_JAVA_HOME=${!RUNTIME_JAVA_HOME_VAR}" >> "$GITHUB_ENV"
+
+      - name: "Run buildAndPushRelease.py (--dev-mode)"
+        run: |
+          # Assemble an unsigned release, in dev mode, publish locally.
+          python3 ./dev-tools/scripts/buildAndPushRelease.py --dev-mode --push-local ${{ env.LUCENE_RELEASE_DIR }}
+
+      - name: "Run smokeTestRelease.py (runtime java: ${{ matrix.java-version }})"
+        run: |
+          python3 -u dev-tools/scripts/smokeTestRelease.py \
+            --not-signed \
+            --tmp-dir ${{ env.TMP_DIR }} \
+            file://`realpath ${{ env.LUCENE_RELEASE_DIR }}/lucene*` \
+            -Ptests.filter="@skipall"
+
+      - name: "Store smoke tester logs"
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: smoke-tester-logs-jdk-${{ matrix.java-version }}
+          path: |
+            ${{ env.TMP_DIR }}/**/*.log

README.md

@@ -39,7 +39,7 @@ comprehensive documentation, visit:
 
 ### Basic steps:
 
-1. Install OpenJDK 17 or 18.
+1. Install OpenJDK 21.
 2. Clone Lucene's git repository (or download the source distribution).
 3. Run gradle launcher script (`gradlew`).
 

build.gradle

@@ -22,7 +22,7 @@ plugins {
   id "base"
   id "com.palantir.consistent-versions" version "2.11.0"
   id "org.owasp.dependencycheck" version "7.2.0"
-  id 'de.thetaphi.forbiddenapis' version '3.6' apply false
+  id 'de.thetaphi.forbiddenapis' version '3.7' apply false
   id "de.undercouch.download" version "5.2.0" apply false
   id "net.ltgt.errorprone" version "3.1.0" apply false
   id 'com.diffplug.spotless' version "6.5.2" apply false
@@ -73,7 +73,7 @@ ext {
   }
 
   // Minimum Java version required to compile and run Lucene.
-  minJavaVersion = JavaVersion.VERSION_17
+  minJavaVersion = JavaVersion.VERSION_21
 
   // snapshot build marker used in scripts.
   snapshotBuild = version.contains("SNAPSHOT")

buildSrc/build.gradle

@@ -21,7 +21,7 @@ repositories {
 
 ext {
   // Minimum Java version required to compile buildSrc.
-  minJavaVersion = JavaVersion.VERSION_17
+  minJavaVersion = JavaVersion.VERSION_21
 }
 
 

buildSrc/scriptDepVersions.gradle

@@ -22,7 +22,7 @@
 ext {
   scriptDepVersions = [
       "apache-rat": "0.14",
-      "asm": "9.6",
+      "asm": "9.7",
       "commons-codec": "1.13",
       "ecj": "3.36.0",
       "flexmark": "0.61.24",

buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -55,8 +55,8 @@ public static void main(String[] args) {
 
   public static void checkVersion() {
     int major = Runtime.version().feature();
-    if (major < 17 || major > 21) {
-      throw new IllegalStateException("java version must be between 17 and 21, your version: " + major);
+    if (major != 21) {
+      throw new IllegalStateException("java version must be 21, your version: " + major);
     }
   }
 

dev-tools/missing-doclet/build.gradle

@@ -23,8 +23,8 @@ version = "1.0.0-SNAPSHOT"
 group = "org.apache.lucene.tools"
 description = 'Doclet-based javadoc validation'
 
-sourceCompatibility = JavaVersion.VERSION_11
-targetCompatibility = JavaVersion.VERSION_11
+sourceCompatibility = JavaVersion.VERSION_21
+targetCompatibility = JavaVersion.VERSION_21
 
 tasks.withType(JavaCompile) {
   options.compilerArgs += ["--release", targetCompatibility.toString()]

dev-tools/missing-doclet/src/main/java/org/apache/lucene/missingdoclet/MissingDoclet.java

@@ -237,6 +237,7 @@ private void check(Element element) {
       case CLASS:
       case INTERFACE:
       case ENUM:
+      case RECORD:
       case ANNOTATION_TYPE:
         if (level(element) >= CLASS) {
           checkComment(element);

dev-tools/scripts/releaseWizard.py

@@ -66,7 +66,7 @@
 from scriptutil import BranchType, Version, download, run
 
 # Lucene-to-Java version mapping
-java_versions = {6: 8, 7: 8, 8: 8, 9: 11, 10: 17}
+java_versions = {6: 8, 7: 8, 8: 8, 9: 11, 10: 21}
 editor = None
 
 # Edit this to add other global jinja2 variables or filters

dev-tools/scripts/smokeTestRelease.py

@@ -39,6 +39,8 @@
 from collections import namedtuple
 import scriptutil
 
+BASE_JAVA_VERSION = "21"
+
 # This tool expects to find /lucene off the base URL.  You
 # must have a working gpg, tar, unzip in your path.  This has been
 # tested on Linux and on Cygwin under Windows 7.
@@ -144,10 +146,10 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
       'Implementation-Vendor: The Apache Software Foundation',
       'Specification-Title: Lucene Search Engine:',
       'Implementation-Title: org.apache.lucene',
-      'X-Compile-Source-JDK: 17',
-      'X-Compile-Target-JDK: 17',
+      'X-Compile-Source-JDK: %s' % BASE_JAVA_VERSION,
+      'X-Compile-Target-JDK: %s' % BASE_JAVA_VERSION,
       'Specification-Version: %s' % version,
-      'X-Build-JDK: 17.',
+      'X-Build-JDK: %s.' % BASE_JAVA_VERSION,
       'Extension-Name: org.apache.lucene'):
       if type(verify) is not tuple:
         verify = (verify,)
@@ -156,9 +158,9 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
           break
       else:
         if len(verify) == 1:
-          raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF' % (desc, verify[0]))
+          raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify[0], s))
         else:
-          raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF' % (desc, verify))
+          raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify, s))
 
     if gitRevision != 'skip':
       # Make sure this matches the version and git revision we think we are releasing:
@@ -575,7 +577,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
   #       raise RuntimeError('lucene: file "%s" is missing from artifact %s' % (fileName, artifact))
   #     in_root_folder.remove(fileName)
 
-  expected_folders = ['analysis', 'analysis.tests', 'backward-codecs', 'benchmark', 'classification', 'codecs', 'core', 'core.tests',
+  expected_folders = ['analysis', 'analysis.tests', 'backward-codecs', 'benchmark', 'benchmark-jmh', 'classification', 'codecs', 'core', 'core.tests',
                       'distribution.tests', 'demo', 'expressions', 'facet', 'grouping', 'highlighter', 'join',
                       'luke', 'memory', 'misc', 'monitor', 'queries', 'queryparser', 'replicator',
                       'sandbox', 'spatial-extras', 'spatial-test-fixtures', 'spatial3d', 'suggest', 'test-framework', 'licenses']
@@ -611,20 +613,21 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
 
     validateCmd = './gradlew --no-daemon check -p lucene/documentation'
     print('    run "%s"' % validateCmd)
-    java.run_java17(validateCmd, '%s/validate.log' % unpackPath)
-
-    print("    run tests w/ Java 17 and testArgs='%s'..." % testArgs)
-    java.run_java17('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
-    print("    compile jars w/ Java 17")
-    java.run_java17('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
-    testDemo(java.run_java17, isSrc, version, '17')
-
-    if java.run_java19:
-      print("    run tests w/ Java 19 and testArgs='%s'..." % testArgs)
-      java.run_java19('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
-      print("    compile jars w/ Java 19")
-      java.run_java19('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
-      testDemo(java.run_java19, isSrc, version, '19')
+    java.run_java(validateCmd, '%s/validate.log' % unpackPath)
+
+    print("    run tests w/ Java %s and testArgs='%s'..." % (BASE_JAVA_VERSION, testArgs))
+    java.run_java('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
+    print("    compile jars w/ Java %s" % BASE_JAVA_VERSION)
+    java.run_java('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
+    testDemo(java.run_java, isSrc, version, BASE_JAVA_VERSION)
+
+    if java.run_alt_javas:
+      for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions):
+        print("    run tests w/ Java %s and testArgs='%s'..." % (alt_java_version, testArgs))
+        run_alt_java('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
+        print("    compile jars w/ Java %s" % alt_java_version)
+        run_alt_java('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
+        testDemo(run_alt_java, isSrc, version, alt_java_version)
 
     print('  confirm all releases have coverage in TestBackwardsCompatibility')
     confirmAllReleasesAreTestedForBackCompat(version, unpackPath)
@@ -633,9 +636,10 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
 
     checkAllJARs(os.getcwd(), gitRevision, version)
 
-    testDemo(java.run_java17, isSrc, version, '17')
-    if java.run_java19:
-      testDemo(java.run_java19, isSrc, version, '19')
+    testDemo(java.run_java, isSrc, version, BASE_JAVA_VERSION)
+    if java.run_alt_javas:
+      for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions):
+        testDemo(run_alt_java, isSrc, version, alt_java_version)
 
   testChangesText('.', version)
 
@@ -664,7 +668,7 @@ def testDemo(run_java, isSrc, version, jdk):
     checkIndexCmd = 'java -ea %s --module org.apache.lucene.core/org.apache.lucene.index.CheckIndex index' % cp
     indexFilesCmd = 'java -Dsmoketester=true %s --module org.apache.lucene.demo/org.apache.lucene.demo.IndexFiles -index index -docs %s' % (cp, docsDir)
     searchFilesCmd = 'java %s --module org.apache.lucene.demo/org.apache.lucene.demo.SearchFiles -index index -query lucene' % cp
-      
+
   run_java(indexFilesCmd, 'index.log')
   run_java(searchFilesCmd, 'search.log')
   reMatchingDocs = re.compile('(\d+) total matching documents')
@@ -911,33 +915,49 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
         sys.stdout.write('.')
 
 
-def make_java_config(parser, java19_home):
-  def _make_runner(java_home, version):
-    print('Java %s JAVA_HOME=%s' % (version, java_home))
+def make_java_config(parser, alt_java_homes):
+  def _make_runner(java_home, is_base_version=False):
     if cygwin:
       java_home = subprocess.check_output('cygpath -u "%s"' % java_home, shell=True).decode('utf-8').strip()
     cmd_prefix = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % \
                  (java_home, java_home, java_home)
     s = subprocess.check_output('%s; java -version' % cmd_prefix,
                                 shell=True, stderr=subprocess.STDOUT).decode('utf-8')
-    if s.find(' version "%s' % version) == -1:
-      parser.error('got wrong version for java %s:\n%s' % (version, s))
+
+    actual_version = re.search(r'version "([1-9][0-9]*)', s).group(1)
+    print('Java %s JAVA_HOME=%s' % (actual_version, java_home))
+
+    # validate Java version
+    if is_base_version:
+      if BASE_JAVA_VERSION != actual_version:
+        parser.error('got wrong base version for java %s:\n%s' % (BASE_JAVA_VERSION, s))
+    else:
+      if int(actual_version) < int(BASE_JAVA_VERSION):
+        parser.error('got wrong version for java %s, less than base version %s:\n%s' % (actual_version, BASE_JAVA_VERSION, s))
+
     def run_java(cmd, logfile):
       run('%s; %s' % (cmd_prefix, cmd), logfile)
-    return run_java
-  java17_home =  os.environ.get('JAVA_HOME')
-  if java17_home is None:
-    parser.error('JAVA_HOME must be set')
-  run_java17 = _make_runner(java17_home, '17')
-  run_java19 = None
-  if java19_home is not None:
-    run_java19 = _make_runner(java19_home, '19')
 
-  jc = namedtuple('JavaConfig', 'run_java17 java17_home run_java19 java19_home')
-  return jc(run_java17, java17_home, run_java19, java19_home)
+    return run_java, actual_version
+
+  java_home =  os.environ.get('JAVA_HOME')
+  if java_home is None:
+    parser.error('JAVA_HOME must be set')
+  run_java, _ = _make_runner(java_home, True)
+  run_alt_javas = []
+  alt_java_versions = []
+  if alt_java_homes:
+    for alt_java_home in alt_java_homes:
+      run_alt_java, version = _make_runner(alt_java_home)
+      run_alt_javas.append(run_alt_java)
+      alt_java_versions.append(version)
+
+  jc = namedtuple('JavaConfig', 'run_java java_home run_alt_javas alt_java_homes alt_java_versions')
+  return jc(run_java, java_home, run_alt_javas, alt_java_homes, alt_java_versions)
 
 version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
 revision_re = re.compile(r'rev-([a-f\d]+)')
+
 def parse_config():
   epilogue = textwrap.dedent('''
     Example usage:
@@ -956,8 +976,8 @@ def parse_config():
                       help='GIT revision number that release was built with, defaults to that in URL')
   parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
                       help='Version of the release, defaults to that in URL')
-  parser.add_argument('--test-java19', metavar='java19_home',
-                      help='Path to Java home directory, to run tests with if specified')
+  parser.add_argument('--test-alternative-java', action='append',
+                      help='Path to alternative Java home directory, to run tests with if specified')
   parser.add_argument('--download-only', action='store_true', default=False,
                       help='Only perform download and sha hash check steps')
   parser.add_argument('url', help='Url pointing to release to test')
@@ -984,7 +1004,7 @@ def parse_config():
   if c.local_keys is not None and not os.path.exists(c.local_keys):
     parser.error('Local KEYS file "%s" not found' % c.local_keys)
 
-  c.java = make_java_config(parser, c.test_java19)
+  c.java = make_java_config(parser, c.test_alternative_java)
 
   if c.tmp_dir:
     c.tmp_dir = os.path.abspath(c.tmp_dir)

gradle/documentation/markdown.gradle

@@ -103,6 +103,8 @@ class MarkdownFilter extends FilterReader {
     // first replace LUCENE and SOLR issue numbers with a markdown link
     markdownSource = markdownSource.replaceAll(/(?s)\b(LUCENE|SOLR)\-\d+\b/,
       '[$0](https://issues.apache.org/jira/browse/$0)');
+    markdownSource = markdownSource.replaceAll(/(?s)\b(GITHUB#|GH-)(\d+)\b/,
+      '[$0](https://github.com/apache/lucene/issues/$2)');
 
     // convert the markdown
     MutableDataSet options = new MutableDataSet();

gradle/documentation/render-javadoc.gradle

@@ -90,14 +90,14 @@ allprojects {
 // Set up titles and link up some offline docs for all documentation
 // (they may be unused but this doesn't do any harm).
 
-def javaJavadocPackages = rootProject.file("${resources}/java11/")
+def javaJavadocPackages = rootProject.file("${resources}/java21/")
 def junitJavadocPackages = rootProject.file("${resources}/junit/")
 allprojects {
   project.tasks.withType(RenderJavadocTask) {
     title = "Lucene ${project.version} ${project.name} API"
 
     offlineLinks += [
-        "https://docs.oracle.com/en/java/javase/11/docs/api/": javaJavadocPackages,
+        "https://docs.oracle.com/en/java/javase/21/docs/api/": javaJavadocPackages,
         "https://junit.org/junit4/javadoc/4.12/": junitJavadocPackages
     ]