Merge remote-tracking branch 'upstream/main' into ubi

.github/workflows/bin-solr-test.yml

@@ -24,11 +24,11 @@ jobs:
     steps:
     # Setup
     - uses: actions/checkout@v4
-    - name: Set up JDK 11
+    - name: Set up JDK
       uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: 11
+        java-version: 21
         java-package: jdk
     - name: Setup Gradle
       uses: gradle/actions/setup-gradle@v4

.github/workflows/docker-test.yml

@@ -26,11 +26,11 @@ jobs:
     steps:
     # Setup
     - uses: actions/checkout@v4
-    - name: Set up JDK 11
+    - name: Set up JDK 21
       uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: 11
+        java-version: 21
         java-package: jdk
     - name: Setup Gradle
       uses: gradle/actions/setup-gradle@v4

.github/workflows/gradle-precommit.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   test:
-    name: gradle check w/ Java 11
+    name: gradle check
 
     runs-on: ubuntu-latest
 
@@ -19,11 +19,11 @@ jobs:
     # Setup
     - uses: actions/checkout@v4
 
-    - name: Set up JDK 11
+    - name: Set up JDK
       uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: 11
+        java-version: 21
         java-package: jdk
 
     - name: Setup Gradle

.github/workflows/solrj-test.yml

@@ -21,11 +21,11 @@ jobs:
     steps:
     # Setup
     - uses: actions/checkout@v4
-    - name: Set up JDK 11
+    - name: Set up JDK 21
       uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: 11
+        java-version: 21
         java-package: jdk
     - name: Setup Gradle
       uses: gradle/actions/setup-gradle@v4

README.md

@@ -22,7 +22,7 @@ Solr is the blazing-fast, open source, multi-modal search platform built on [Apa
 It powers full-text, vector, and geospatial search at many of the world's largest organizations.
 
 [![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/badge/icon?subject=Solr%20Artifacts)](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/)
-[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/badge/icon?subject=Solr%20Check)](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/)
+[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Lint-main/badge/icon?subject=Solr%20Lint)](https://ci-builds.apache.org/job/Solr/job/Solr-Lint-main/)
 
 For a complete description of the Solr project, team composition, source
 code repositories, and other details, please see the Solr web site at
@@ -94,7 +94,7 @@ Solr uses [Gradle](https://gradle.org/) for its build system. Here are some usef
 
 ```
 cd ./solr/packaging/build/dev
-bin/solr start -c
+bin/solr start
 ```
 
 - Open a web browser and go to http://localhost:8983/solr/ to access the Solr Admin interface. You can also use the `bin/solr` script to create and manage Solr collections. For example use the `bin/solr post` tool to index some sample data.
@@ -108,4 +108,3 @@ To get involved in the developer community:
 - Slack: `#solr-dev` in the `the-asf` organization.  Sign up at https://the-asf.slack.com/messages/CE70MDPMF
 - [Issue Tracker (JIRA)](https://issues.apache.org/jira/browse/SOLR)
 - IRC: `#solr-dev` on [libera.chat](https://web.libera.chat/?channels=#solr-dev)
-

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -22,7 +22,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
-import java.net.URL;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -54,10 +54,10 @@ public static void main(String[] args) {
   }
 
   public static void checkVersion() {
-    int major = Runtime.getRuntime().version().feature();
-    if (major < 11 || major > 21) {
+    int major = Runtime.version().feature();
+    if (major < 21 || major > 23) {
       throw new IllegalStateException(
-          "java version must be between 11 and 21, your version: " + major);
+          "java version must be between 21 and 23, your version: " + major);
     }
   }
 
@@ -89,12 +89,12 @@ public void run(Path destination) throws IOException, NoSuchAlgorithmException {
       }
     }
 
-    URL url =
-        new URL(
+    URI uri =
+        URI.create(
             "https://raw.githubusercontent.com/gradle/gradle/v"
                 + wrapperVersion
                 + "/gradle/wrapper/gradle-wrapper.jar");
-    System.err.println("Downloading gradle-wrapper.jar from " + url);
+    System.err.println("Downloading gradle-wrapper.jar from " + uri);
 
     // Zero-copy save the jar to a temp file
     Path temp = Files.createTempFile(destination.getParent(), ".gradle-wrapper", ".tmp");
@@ -103,7 +103,7 @@ public void run(Path destination) throws IOException, NoSuchAlgorithmException {
       int retryDelay = 30;
       HttpURLConnection connection;
       while (true) {
-        connection = (HttpURLConnection) url.openConnection();
+        connection = (HttpURLConnection) uri.toURL().openConnection();
         try {
           connection.connect();
         } catch (IOException e) {

build-tools/scriptDepVersions.gradle

@@ -21,10 +21,11 @@
 
 ext {
   scriptDepVersions = [
-      "min-java-version": "11",
+      "min-java-version": "21",
+      "min-solrj-java-version" : "17",
       "apache-rat": "0.15",
       "commons-codec": "1.16.0",
-      "ecj": "3.33.0",
+      "ecj": "3.39.0",
       "javacc": "7.0.12",
       "jgit": "6.7.0.202309050840-r",
       "flexmark": "0.64.8",

build.gradle

@@ -35,7 +35,7 @@ apply from: file('build-tools/scriptDepVersions.gradle')
 
 // Declare default Java versions for the entire project and for SolrJ separately
 rootProject.ext.minJavaVersionDefault = JavaVersion.toVersion(scriptDepVersions['min-java-version'])
-rootProject.ext.minJavaVersionSolrJ = JavaVersion.toVersion(scriptDepVersions['min-java-version'])
+rootProject.ext.minJavaVersionSolrJ = JavaVersion.toVersion(scriptDepVersions['min-solrj-java-version'])
 
 apply from: file('gradle/globals.gradle')
 

dev-docs/apis.adoc

@@ -81,4 +81,3 @@ A good example for each of these steps can be seen in Solr's v2 "add-replica-pro
 While we've settled on JAX-RS as our framework for defining v2 APIs going forward, Solr still retains many v2 APIs that were written using an older homegrown framework.
 This framework defines APIs using annotations (e.g. `@EndPoint`) similar to those used by JAX-RS, but lacks the full range of features and 3rd-party tooling.
 We're in the process of migrating these API definitions to JAX-RS and hope to remove all support for this legacy framework in a future release.
-

dev-docs/asf-jenkins.adoc

@@ -0,0 +1,73 @@
+= ASF Jenkins Setup
+:toc: left
+
+The Solr project uses a Jenkins instance provided by the Apache Software Foundation ("ASF") for running tests, validation, etc.
+
+This file aims to document our [ASF Jenkins](https://ci-builds.apache.org/job/Solr/) usage and administration, to prevent it from becoming "tribal knowledge" understood by just a few.
+
+== Jobs
+
+We run a number of jobs on Jenkins, each validating an overlapping set of concerns:
+
+* `Solr-Artifacts-*` - daily jobs that run `./gradlew assemble` to ensure that build artifacts (except docker images) can be created successfully
+* `Solr-Lint-*` - daily jobs that run static analysis (i.e. `precommit` and `check -x test`) on a branch
+* `Solr-Test-*` - "hourly" jobs that run all (non-integration) tests (i.e. `./gradlew test`)
+* `Solr-TestIntegration-*` - daily jobs that run project integration tests (i.e. `./gradlew integrationTests`)
+* `Solr-Docker-Nightly-*` - daily jobs that `./gradlew testDocker dockerPush` to validate docker image packaging.  Snapshot images are pushed to hub.docker.com
+* `Solr-reference-guide-*` - daily jobs that build the Solr reference guide via `./gradlew checkSite` and push the resulting artifact to the staging/preview site `nightlies.apache.org`
+* `Solr-Smoketest-*` - daily jobs that produce a snapshot release (via the `assembleRelease` task) and run the release smoketester
+
+Most jobs that validate particular build artifacts are run "daily", which is sufficient to prevent any large breaks from creeping into the build.
+On the other hand, jobs that run tests are triggered "hourly" in order to squeeze as many test runs as possible out of our Jenkins hardware.
+This is a necessary consequence of Solr's heavy use of randomization in its test-suite.
+"Hourly" scheduling ensures that a test run is either currently running or in the build queue at all times, and enables us to get the maximum data points from our hardware.
+
+== Jenkins Agents
+
+All Solr jobs run on Jenkins agents marked with the 'solr' label.
+Currently, this maps to two Jenkins agents:
+
+* `lucene-solr-1` - available at lucene1-us-west.apache.org
+* `lucene-solr-2` - available (confusingly) at lucene-us-west.apache.org
+
+These agents are "project-specific" VMs shared by the Lucene and Solr projects.
+That is: they are VMs requested by a project for their exclusive use.
+(INFRA policy appears to be that each Apache project may request 1 dedicated VM; it's unclear how Solr ended up with 2.)
+
+Maintenance of these agent VMs falls into a bit of a gray area.
+INFRA will still intervene when asked: to reboot nodes, to deploy OS upgrades, etc.
+But some burden also falls on Lucene and Solr as project teams to monitor the the VMs and keep them healthy.
+
+=== Accessing Jenkins Agents
+
+With a few steps, Solr committers can access our project's Jenkins agent VMs via SSH to troubleshoot and resolve issues.
+
+1. Ensure your account on id.apache.org has an SSH key associated with it.
+2. Ask INFRA to give your Apache ID SSH access to these boxes. (See [this JIRA ticket](https://issues.apache.org/jira/browse/INFRA-3682) for an example.)
+3. SSH into the desired box with: `ssh <apache-id>@$HOSTNAME` (where `$HOSTNAME` is either `lucene1-us-west.apache.org` or `lucene-us-west.apache.org`)
+
+Often, SSH access on the boxes is not sufficient, and administrators require "root" access to diagnose and solve problems.
+Sudo/su priveleges can be accessed via a one-time pad ("OTP") challenge, managed by the "Orthrus PAM" module.
+Users in need of root access can perform the following steps:
+
+1. Open the ASF's [OTP Generator Tool](https://selfserve.apache.org/otp-calculator.html) in your browser of choice
+2. Run `ortpasswd` on the machine.  This will print out a OTP "challenge" (e.g. `otp-md5 497 lu6126`) and provide a password prompt.  This password prompt should be given a OTP password, generated in steps 3-5 below.
+3. Copy the "challenge" from the previous step into the relevant field on the "OTP Generator Tool" form.
+4. Choose a password to use for OTP Challenges (or recall one you've used in the past), and type this into the relevant field on the "OTP Generator Tool" form.
+5. Click "Compute", and copy the first line from the "Response" box into your SSH session's password prompt.  You're now established in the "Orthrus PAM" system.
+6. Run a command requesting `su` escalation (e.g. `sudo su -`).  This should print another "challenge" and password prompt.  Repeat steps 3-5.
+
+If this fails at any point, open a ticket with INFRA.
+You may need to be added to the 'sudoers' file for the VM(s) in question.
+
+=== Known Jenkins Issues
+
+One recurring problem with the Jenkins agents is that they periodically run out of disk-space.
+Usually this happens when enough "workspaces" are orphaned or left behind, consuming all of the agent's disk space.
+
+Solr Jenkins jobs are currently configured to clean up the previous workspace at the *start* of the subsequent run.
+This avoids orphans in the common case but leaves workspaces behind any time a job is renamed or deleted (as happens during the Solr release process).
+
+Luckily, this has an easy fix: SSH into the agent VM and delete any workspaces no longer needed in `/home/jenkins/jenkins-slave/workspace/Solr`.
+Any workspace that doesn't correspond to a [currently existing job](https://ci-builds.apache.org/job/Solr/) can be safely deleted.
+(It may also be worth comparing the Lucene workspaces in `/home/jenkins/jenkins-slave/workspace/Lucene` to [that project's list of jobs](https://ci-builds.apache.org/job/Lucene/).)

dev-docs/solr-source-code.adoc

@@ -2,10 +2,10 @@
 
 ## Building Solr from Source
 
-Download the Java 11 JDK (Java Development Kit) or later.
+Download the Java 21 JDK (Java Development Kit) or later.
 We recommend the OpenJDK distribution Eclipse Temurin available from https://adoptium.net/.
 You will need the JDK installed, and the $JAVA_HOME/bin (Windows: %JAVA_HOME%\bin) folder included on your command path.
-To test this, issue a "java -version" command from your shell (command prompt) and verify that the Java version is 11 or later.
+To test this, issue a "java -version" command from your shell (command prompt) and verify that the Java version is 21 or later.
 See the xref:jvms.adoc[JVM developer doc] for more information on Gradle and JVMs.
 
 Clone the latest Apache Solr source code directly from the Git repository: <https://solr.apache.org/community.html#version-control>.

dev-tools/scripts/releaseWizard.yaml

@@ -1864,16 +1864,9 @@ groups:
       root_folder: '{{ git_checkout_folder }}'
       commands_text: |
         Run these commands to delete proposed versions from distribution directory.
-        Note, as long as we have some releases (7.x, 8.x) in Lucene dist repo and other
-        releases (9.0 ->) in the Solr dist repo, we may need to delete two places.
-
+        
         WARNING: Validate that the proposal is correct!
       commands:
-      - !Command
-        cmd: |
-          svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %}  https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}
-        logfile: svn-rm-solr.log
-        comment: Delete from Lucene dist area
       - !Command
         cmd: |
           svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %}  https://dist.apache.org/repos/dist/release/solr/solr/{{ ver }}{% endfor %}

dev-tools/scripts/smokeTestRelease.py

@@ -774,8 +774,6 @@ def testSolrExample(binaryDistPath, javaPath):
       raise RuntimeError('Failed to run the techproducts example, check log for previous errors.')
 
     os.chdir('example')
-    print('      test utf8...')
-    run('sh ./exampledocs/test_utf8.sh http://localhost:8983/solr/techproducts', 'utf8.log')
     print('      run query...')
     s = load('http://localhost:8983/solr/techproducts/select/?q=video')
     if s.find('"numFound":3,') == -1:

gradle/documentation/render-javadoc.gradle

@@ -32,7 +32,7 @@ allprojects {
       missingdoclet "org.apache.solr.tools:missing-doclet"
     }
 
-    ext {
+    project.ext {
       relativeDocPath = project.path.replaceFirst(/:\w+:/, "").replace(':', '/')
     }
 

gradle/globals.gradle

@@ -37,7 +37,7 @@ allprojects {
   // so :solr:core will have solr-core.jar, etc.
   project.archivesBaseName = project.path.replaceAll("^:", "").replace(':', '-')
 
-  ext {
+  project.ext {
     // Utility method to support passing overrides via -P or -D.
     propertyOrDefault = { propName, defValue ->
       def result
@@ -173,5 +173,6 @@ allprojects {
     // Assign different java version for client-side modules 'api' and 'solrj*'
     var isSolrJ = project.name.matches("^(solrj.*|api)\$")
     minJavaVersion = isSolrJ ? rootProject.minJavaVersionSolrJ : rootProject.minJavaVersionDefault
+    minJavaTestVersion = rootProject.minJavaVersionDefault
   }
 }

gradle/java/javac.gradle

@@ -19,14 +19,19 @@
 
 allprojects {
   plugins.withType(JavaPlugin) {
-    sourceCompatibility = project.minJavaVersion
-    targetCompatibility = project.minJavaVersion
-
-    // Use 'release' flag instead of 'source' and 'target'
-    tasks.withType(JavaCompile) {
-      options.compilerArgs += ["--release", project.minJavaVersion.toString()]
-    }
-
+      // Use 'release' flag instead of 'source' and 'target'
+      tasks.withType(JavaCompile) {
+        compileTestJava {
+          sourceCompatibility = project.minJavaTestVersion
+          targetCompatibility = project.minJavaTestVersion
+          options.compilerArgs += ["--release", project.minJavaTestVersion.toString()]
+        }
+        compileJava {
+          sourceCompatibility = project.minJavaVersion
+          targetCompatibility = project.minJavaVersion
+          options.compilerArgs += ["--release", project.minJavaVersion.toString()]
+        }
+      }
     // Configure warnings.
     tasks.withType(JavaCompile) {
       options.encoding = "UTF-8"
@@ -51,22 +56,12 @@ allprojects {
         "-Xdoclint:all/protected",
         "-Xdoclint:-missing",
         "-Xdoclint:-accessibility",
+        "-Xlint:synchronization",
+        "-Xlint:text-blocks",
         "-proc:none",  // proc:none was added because of LOG4J2-1925 / JDK-8186647
+        "-Xlint:removal"
       ]
 
-      // enable some warnings only relevant to newer language features
-      if (rootProject.runtimeJavaVersion >= JavaVersion.VERSION_15) {
-        options.compilerArgs += [
-          "-Xlint:text-blocks",
-        ]
-      }
-
-      if (rootProject.runtimeJavaVersion >= JavaVersion.VERSION_16) {
-        options.compilerArgs += [
-          "-Xlint:synchronization",
-        ]
-      }
-
       if (propertyOrDefault("javac.failOnWarnings", true).toBoolean()) {
         options.compilerArgs += "-Werror"
       }

gradle/node.gradle

@@ -34,7 +34,7 @@ configure([project(":solr:packaging"), project(":solr:solr-ref-guide"), project(
         }
     }
 
-    ext {
+    project.ext {
         rootNodeDir = "$rootDir/.gradle/node"
         nodeProjectDir = file("$rootNodeDir/$project.name")
     }

gradle/solr/packaging.gradle

@@ -38,7 +38,7 @@
 
 configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules:") || project.path == ":solr:prometheus-exporter" || project.path == ":solr:cross-dc-manager" }) {
   plugins.withType(JavaPlugin) {
-    ext {
+    project.ext {
       packagingDir = file("${buildDir}/packaging")
       if (project.path.startsWith(":solr:prometheus-exporter") || project.path.startsWith(":solr:cross-dc-manager")) {
         deps = packagingDir

gradle/template.gradle.properties

@@ -49,6 +49,11 @@
 # tests.minheapsize=512m
 # tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1
 #
+# If you want tests to produce an html report (which intellij provides a clickable link for
+# at the end of a failed build) set this to true, defaults to false to save a few seconds.
+#
+# tests.html=false
+#
 #################
 # Gradle Daemon #
 #################
@@ -98,5 +103,8 @@ org.gradle.workers.max=@MAX_WORKERS@
 # Maximum number of test JVMs forked per test task.
 tests.jvms=@TEST_JVMS@
 
+# By default skip html generation
+tests.html=false
+
 # Disable auto JVM provisioning (we don't use toolchains yet but want no surprises).
 org.gradle.java.installations.auto-download=false