diff --git a/build.gradle b/build.gradle index 24e8b580583c..6032f3a72cc0 100644 --- a/build.gradle +++ b/build.gradle @@ -119,6 +119,7 @@ apply from: file('gradle/ide/eclipse.gradle') // (java, tests) apply from: file('gradle/java/folder-layout.gradle') apply from: file('gradle/java/javac.gradle') +apply from: file('gradle/java/memorysegment-mrjar.gradle') apply from: file('gradle/testing/defaults-tests.gradle') apply from: file('gradle/testing/randomization.gradle') apply from: file('gradle/testing/fail-on-no-tests.gradle') diff --git a/gradle/generation/local-settings.gradle b/gradle/generation/local-settings.gradle index 388e6a129db3..eaf380bad04b 100644 --- a/gradle/generation/local-settings.gradle +++ b/gradle/generation/local-settings.gradle @@ -101,7 +101,7 @@ tests.jvms=${testsJvms} org.gradle.java.installations.auto-download=true # Set these to enable automatic JVM location discovery. -org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME +org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME #org.gradle.java.installations.paths=(custom paths) """, "UTF-8") diff --git a/gradle/java/javac.gradle b/gradle/java/javac.gradle index 67a2e620618f..5cc3c9aaad2f 100644 --- a/gradle/java/javac.gradle +++ b/gradle/java/javac.gradle @@ -79,48 +79,3 @@ allprojects { } } } - -configure(project(":lucene:core")) { - plugins.withType(JavaPlugin) { - sourceSets { - main19 { - java { - srcDirs = ['src/java19'] - } - } - } - - configurations { - // Inherit any dependencies from the main source set. - main19Implementation.extendsFrom implementation - } - - dependencies { - // We need the main classes to compile our Java 19 pieces. - main19Implementation sourceSets.main.output - } - - tasks.named('compileMain19Java').configure { - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(19) - } - - // undo alternative JDK support: - options.forkOptions.javaHome = null - - sourceCompatibility = 19 - targetCompatibility = 19 - options.compilerArgs += ["--release", 19 as String, "--enable-preview"] - } - - tasks.named('jar').configure { - into('META-INF/versions/19') { - from sourceSets.main19.output - } - - manifest.attributes( - 'Multi-Release': 'true' - ) - } - } -} diff --git a/gradle/java/memorysegment-mrjar.gradle b/gradle/java/memorysegment-mrjar.gradle new file mode 100644 index 000000000000..f36f8873b39b --- /dev/null +++ b/gradle/java/memorysegment-mrjar.gradle @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Produce an MR-JAR with Java 19 MemorySegment implementation for MMapDirectory + +configure(project(":lucene:core")) { + plugins.withType(JavaPlugin) { + sourceSets { + main19 { + java { + srcDirs = ['src/java19'] + } + } + } + + configurations { + // Inherit any dependencies from the main source set. + main19Implementation.extendsFrom implementation + } + + dependencies { + // We need the main classes to compile our Java 19 pieces. + main19Implementation sourceSets.main.output + } + + def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor -> + destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details -> + if (!details.directory) { + logger.info("Patching: ${details.file}") + new RandomAccessFile(details.file, 'rw').withCloseable { f -> + int magic = f.readInt(); + if (magic != (int)0xCAFEBABE) { + throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}") + } + f.seek(6L) + short major = f.readShort() + if (major != expectedMajor) { + throw new GradleException("Invalid Java class file version ${major}: ${details.file}") + } + // patch the minor version to 0 (remove preview flag): + f.seek(4L) + f.writeShort(0) + } + } + } + } + + tasks.named('compileMain19Java').configure { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(19) + } + + // undo alternative JDK support: + options.forkOptions.javaHome = null + + sourceCompatibility = 19 + targetCompatibility = 19 + options.compilerArgs += ["--release", 19 as String, "--enable-preview"] + + doLast { + patchClassFiles(destinationDirectory, 63) + } + } + + tasks.named('jar').configure { + into('META-INF/versions/19') { + from sourceSets.main19.output + } + + manifest.attributes( + 'Multi-Release': 'true' + ) + } + } +} diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle index 6324800e213c..d23fcdc267af 100644 --- a/gradle/testing/defaults-tests.gradle +++ b/gradle/testing/defaults-tests.gradle @@ -124,10 +124,6 @@ allprojects { // (if the runner JVM does not support them, it will fail tests): jvmArgs '--add-modules', 'jdk.unsupported,jdk.management' - if (rootProject.runtimeJavaVersion == JavaVersion.VERSION_19) { - jvmArgs '--enable-preview' - } - systemProperty 'java.util.logging.config.file', file("${resources}/logging.properties") systemProperty 'java.awt.headless', 'true' systemProperty 'jdk.map.althashing.threshold', '0' diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 7a54e8cc1e36..d9424ccf7b1f 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -157,6 +157,11 @@ New Features use numeric fields that perform well both for filtering and sorting. (Francisco Fernández Castaño) +* GITHUB#12033: Support for Java 19 foreign memory support is now enabled by default, + no need to pass "--enable-preview" on the command line. If exactly Java 19 is used, + MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and + indexes closed while queries are running can no longer crash the JVM. (Uwe Schindler) + Improvements --------------------- * GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 9d12f1e438e2..2fc3b9e44157 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -61,8 +61,8 @@ * the workaround will be automatically enabled (with no guarantees; if you discover any problems, * you can disable it). * - *

On Java 19 with {@code --enable-preview} command line setting, this class will use the - * modern {@code MemorySegment} API which allows to safely unmap. + *

On exactly Java 19 this class will use the modern {@code MemorySegment} API which + * allows to safely unmap. * *

NOTE: Accessing this class either directly or indirectly from a thread while it's * interrupted can close the underlying channel immediately if at the same time the thread is @@ -106,8 +106,7 @@ public class MMapDirectory extends FSDirectory { * Default max chunk size: * *

@@ -189,9 +188,8 @@ public MMapDirectory(Path path, LockFactory lockFactory, long maxChunkSize) thro * non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented * internal cleanup functionality. * - *

On Java 19 with {@code --enable-preview} command line setting, this class will use the - * modern {@code MemorySegment} API which allows to safely unmap. The following warnings no - * longer apply in that case! + *

On exactly Java 19 this class will use the modern {@code MemorySegment} API which allows to + * safely unmap. The following warnings no longer apply in that case! * *

NOTE: Enabling this is completely unsupported by Java and may lead to JVM crashes if * IndexInput is closed while another thread is still accessing it (SIGSEGV). @@ -351,39 +349,33 @@ default IOException convertMapFailedIOException( private static MMapIndexInputProvider lookupProvider() { final var lookup = MethodHandles.lookup(); - try { - final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); - // we use method handles, so we do not need to deal with setAccessible as we have private - // access through the lookup: - final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class)); + final int runtimeVersion = Runtime.version().feature(); + if (runtimeVersion == 19) { try { - return (MMapIndexInputProvider) constr.invoke(); - } catch (RuntimeException | Error e) { - throw e; - } catch (Throwable th) { - throw new AssertionError(th); + final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); + // we use method handles, so we do not need to deal with setAccessible as we have private + // access through the lookup: + final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class)); + try { + return (MMapIndexInputProvider) constr.invoke(); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable th) { + throw new AssertionError(th); + } + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new LinkageError( + "MemorySegmentIndexInputProvider is missing correctly typed constructor", e); + } catch (ClassNotFoundException cnfe) { + throw new LinkageError( + "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe); } - } catch ( - @SuppressWarnings("unused") - ClassNotFoundException e) { - // we're before Java 19 - return new MappedByteBufferIndexInputProvider(); - } catch ( - @SuppressWarnings("unused") - UnsupportedClassVersionError e) { + } else if (runtimeVersion >= 20) { var log = Logger.getLogger(lookup.lookupClass().getName()); - if (Runtime.version().feature() == 19) { - log.warning( - "You are running with Java 19. To make full use of MMapDirectory, please pass '--enable-preview' to the Java command line."); - } else { - log.warning( - "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene."); - } - return new MappedByteBufferIndexInputProvider(); - } catch (NoSuchMethodException | IllegalAccessException e) { - throw new LinkageError( - "MemorySegmentIndexInputProvider is missing correctly typed constructor", e); + log.warning( + "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene."); } + return new MappedByteBufferIndexInputProvider(); } static { diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java index 579b0c27108c..5faf12484cc1 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java @@ -52,8 +52,6 @@ public void testCorrectImplementation() { assertTrue( "on Java 19 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs", isMemorySegmentImpl()); - } else if (runtimeVersion > 19) { - // TODO: We don't know how this is handled in later Java versions, so make no assumptions! } else { assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass()); }