From d382b7b621ee13932cd3b948c51ab4f54a1b0393 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 23 Dec 2022 15:00:02 +0100 Subject: [PATCH 1/7] Patch class files for Java 19 code to no longer have the "preview" flag. This enables Java 19 memory segments by default --- gradle/java/javac.gradle | 27 ++++++++ gradle/testing/defaults-tests.gradle | 4 -- .../apache/lucene/store/MMapDirectory.java | 63 ++++++++----------- 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/gradle/java/javac.gradle b/gradle/java/javac.gradle index 67a2e620618f..b59de2a303e3 100644 --- a/gradle/java/javac.gradle +++ b/gradle/java/javac.gradle @@ -100,6 +100,29 @@ configure(project(":lucene:core")) { main19Implementation sourceSets.main.output } + def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor -> + destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details -> + if (!details.directory) { + logger.info("Patching: ${details.file}") + new RandomAccessFile(details.file, 'rw').withCloseable { f -> + f.seek(0L) + int magic = f.readInt(); + if (magic != (int)0xCAFEBABE) { + throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}") + } + f.seek(6L) + short major = f.readShort() + if (major != expectedMajor) { + throw new GradleException("Invalid Java class file version ${major}: ${details.file}") + } + // patch the minor version to 0 (remove preview flag): + f.seek(4L) + f.writeShort(0) + } + } + } + } + tasks.named('compileMain19Java').configure { javaCompiler = javaToolchains.compilerFor { languageVersion = JavaLanguageVersion.of(19) @@ -111,6 +134,10 @@ configure(project(":lucene:core")) { sourceCompatibility = 19 targetCompatibility = 19 options.compilerArgs += ["--release", 19 as String, "--enable-preview"] + + doLast { + patchClassFiles(destinationDirectory, 63) + } } tasks.named('jar').configure { diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle index 6324800e213c..d23fcdc267af 100644 --- a/gradle/testing/defaults-tests.gradle +++ b/gradle/testing/defaults-tests.gradle @@ -124,10 +124,6 @@ allprojects { // (if the runner JVM does not support them, it will fail tests): jvmArgs '--add-modules', 'jdk.unsupported,jdk.management' - if (rootProject.runtimeJavaVersion == JavaVersion.VERSION_19) { - jvmArgs '--enable-preview' - } - systemProperty 'java.util.logging.config.file', file("${resources}/logging.properties") systemProperty 'java.awt.headless', 'true' systemProperty 'jdk.map.althashing.threshold', '0' diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 9d12f1e438e2..aae98a0db6dd 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -61,8 +61,8 @@ * the workaround will be automatically enabled (with no guarantees; if you discover any problems, * you can disable it). * - *

On Java 19 with {@code --enable-preview} command line setting, this class will use the - * modern {@code MemorySegment} API which allows to safely unmap. + *

On exactly Java 19 this class will use the modern {@code MemorySegment} API which + * allows to safely unmap. * *

NOTE: Accessing this class either directly or indirectly from a thread while it's * interrupted can close the underlying channel immediately if at the same time the thread is @@ -106,8 +106,7 @@ public class MMapDirectory extends FSDirectory { * Default max chunk size: * *

@@ -189,9 +188,8 @@ public MMapDirectory(Path path, LockFactory lockFactory, long maxChunkSize) thro * non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented * internal cleanup functionality. * - *

On Java 19 with {@code --enable-preview} command line setting, this class will use the - * modern {@code MemorySegment} API which allows to safely unmap. The following warnings no - * longer apply in that case! + *

On exactly Java 19 this class will use the modern {@code MemorySegment} API which allows to + * safely unmap. The following warnings no longer apply in that case! * *

NOTE: Enabling this is completely unsupported by Java and may lead to JVM crashes if * IndexInput is closed while another thread is still accessing it (SIGSEGV). @@ -351,39 +349,32 @@ default IOException convertMapFailedIOException( private static MMapIndexInputProvider lookupProvider() { final var lookup = MethodHandles.lookup(); - try { - final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); - // we use method handles, so we do not need to deal with setAccessible as we have private - // access through the lookup: - final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class)); + if (Runtime.version().feature() == 19) { try { - return (MMapIndexInputProvider) constr.invoke(); - } catch (RuntimeException | Error e) { - throw e; - } catch (Throwable th) { - throw new AssertionError(th); + final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); + // we use method handles, so we do not need to deal with setAccessible as we have private + // access through the lookup: + final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class)); + try { + return (MMapIndexInputProvider) constr.invoke(); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable th) { + throw new AssertionError(th); + } + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new LinkageError( + "MemorySegmentIndexInputProvider is missing correctly typed constructor", e); + } catch (ClassNotFoundException cnfe) { + throw new LinkageError( + "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe); } - } catch ( - @SuppressWarnings("unused") - ClassNotFoundException e) { - // we're before Java 19 - return new MappedByteBufferIndexInputProvider(); - } catch ( - @SuppressWarnings("unused") - UnsupportedClassVersionError e) { + } else if (Runtime.version().feature() >= 20) { var log = Logger.getLogger(lookup.lookupClass().getName()); - if (Runtime.version().feature() == 19) { - log.warning( - "You are running with Java 19. To make full use of MMapDirectory, please pass '--enable-preview' to the Java command line."); - } else { - log.warning( - "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene."); - } - return new MappedByteBufferIndexInputProvider(); - } catch (NoSuchMethodException | IllegalAccessException e) { - throw new LinkageError( - "MemorySegmentIndexInputProvider is missing correctly typed constructor", e); + log.warning( + "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene."); } + return new MappedByteBufferIndexInputProvider(); } static { From 7b314a3775bed3a66ca9b3477e77c78f19565570 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 23 Dec 2022 15:28:46 +0100 Subject: [PATCH 2/7] Put all Java 19 MR-JAR compilation and patching into separate gradle file --- build.gradle | 1 + gradle/java/javac.gradle | 72 --------------------- gradle/java/memorysegment-mrjar.gradle | 90 ++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 72 deletions(-) create mode 100644 gradle/java/memorysegment-mrjar.gradle diff --git a/build.gradle b/build.gradle index 24e8b580583c..6032f3a72cc0 100644 --- a/build.gradle +++ b/build.gradle @@ -119,6 +119,7 @@ apply from: file('gradle/ide/eclipse.gradle') // (java, tests) apply from: file('gradle/java/folder-layout.gradle') apply from: file('gradle/java/javac.gradle') +apply from: file('gradle/java/memorysegment-mrjar.gradle') apply from: file('gradle/testing/defaults-tests.gradle') apply from: file('gradle/testing/randomization.gradle') apply from: file('gradle/testing/fail-on-no-tests.gradle') diff --git a/gradle/java/javac.gradle b/gradle/java/javac.gradle index b59de2a303e3..5cc3c9aaad2f 100644 --- a/gradle/java/javac.gradle +++ b/gradle/java/javac.gradle @@ -79,75 +79,3 @@ allprojects { } } } - -configure(project(":lucene:core")) { - plugins.withType(JavaPlugin) { - sourceSets { - main19 { - java { - srcDirs = ['src/java19'] - } - } - } - - configurations { - // Inherit any dependencies from the main source set. - main19Implementation.extendsFrom implementation - } - - dependencies { - // We need the main classes to compile our Java 19 pieces. - main19Implementation sourceSets.main.output - } - - def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor -> - destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details -> - if (!details.directory) { - logger.info("Patching: ${details.file}") - new RandomAccessFile(details.file, 'rw').withCloseable { f -> - f.seek(0L) - int magic = f.readInt(); - if (magic != (int)0xCAFEBABE) { - throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}") - } - f.seek(6L) - short major = f.readShort() - if (major != expectedMajor) { - throw new GradleException("Invalid Java class file version ${major}: ${details.file}") - } - // patch the minor version to 0 (remove preview flag): - f.seek(4L) - f.writeShort(0) - } - } - } - } - - tasks.named('compileMain19Java').configure { - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(19) - } - - // undo alternative JDK support: - options.forkOptions.javaHome = null - - sourceCompatibility = 19 - targetCompatibility = 19 - options.compilerArgs += ["--release", 19 as String, "--enable-preview"] - - doLast { - patchClassFiles(destinationDirectory, 63) - } - } - - tasks.named('jar').configure { - into('META-INF/versions/19') { - from sourceSets.main19.output - } - - manifest.attributes( - 'Multi-Release': 'true' - ) - } - } -} diff --git a/gradle/java/memorysegment-mrjar.gradle b/gradle/java/memorysegment-mrjar.gradle new file mode 100644 index 000000000000..7943e3dbe1c2 --- /dev/null +++ b/gradle/java/memorysegment-mrjar.gradle @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Produce an MR-JAR with Java 19 MemorySegment implementation for MMapDirectory + +configure(project(":lucene:core")) { + plugins.withType(JavaPlugin) { + sourceSets { + main19 { + java { + srcDirs = ['src/java19'] + } + } + } + + configurations { + // Inherit any dependencies from the main source set. + main19Implementation.extendsFrom implementation + } + + dependencies { + // We need the main classes to compile our Java 19 pieces. + main19Implementation sourceSets.main.output + } + + def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor -> + destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details -> + if (!details.directory) { + logger.info("Patching: ${details.file}") + new RandomAccessFile(details.file, 'rw').withCloseable { f -> + f.seek(0L) + int magic = f.readInt(); + if (magic != (int)0xCAFEBABE) { + throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}") + } + f.seek(6L) + short major = f.readShort() + if (major != expectedMajor) { + throw new GradleException("Invalid Java class file version ${major}: ${details.file}") + } + // patch the minor version to 0 (remove preview flag): + f.seek(4L) + f.writeShort(0) + } + } + } + } + + tasks.named('compileMain19Java').configure { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(19) + } + + // undo alternative JDK support: + options.forkOptions.javaHome = null + + sourceCompatibility = 19 + targetCompatibility = 19 + options.compilerArgs += ["--release", 19 as String, "--enable-preview"] + + doLast { + patchClassFiles(destinationDirectory, 63) + } + } + + tasks.named('jar').configure { + into('META-INF/versions/19') { + from sourceSets.main19.output + } + + manifest.attributes( + 'Multi-Release': 'true' + ) + } + } +} From 41263cd4a0f0520da396e76a33a971cf81787b8b Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 23 Dec 2022 16:06:32 +0100 Subject: [PATCH 3/7] Add CHANGES.txt --- lucene/CHANGES.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 7a54e8cc1e36..d9424ccf7b1f 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -157,6 +157,11 @@ New Features use numeric fields that perform well both for filtering and sorting. (Francisco Fernández Castaño) +* GITHUB#12033: Support for Java 19 foreign memory support is now enabled by default, + no need to pass "--enable-preview" on the command line. If exactly Java 19 is used, + MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and + indexes closed while queries are running can no longer crash the JVM. (Uwe Schindler) + Improvements --------------------- * GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori From b7e31bbf9d784ec8fef3d6b7574aa5dc0b3f0279 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 23 Dec 2022 16:34:35 +0100 Subject: [PATCH 4/7] remove useless seek --- gradle/java/memorysegment-mrjar.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle/java/memorysegment-mrjar.gradle b/gradle/java/memorysegment-mrjar.gradle index 7943e3dbe1c2..f36f8873b39b 100644 --- a/gradle/java/memorysegment-mrjar.gradle +++ b/gradle/java/memorysegment-mrjar.gradle @@ -42,7 +42,6 @@ configure(project(":lucene:core")) { if (!details.directory) { logger.info("Patching: ${details.file}") new RandomAccessFile(details.file, 'rw').withCloseable { f -> - f.seek(0L) int magic = f.readInt(); if (magic != (int)0xCAFEBABE) { throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}") From 59bce69f3f9bc83a8e2913e168fed8a9637e3306 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 25 Dec 2022 13:50:44 +0100 Subject: [PATCH 5/7] Remove obsolete case --- .../src/test/org/apache/lucene/store/TestMmapDirectory.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java index 579b0c27108c..5faf12484cc1 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java @@ -52,8 +52,6 @@ public void testCorrectImplementation() { assertTrue( "on Java 19 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs", isMemorySegmentImpl()); - } else if (runtimeVersion > 19) { - // TODO: We don't know how this is handled in later Java versions, so make no assumptions! } else { assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass()); } From 79d31026c4032be1b1ac2357f5eae83336414266 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 25 Dec 2022 13:54:07 +0100 Subject: [PATCH 6/7] Prepare for Java 20 checks --- .../core/src/java/org/apache/lucene/store/MMapDirectory.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index aae98a0db6dd..2fc3b9e44157 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -349,7 +349,8 @@ default IOException convertMapFailedIOException( private static MMapIndexInputProvider lookupProvider() { final var lookup = MethodHandles.lookup(); - if (Runtime.version().feature() == 19) { + final int runtimeVersion = Runtime.version().feature(); + if (runtimeVersion == 19) { try { final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); // we use method handles, so we do not need to deal with setAccessible as we have private @@ -369,7 +370,7 @@ private static MMapIndexInputProvider lookupProvider() { throw new LinkageError( "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe); } - } else if (Runtime.version().feature() >= 20) { + } else if (runtimeVersion >= 20) { var log = Logger.getLogger(lookup.lookupClass().getName()); log.warning( "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene."); From b2ce3219a6a49e42bac6519c024f3b926c16965a Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 25 Dec 2022 14:19:43 +0100 Subject: [PATCH 7/7] Prepare for Java 20/21 --- gradle/generation/local-settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/generation/local-settings.gradle b/gradle/generation/local-settings.gradle index 388e6a129db3..eaf380bad04b 100644 --- a/gradle/generation/local-settings.gradle +++ b/gradle/generation/local-settings.gradle @@ -101,7 +101,7 @@ tests.jvms=${testsJvms} org.gradle.java.installations.auto-download=true # Set these to enable automatic JVM location discovery. -org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME +org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME #org.gradle.java.installations.paths=(custom paths) """, "UTF-8")