Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Register all polyglot java import classes for refective access #9997

Merged
merged 22 commits into from
May 23, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
5cddb23
Register all polyglot java import classes for refective access
JaroslavTulach May 18, 2024
773af10
org.enso.syntax2.Parser may be loaded during image build time now
JaroslavTulach May 19, 2024
e70f53a
Print out summary in EnsoLibraryFeature
JaroslavTulach May 19, 2024
ee24418
Don't initialize HttpClient in static class initializer
JaroslavTulach May 19, 2024
6886b12
Simpler and less verbose lookupJavaClass
JaroslavTulach May 19, 2024
6f3bfad
Test native-image to include instance methods of additional classes
JaroslavTulach May 19, 2024
e409b00
Right now we only analyze Standard.Base
JaroslavTulach May 19, 2024
e9551b7
Don't use asGuestValue in GetSourceLocationNode
JaroslavTulach May 19, 2024
e9a55f0
Report missing Java classes as DataflowError
JaroslavTulach May 19, 2024
5cc50de
Execute all Text. tests in CI
JaroslavTulach May 19, 2024
8ab9362
Initialize Parser before executing the tests
JaroslavTulach May 20, 2024
da144be
License framework believes guava and checkerframework are gone from t…
JaroslavTulach May 20, 2024
c5f03b0
./runner --run test/Base_Tests/ ^Text succeeds
JaroslavTulach May 20, 2024
e08f747
Don't call safepoint() on Espresso, it is not needed
JaroslavTulach May 20, 2024
8d1d8af
Skip test/Base_Tests on Espresso, as we don't have regex engine right…
JaroslavTulach May 20, 2024
1be75e6
Reverting Environment_Utils.safepoint() changes
JaroslavTulach May 20, 2024
d36bc75
Merge remote-tracking branch 'origin/develop' into wip/jtulach/EnsoLi…
JaroslavTulach May 20, 2024
5a1743a
Text - general group
JaroslavTulach May 20, 2024
e8a3325
Merge remote-tracking branch 'origin/develop' into wip/jtulach/EnsoLi…
JaroslavTulach May 21, 2024
3db825c
CompareException is imported by Statistics
JaroslavTulach May 21, 2024
d22357c
Merge remote-tracking branch 'origin/develop' into wip/jtulach/EnsoLi…
JaroslavTulach May 22, 2024
aaa5e9e
Moving all extra polyglot java import into a single file
JaroslavTulach May 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -2433,14 +2433,15 @@ lazy val `engine-runner` = project
// "-g",
// "-H:+SourceLevelDebug",
// "-H:-DeleteLocalSymbols",
// you may need to set smallJdk := None to use following flags:
// "--trace-class-initialization=org.enso.syntax2.Parser",
"-Dnic=nic"
),
mainClass = Some("org.enso.runner.Main"),
initializeAtRuntime = Seq(
"org.jline.nativ.JLineLibrary",
"org.jline.terminal.impl.jna",
"io.methvin.watchservice.jna.CarbonAPI",
"org.enso.syntax2.Parser",
"zio.internal.ZScheduler$$anon$4",
"org.enso.runner.Main$",
"sun.awt",
Expand Down Expand Up @@ -2474,6 +2475,7 @@ lazy val `engine-runner` = project
.dependsOn(`library-manager`)
.dependsOn(`language-server`)
.dependsOn(`edition-updater`)
.dependsOn(`runtime-parser`)
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved
.dependsOn(`logging-service`)
.dependsOn(`logging-service-logback` % Runtime)
.dependsOn(`polyglot-api`)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package org.enso.runner;

import static scala.jdk.javaapi.CollectionConverters.asJava;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashSet;
import java.util.TreeSet;
import org.enso.compiler.core.ir.module.scope.imports.Polyglot;
import org.enso.pkg.PackageManager$;
import org.graalvm.nativeimage.hosted.Feature;
import org.graalvm.nativeimage.hosted.RuntimeReflection;

public final class EnsoLibraryFeature implements Feature {
@Override
public void beforeAnalysis(BeforeAnalysisAccess access) {
var libs = new LinkedHashSet<Path>();
for (var p : access.getApplicationClassPath()) {
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved
var p1 = p.getParent();
if (p1 != null && p1.getFileName().toString().equals("java")) {
var p2 = p1.getParent();
if (p2 != null
&& p2.getFileName().toString().equals("polyglot")
&& p2.getParent() != null) {
libs.add(p2.getParent());
}
}
}

var classes = new TreeSet<String>();
try (var parser = new org.enso.compiler.core.EnsoParser()) {
for (var p : libs) {
var result = PackageManager$.MODULE$.Default().loadPackage(p.toFile());
if (result.isSuccess()) {
var pkg = result.get();
for (var src : pkg.listSourcesJava()) {
var code = Files.readString(src.file().toPath());
var ir = parser.compile(code);
for (var imp : asJava(ir.imports())) {
if (imp instanceof Polyglot poly && poly.entity() instanceof Polyglot.Java entity) {
var name = new StringBuilder(entity.getJavaName());
Class<?> clazz;
for (; ; ) {
clazz = access.findClassByName(name.toString());
if (clazz != null) {
break;
}
int at = name.toString().lastIndexOf('.');
if (at < 0) {
throw new IllegalStateException("Cannot load " + entity.getJavaName());
}
name.setCharAt(at, '$');
}
classes.add(clazz.getName());
RuntimeReflection.register(clazz);
RuntimeReflection.register(clazz.getConstructors());
RuntimeReflection.register(clazz.getMethods());
RuntimeReflection.register(clazz.getFields());
RuntimeReflection.registerAllConstructors(clazz);
RuntimeReflection.registerAllFields(clazz);
RuntimeReflection.registerAllMethods(clazz);
}
}
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
throw new IllegalStateException(ex);
}
System.err.println("Summary for polyglot import java:");
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now the Standard.Base library needs 153 classes registered for reflection. Libs guys, @radeusgd, @AdRiley, @jdunkerley, @GregoryTravis - isn't that a bit too much?

Summary for polyglot import java:
  com.fasterxml.jackson.core.JsonProcessingException
  com.fasterxml.jackson.databind.JsonNode
  com.fasterxml.jackson.databind.ObjectMapper
  com.fasterxml.jackson.databind.node.ArrayNode
  com.fasterxml.jackson.databind.node.BooleanNode
  com.fasterxml.jackson.databind.node.DoubleNode
  com.fasterxml.jackson.databind.node.JsonNodeFactory
  com.fasterxml.jackson.databind.node.JsonNodeType
  com.fasterxml.jackson.databind.node.LongNode
  com.fasterxml.jackson.databind.node.NullNode
  com.fasterxml.jackson.databind.node.ObjectNode
  com.fasterxml.jackson.databind.node.TextNode
  com.ibm.icu.lang.UCharacter
  com.ibm.icu.text.BreakIterator
  com.ibm.icu.text.Normalizer2$Mode
  java.io.ByteArrayInputStream
  java.io.ByteArrayOutputStream
  java.io.File
  java.io.FileNotFoundException
  java.io.IOException
  java.io.InputStream
  java.io.OutputStream
  java.io.StringReader
  java.io.UncheckedIOException
  java.lang.ArithmeticException
  java.lang.ClassCastException
  java.lang.Double
  java.lang.Exception
  java.lang.IllegalArgumentException
  java.lang.IllegalStateException
  java.lang.IndexOutOfBoundsException
  java.lang.Integer
  java.lang.Long
  java.lang.Math
  java.lang.NullPointerException
  java.lang.NumberFormatException
  java.lang.OutOfMemoryError
  java.lang.StringBuilder
  java.lang.System
  java.lang.Throwable
  java.math.BigDecimal
  java.math.MathContext
  java.math.RoundingMode
  java.net.InetSocketAddress
  java.net.ProxySelector
  java.net.URI
  java.net.URISyntaxException
  java.net.URLEncoder
  java.net.http.HttpClient
  java.net.http.HttpRequest
  java.net.http.HttpRequest$BodyPublisher
  java.nio.charset.Charset
  java.nio.charset.UnsupportedCharsetException
  java.nio.file.AccessDeniedException
  java.nio.file.DirectoryNotEmptyException
  java.nio.file.FileAlreadyExistsException
  java.nio.file.FileSystemException
  java.nio.file.FileSystems
  java.nio.file.NoSuchFileException
  java.nio.file.NotDirectoryException
  java.nio.file.Path
  java.nio.file.StandardCopyOption
  java.nio.file.StandardOpenOption
  java.nio.file.attribute.PosixFilePermission
  java.text.DecimalFormat
  java.text.DecimalFormatSymbols
  java.text.NumberFormat
  java.text.ParseException
  java.time.DateTimeException
  java.time.DayOfWeek
  java.time.Duration
  java.time.Instant
  java.time.LocalTime
  java.time.Period
  java.time.ZoneId
  java.time.ZoneOffset
  java.time.ZonedDateTime
  java.time.format.DateTimeFormatter
  java.time.format.DateTimeFormatterBuilder
  java.time.format.SignStyle
  java.time.format.TextStyle
  java.time.temporal.ChronoField
  java.time.temporal.ChronoUnit
  java.time.temporal.IsoFields
  java.time.temporal.TemporalAdjuster
  java.time.temporal.TemporalAdjusters
  java.time.temporal.TemporalUnit
  java.util.Base64
  java.util.Locale
  java.util.Random
  java.util.UUID
  java.util.logging.Logger
  javax.net.ssl.SSLContext
  javax.xml.parsers.DocumentBuilder
  javax.xml.parsers.DocumentBuilderFactory
  javax.xml.xpath.XPathConstants
  javax.xml.xpath.XPathFactory
  org.enso.base.Array_Utils
  org.enso.base.CompareException
  org.enso.base.DryRunFileManager
  org.enso.base.Encoding_Utils
  org.enso.base.Environment_Utils
  org.enso.base.FileLineReader
  org.enso.base.Regex_Utils
  org.enso.base.Text_Utils
  org.enso.base.Time_Utils
  org.enso.base.XML_Utils
  org.enso.base.arrays.LongArrayList
  org.enso.base.encoding.ReportingStreamDecoder
  org.enso.base.encoding.ReportingStreamEncoder
  org.enso.base.enso_cloud.AuthenticationProvider
  org.enso.base.enso_cloud.CacheSettings
  org.enso.base.enso_cloud.CloudAPI
  org.enso.base.enso_cloud.CloudRequestCache
  org.enso.base.enso_cloud.DataLinkSPI
  org.enso.base.enso_cloud.EnsoSecretAccessDenied
  org.enso.base.enso_cloud.EnsoSecretHelper
  org.enso.base.enso_cloud.HideableValue
  org.enso.base.enso_cloud.audit.AuditLog
  org.enso.base.file_format.FileFormatSPI
  org.enso.base.file_system.FileSystemSPI
  org.enso.base.net.URITransformer
  org.enso.base.net.URIWithSecrets
  org.enso.base.net.http.MultipartBodyBuilder
  org.enso.base.net.http.UrlencodedBodyBuilder
  org.enso.base.numeric.ConversionResult
  org.enso.base.numeric.Decimal_Utils
  org.enso.base.polyglot.WrappedDataflowError
  org.enso.base.random.RandomInstanceHolder
  org.enso.base.random.Random_Utils
  org.enso.base.statistics.CorrelationStatistics
  org.enso.base.statistics.FitError
  org.enso.base.statistics.Rank
  org.enso.base.statistics.Regression
  org.enso.base.statistics.Statistic
  org.enso.base.text.Replacer_Cache
  org.enso.base.text.TextFoldingStrategy
  org.enso.base.time.CustomTemporalUnits
  org.enso.base.time.Date_Period_Utils
  org.enso.base.time.EnsoDateTimeFormatter
  org.enso.base.time.FormatterCache
  org.enso.base.time.FormatterCacheKey
  org.enso.base.time.FormatterKind
  org.enso.polyglot.common_utils.Core_Math_Utils
  org.graalvm.collections.Pair
  org.w3c.dom.Document
  org.w3c.dom.Element
  org.w3c.dom.Node
  org.w3c.dom.NodeList
  org.w3c.dom.Text
  org.xml.sax.InputSource
  org.xml.sax.SAXException
  org.xml.sax.SAXParseException
Registered 153 classes for reflection

Copy link
Member Author

@JaroslavTulach JaroslavTulach May 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For every registered class, I have to register all its public:

  • methods
  • constructors
  • fields

That may be too much - as not all these methods are really called by Enso code. As such, it might be better to use following pattern. Rather than calling org.w3c.dom.Element.getTagName() you could wrap the call into:

class XML_Utils {
  public static String getTagName(Element e) {
    return e.getTagName();
  }
}

that way we would concentrate and select only the needed methods and the transitive closure of needed code computed by native-image would be smaller.

If we don't create the wrapper methods then we'll be forced to polyglot java import additional classes like in 6f3bfad as the native image needs to know types of classes we want to invoke instance methods on.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

isn't that a bit too much?

Looking at the list you provided, every of these classes seems needed to provide some of the functionality of the standard library. The std-lib provides a broad variety of functionalities so it seems expected it may have some dependencies. 153 does not seem that large to me.

I guess we may want to discuss whether we should split off some of these dependencies away from Base into separate libraries. I guess we could indeed do so, to keep a 'lighter' "core". IF we split off, I suspect most of our base workflows will for the time being import most of the separated parts anyway, as they are expected to be available for the users out of the box. So what could be the benefit of splitting?

I expect @jdunkerley may have some opinions on whether we want to keep a 'big' Base library or make it more modularized.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

153 does not seem that large to me.

We shouldn't be counting just the classes, but rather methods. The problem with current approach is that by using a method rich class like HttpClient we "bring in" all its methods (including for example such a megamorphic methods like equals, hashCode and toString), while we are probably using just a few. The

class XML_Utils {
  public static String getTagName(Element e) {
    return e.getTagName();
  }
}

allows us to select only the needed methods and the transitive closure of needed code computed by native-image would be smaller. Hence I'd like you to use this static method wrapper where possible for Standard libraries to be eligible for native image compilation.

Right now the ./runner has 418MB (with Graal.js and GraalPython compiled in). By carefully selecting what goes into the image we might get to 410MB or maybe even below 400MB.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now the ./runner has 418MB (with Graal.js and GraalPython compiled in). By carefully selecting what goes into the image we might get to 410MB or maybe even below 400MB.

That's like 5%? Seems like a lot of hassle and maintenance for reducing the image by that amount.

Copy link
Member Author

@JaroslavTulach JaroslavTulach May 22, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's like 5%?

Yes, it is not going to be much.

Seems like a lot of hassle and maintenance for reducing the image by that amount.

On contrary, I see that (e.g. exactly specifying what methods are used) as a good coding and architecture practice regardless of the expected gain in the size of native executable. Way more responsible than importing everything and hoping someone makes the result efficient.

If this native compilation of libraries gets to production, we should make sure we squeeze Standard libraries to make them as much effective as possible.

for (var className : classes) {
System.err.println(" " + className);
}
System.err.println("Registered " + classes.size() + " classes for reflection");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Args=--features=org.enso.runner.EnsoLibraryFeature
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
Expand Down Expand Up @@ -522,34 +521,35 @@ public boolean isColorTerminalOutput() {
*/
@TruffleBoundary
public Object lookupJavaClass(String className) {
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved
var items = Arrays.asList(className.split("\\."));
var binaryName = new StringBuilder(className);
var collectedExceptions = new ArrayList<Exception>();
for (int i = items.size() - 1; i >= 0; i--) {
String pkgName = String.join(".", items.subList(0, i));
String curClassName = items.get(i);
List<String> nestedClassPart =
i < items.size() - 1 ? items.subList(i + 1, items.size()) : List.of();
for (; ; ) {
var fqn = binaryName.toString();
try {
var hostSymbol = lookupHostSymbol(pkgName, curClassName);
if (nestedClassPart.isEmpty()) {
var hostSymbol = lookupHostSymbol(fqn);
if (hostSymbol != null) {
return hostSymbol;
} else {
var fullInnerClassName = curClassName + "$" + String.join("$", nestedClassPart);
return lookupHostSymbol(pkgName, fullInnerClassName);
}
} catch (ClassNotFoundException | RuntimeException | InteropException ex) {
collectedExceptions.add(ex);
}
var at = fqn.lastIndexOf('.');
if (at < 0) {
break;
}
binaryName.setCharAt(at, '$');
}
var level = Level.WARNING;
for (var ex : collectedExceptions) {
logger.log(Level.WARNING, null, ex);
logger.log(level, ex.getMessage());
level = Level.FINE;
logger.log(Level.FINE, null, ex);
}
return null;
}

private Object lookupHostSymbol(String pkgName, String curClassName)
private Object lookupHostSymbol(String fqn)
throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException {
var fqn = pkgName + "." + curClassName;
try {
if (findGuestJava() == null) {
return environment.asHostSymbol(hostClassLoader.loadClass(fqn));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import java.nio.charset.StandardCharsets;

public final class Parser implements AutoCloseable {
static {
private static void initializeLibraries() {
String os = System.getProperty("os.name");
String name;
if (os.startsWith("Mac")) {
Expand Down Expand Up @@ -77,6 +77,7 @@ private Parser(long stateIn) {
static native long getUuidLow(long metadata, long codeOffset, long codeLength);

public static Parser create() {
initializeLibraries();
var state = allocState();
return new Parser(state);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,7 @@
class AuditLogAPI {
private static final Logger logger = Logger.getLogger(AuditLogAPI.class.getName());
public static AuditLogAPI INSTANCE = new AuditLogAPI();
private final HttpClient httpClient =
HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build();
private HttpClient httpClient;
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved
private final ExecutorService executorService;

private AuditLogAPI() {
Expand Down Expand Up @@ -70,6 +69,9 @@ private HttpRequest buildRequest(LogMessage message) {
private void sendLogRequest(HttpRequest request, int retryCount) throws RequestFailureException {
try {
try {
if (httpClient == null) {
httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build();
}
HttpResponse<String> response =
httpClient.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
Expand Down
Loading