[GR-20846] Add native-image --target-info and additional image string…

…s for diagnosis.

PullRequest: graal/5354

substratevm/mx.substratevm/mx_substratevm.py

@@ -914,7 +914,7 @@ def _helloworld(native_image, javac_command, path, build_only, args):
     for key, value in javaProperties.items():
         args.append("-D" + key + "=" + value)
 
-    native_image(["-H:Path=" + path, '-H:+VerifyNamingConventions', '-cp', path, 'HelloWorld'] + args)
+    native_image(["--native-image-info", "-H:Path=" + path, '-H:+VerifyNamingConventions', '-cp', path, 'HelloWorld'] + args)
 
     if not build_only:
         expected_output = [output + os.linesep]

substratevm/src/com.oracle.svm.core.graal.llvm/src/com/oracle/svm/core/graal/llvm/LLVMNativeImageCodeCache.java

@@ -622,10 +622,10 @@ protected void defineMethodSymbol(String name, boolean global, Element section,
     }
 
     @Override
-    public String[] getCCInputFiles(Path tempDirectory, String imageName) {
-        String bitcodeFileName = getLinkedPath().toString();
-        String relocatableFileName = tempDirectory.resolve(imageName + ObjectFile.getFilenameSuffix()).toString();
-        return new String[]{relocatableFileName, bitcodeFileName};
+    public Path[] getCCInputFiles(Path tempDirectory, String imageName) {
+        Path bitcodeFileName = getLinkedPath();
+        Path relocatableFileName = tempDirectory.resolve(imageName + ObjectFile.getFilenameSuffix());
+        return new Path[]{relocatableFileName, bitcodeFileName};
     }
 
     @Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/LinkerInvocation.java

@@ -29,11 +29,11 @@
 
 public interface LinkerInvocation {
 
-    List<String> getInputFiles();
+    List<Path> getInputFiles();
 
-    void addInputFile(String filename);
+    void addInputFile(Path filename);
 
-    void addInputFile(int index, String filename);
+    void addInputFile(int index, Path filename);
 
     List<String> getLibPaths();
 
@@ -57,10 +57,6 @@ public interface LinkerInvocation {
 
     void addLinkedLibrary(int index, String libname);
 
-    String getCompilerCommand();
-
-    void setCompilerCommand(String command);
-
     List<String> getCommand();
 
     void addAdditionalPreOption(String option);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java

@@ -24,6 +24,8 @@
  */
 package com.oracle.svm.core;
 
+import static org.graalvm.compiler.options.OptionType.User;
+
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.function.Predicate;
@@ -392,11 +394,17 @@ public static Predicate<String> makeFilter(String[] definedFilters) {
     @Option(help = "Fold SecurityManager getter.", stability = OptionStability.EXPERIMENTAL, type = OptionType.Expert) //
     public static final HostedOptionKey<Boolean> FoldSecurityManagerGetter = new HostedOptionKey<>(true);
 
+    @APIOption(name = "native-compiler-path")//
     @Option(help = "Provide custom path to C compiler used for query code compilation and linking.", type = OptionType.User)//
     public static final HostedOptionKey<String> CCompilerPath = new HostedOptionKey<>(null);
+    @APIOption(name = "native-compiler-options")//
     @Option(help = "Provide custom C compiler option used for query code compilation.", type = OptionType.User)//
     public static final HostedOptionKey<String[]> CCompilerOption = new HostedOptionKey<>(new String[0]);
 
+    @APIOption(name = "native-image-info")//
+    @Option(help = "Show native-toolchain information and image-build settings", type = User)//
+    public static final HostedOptionKey<Boolean> DumpTargetInfo = new HostedOptionKey<>(false);
+
     @Option(help = "Provide a path for the libmusl bundle so the resulting object file is linked against it.", type = OptionType.Expert)//
     public static final HostedOptionKey<String> UseMuslC = new HostedOptionKey<>(null);
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VM.java

@@ -25,8 +25,8 @@
 package com.oracle.svm.core;
 
 import java.io.CharConversionException;
-import java.nio.ByteBuffer;
 
+import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.type.CCharPointer;
@@ -38,14 +38,17 @@
 import com.oracle.svm.core.util.VMError;
 
 public final class VM {
-    private static final String valueSeparator = "\t";
+    @Platforms(Platform.HOSTED_ONLY.class) //
+    public static final String valueSeparator = "=";
+    @Platforms(Platform.HOSTED_ONLY.class) //
     private static final String versionValue = getVersionValue();
 
     @Platforms(Platform.HOSTED_ONLY.class)
     private static String getVersionValue() {
         String version = System.getProperty("org.graalvm.version");
         VMError.guarantee(version != null);
         version = VM.class.getName() + valueSeparator + "GraalVM " + version;
+        version += " Java " + JavaVersionUtil.JAVA_SPEC;
         String config = System.getProperty("org.graalvm.config", "");
         if (!config.isEmpty()) {
             version += " " + config;
@@ -54,15 +57,16 @@ private static String getVersionValue() {
     }
 
     private static final String VERSION_INFO_SYMBOL_NAME = "__svm_version_info";
-
     private static final CGlobalData<CCharPointer> VERSION_INFO = CGlobalDataFactory.createCString(versionValue, VERSION_INFO_SYMBOL_NAME);
 
+    private static final int versionValueHash = versionValue.hashCode();
+
     public static String getVersion() {
         try {
             CCharPointer versionInfoBytes = VERSION_INFO.get();
-            ByteBuffer buffer = CTypeConversion.asByteBuffer(versionInfoBytes, Math.toIntExact(SubstrateUtil.strlen(versionInfoBytes).rawValue()));
-            String version = Utf8.utf8ToString(true, buffer);
-            VMError.guarantee(versionValue.equals(version), "Version info mismatch: " + VERSION_INFO_SYMBOL_NAME + " contains " + version + " (expected " + versionValue + ")");
+            String version = Utf8.utf8ToString(true, CTypeConversion.asByteBuffer(versionInfoBytes, Math.toIntExact(SubstrateUtil.strlen(versionInfoBytes).rawValue())));
+            VMError.guarantee(version.hashCode() == versionValueHash,
+                            "HashCode mismatch for " + VERSION_INFO_SYMBOL_NAME + ": actual " + version.hashCode() + " (expected " + versionValueHash + ")");
             return SubstrateUtil.split(version, valueSeparator)[1];
         } catch (CharConversionException ignore) {
             throw VMError.shouldNotReachHere("Invalid version info in " + VERSION_INFO_SYMBOL_NAME);

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java

@@ -223,6 +223,7 @@
 import com.oracle.svm.hosted.c.GraalAccess;
 import com.oracle.svm.hosted.c.NativeLibraries;
 import com.oracle.svm.hosted.c.SizeOfSupportImpl;
+import com.oracle.svm.hosted.c.codegen.CCompilerInvoker;
 import com.oracle.svm.hosted.cenum.CEnumCallWrapperSubstitutionProcessor;
 import com.oracle.svm.hosted.classinitialization.ClassInitializationFeature;
 import com.oracle.svm.hosted.classinitialization.ClassInitializationSupport;
@@ -724,6 +725,13 @@ private boolean runPointsToAnalysis(String imageName, OptionValues options, Debu
                     }
                 }
 
+                /*
+                 * Libraries defined via @CLibrary annotations are added at the end of the list of
+                 * libraries so that the written object file AND the static JDK libraries can depend
+                 * on them.
+                 */
+                nativeLibraries.processAnnotated();
+
                 AfterAnalysisAccessImpl postConfig = new AfterAnalysisAccessImpl(featureHandler, loader, bigbang, debug);
                 featureHandler.forEachFeature(feature -> feature.afterAnalysis(postConfig));
 
@@ -837,6 +845,11 @@ private void setupNativeImage(String imageName, OptionValues options, Map<Method
                 HostedSnippetReflectionProvider aSnippetReflection = new HostedSnippetReflectionProvider((SVMHost) aUniverse.hostVM(), aWordTypes);
 
                 prepareLibC();
+
+                CCompilerInvoker compilerInvoker = CCompilerInvoker.create(tempDirectory());
+                compilerInvoker.verifyCompiler();
+                ImageSingletons.add(CCompilerInvoker.class, compilerInvoker);
+
                 nativeLibraries = setupNativeLibraries(imageName, aConstantReflection, aMetaAccess, aSnippetReflection, cEnumProcessor, classInitializationSupport);
 
                 ForeignCallsProvider aForeignCalls = new SubstrateForeignCallsProvider();

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java

@@ -24,22 +24,72 @@
  */
 package com.oracle.svm.hosted;
 
+import java.lang.reflect.Field;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.stream.Collectors;
+
+import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.hosted.Feature;
 
+import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.VM;
 import com.oracle.svm.core.annotate.AutomaticFeature;
+import com.oracle.svm.core.c.CGlobalDataFactory;
+import com.oracle.svm.core.c.libc.LibCBase;
 import com.oracle.svm.core.util.VMError;
-import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl;
+import com.oracle.svm.hosted.c.CGlobalDataFeature;
+import com.oracle.svm.hosted.c.NativeLibraries;
+import com.oracle.svm.hosted.c.codegen.CCompilerInvoker;
 
 @AutomaticFeature
 public class VMFeature implements Feature {
+
+    private NativeLibraries nativeLibraries;
+
     @Override
     public void beforeAnalysis(BeforeAnalysisAccess a) {
-        BeforeAnalysisAccessImpl access = (BeforeAnalysisAccessImpl) a;
+        if (SubstrateOptions.DumpTargetInfo.getValue()) {
+            System.out.println("# Building image for target platform: " + ImageSingletons.lookup(Platform.class).getClass().getName());
+            System.out.println("# Using native toolchain:");
+            ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.dump(x -> System.out.println("#   " + x));
+            System.out.println("# Using CLibrary: " + ImageSingletons.lookup(LibCBase.class).getClass().getName());
+        }
+
+        FeatureImpl.BeforeAnalysisAccessImpl access = (FeatureImpl.BeforeAnalysisAccessImpl) a;
+        String fieldName = "VERSION_INFO";
         try {
-            access.registerAsRead(access.getMetaAccess().lookupJavaField(VM.class.getDeclaredField("VERSION_INFO")));
+            Field declaredField = VM.class.getDeclaredField(fieldName);
+            access.registerAsRead(access.getMetaAccess().lookupJavaField(declaredField));
         } catch (NoSuchFieldException e) {
-            VMError.shouldNotReachHere();
+            VMError.shouldNotReachHere(VM.class.getName() + " should have field " + fieldName);
+        }
+
+        nativeLibraries = access.getNativeLibraries();
+    }
+
+    @Override
+    public void afterAnalysis(AfterAnalysisAccess access) {
+        addCGlobalDataString("Target.Platform", ImageSingletons.lookup(Platform.class).getClass().getName());
+        addCGlobalDataString("Target.LibC", ImageSingletons.lookup(LibCBase.class).getClass().getName());
+
+        addCGlobalDataString("Target.Libraries", String.join("|", nativeLibraries.getLibraries()));
+        addCGlobalDataString("Target.StaticLibraries", nativeLibraries.getStaticLibraries().stream()
+                        .map(Path::getFileName).map(Path::toString).collect(Collectors.joining("|")));
+        addCGlobalDataString("Target.CCompiler", ImageSingletons.lookup(CCompilerInvoker.class).compilerInfo.toString());
+
+        if (SubstrateOptions.DumpTargetInfo.getValue()) {
+            System.out.println("# Static libraries:");
+            Path current = Paths.get(".").toAbsolutePath();
+            nativeLibraries.getStaticLibraries().stream().map(current::relativize).map(Path::toString).forEach(x -> System.out.println("#   " + x));
+            System.out.println("# Other libraries: " + String.join(",", nativeLibraries.getLibraries()));
         }
     }
+
+    private static void addCGlobalDataString(String infoType, String content) {
+        String data = VM.class.getName() + "." + infoType + VM.valueSeparator + content;
+        String symbolName = "__svm_vm_" + infoType.toLowerCase().replace(".", "_");
+        CGlobalDataFeature.singleton().registerAsAccessedOrGet(CGlobalDataFactory.createCString(data, symbolName));
+    }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/CAnnotationProcessor.java

@@ -30,13 +30,12 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.oracle.svm.core.OS;
 import com.oracle.svm.core.util.InterruptImageBuilding;
 import com.oracle.svm.core.util.UserError;
+import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.hosted.c.codegen.CCompilerInvoker;
 import com.oracle.svm.hosted.c.codegen.QueryCodeWriter;
 import com.oracle.svm.hosted.c.info.InfoTreeBuilder;
@@ -48,16 +47,21 @@
 /**
  * Processes native library information for one C Library header file (one { NativeCodeContext }).
  */
-public class CAnnotationProcessor extends CCompilerInvoker {
+public class CAnnotationProcessor {
 
     private final NativeCodeContext codeCtx;
+    private final NativeLibraries nativeLibs;
+    private final CCompilerInvoker compilerInvoker;
+    private final Path tempDirectory;
 
     private NativeCodeInfo codeInfo;
     private QueryCodeWriter writer;
 
-    public CAnnotationProcessor(NativeLibraries nativeLibs, NativeCodeContext codeCtx, Path tempDirectory) {
-        super(nativeLibs, tempDirectory);
+    public CAnnotationProcessor(NativeLibraries nativeLibs, NativeCodeContext codeCtx, CCompilerInvoker compilerInvoker) {
+        this.nativeLibs = nativeLibs;
         this.codeCtx = codeCtx;
+        this.compilerInvoker = compilerInvoker;
+        this.tempDirectory = compilerInvoker.tempDirectory;
     }
 
     public NativeCodeInfo process(CAnnotationProcessorCache cache) {
@@ -98,16 +102,15 @@ public NativeCodeInfo process(CAnnotationProcessorCache cache) {
     }
 
     private void makeQuery(CAnnotationProcessorCache cache, String binaryName) {
-        List<String> command = new ArrayList<>();
-        command.add(binaryName);
         Process printingProcess = null;
         try {
-            printingProcess = startCommand(command);
-            InputStream is = printingProcess.getInputStream();
-            List<String> lines = QueryResultParser.parse(nativeLibs, codeInfo, is);
-            is.close();
-            if (CAnnotationProcessorCache.Options.NewCAPCache.getValue()) {
-                cache.put(codeInfo, lines);
+            ProcessBuilder pb = new ProcessBuilder().command(binaryName).directory(tempDirectory.toFile());
+            printingProcess = pb.start();
+            try (InputStream is = printingProcess.getInputStream()) {
+                List<String> lines = QueryResultParser.parse(nativeLibs, codeInfo, is);
+                if (CAnnotationProcessorCache.Options.NewCAPCache.getValue()) {
+                    cache.put(codeInfo, lines);
+                }
             }
             printingProcess.waitFor();
         } catch (IOException ex) {
@@ -122,17 +125,18 @@ private void makeQuery(CAnnotationProcessorCache cache, String binaryName) {
     }
 
     private Path compileQueryCode(Path queryFile) {
-        /* remove the '.c' or '.cpp' from the end to get the binary name */
-        String binaryName = queryFile.toString().substring(0, queryFile.toString().lastIndexOf("."));
-        if (OS.getCurrent() == OS.WINDOWS) {
-            binaryName = binaryName + ".exe";
+        /* replace the '.c' or '.cpp' from the end to get the binary name */
+        Path fileNamePath = queryFile.getFileName();
+        if (fileNamePath == null) {
+            throw VMError.shouldNotReachHere(queryFile + " invalid queryFile");
         }
-        Path binary = Paths.get(binaryName);
-        return compileAndParseError(codeCtx.getDirectives().getOptions(), queryFile.normalize(), binary.normalize());
+        String fileName = fileNamePath.toString();
+        Path binary = queryFile.resolveSibling(compilerInvoker.asExecutableName(fileName.substring(0, fileName.lastIndexOf("."))));
+        compilerInvoker.compileAndParseError(codeCtx.getDirectives().getOptions(), queryFile, binary, this::reportCompilerError);
+        return binary;
     }
 
-    @Override
-    protected void reportCompilerError(Path queryFile, String line) {
+    protected void reportCompilerError(ProcessBuilder current, Path queryFile, String line) {
         for (String header : codeCtx.getDirectives().getHeaderFiles()) {
             if (line.contains(header.substring(1, header.length() - 1) + ": No such file or directory")) {
                 UserError.abort("Basic header file missing (" + header + "). Make sure headers are available on your system.");
@@ -160,6 +164,12 @@ protected void reportCompilerError(Path queryFile, String line) {
             }
         }
 
-        nativeLibs.getErrors().add(new CInterfaceError("Error compiling query code (in " + queryFile + "). Compiler command " + lastExecutedCommand() + " output included error: " + line, elements));
+        CInterfaceError error = new CInterfaceError(
+                        String.format("Error compiling query code (in %s). Compiler command '%s' output included error: %s",
+                                        queryFile,
+                                        String.join(" ", current.command()),
+                                        line),
+                        elements);
+        nativeLibs.getErrors().add(error);
     }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java

@@ -67,6 +67,7 @@
 import com.oracle.svm.core.option.OptionUtils;
 import com.oracle.svm.core.util.UserError;
 import com.oracle.svm.hosted.NativeImageOptions;
+import com.oracle.svm.hosted.c.codegen.CCompilerInvoker;
 import com.oracle.svm.hosted.c.info.ElementInfo;
 import com.oracle.svm.hosted.classinitialization.ClassInitializationSupport;
 import com.oracle.svm.util.ReflectionUtil;
@@ -383,8 +384,7 @@ public void finish() {
             if (context.isInConfiguration()) {
                 libraries.addAll(context.getDirectives().getLibraries());
                 libraryPaths.addAll(context.getDirectives().getLibraryPaths());
-
-                new CAnnotationProcessor(this, context, tempDirectory).process(cache);
+                new CAnnotationProcessor(this, context, ImageSingletons.lookup(CCompilerInvoker.class)).process(cache);
             }
         }
     }