Merge pull request #4430 from JohanEngelen/sema_plugin

Add support for Semantic Analysis plugins

.github/actions/3-build-cross/action.yml

@@ -145,4 +145,4 @@ runs:
           ${{ inputs.cmake_flags }}
           ${{ inputs.with_pgo == 'true' && '-DDFLAGS_LDC=-fprofile-use=../pgo-ldc/merged.profdata' || '' }}
           ${{ env.CROSS_CMAKE_FLAGS }}
-        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-profdata ldc-prune-cache timetrace2txt
+        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-build-plugin ldc-profdata ldc-prune-cache timetrace2txt

.github/actions/5-install/action.yml

@@ -22,6 +22,7 @@ runs:
         else
           mkdir -p install/bin
           cp build-cross/bin/{ldc2,ldmd2,ldc-build-runtime,ldc-profdata,ldc-prune-cache,timetrace2txt} install/bin/
+          cp build-cross/bin/ldc-build-plugin install/bin/ || true
           cp -R build-cross-libs/lib install/
           cp build-cross/lib/{libldc_rt.*,libLTO-ldc.dylib,LLVMgold-ldc.so} install/lib/ || true
           mkdir install/etc

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 #### Big news
 - Frontend, druntime and Phobos are at version [2.103.1](https://dlang.org/changelog/2.103.0.html), incl. new command-line option `-verror-supplements`. (#4345)
+- The `--plugin` commandline option now also accepts semantic analysis plugins. Semantic analysis plugins are recognized by exporting the symbol: `extern(C) void runSemanticAnalysis(Module m)`. The plugin's `runSemanticAnalysis` function is called for each module, after all other semantic analysis steps (also after DCompute SemA), just before object codegen. (#4430)
+- New tool `ldc-build-plugin` that helps compiling user plugins. It downloads the correct LDC source version (if its not already available), and calls LDC with the correct commandline flags to build a plugin. (#4430)
 - New commandline option `-femit-local-var-lifetime` that enables variable lifetime (scope) annotation to LLVM IR codegen. Lifetime annotation enables stack memory reuse for local variables with non-overlapping scope. (#4395)
 - C files are now automatically preprocessed using the external C compiler (configurable via `-gcc` or the `CC` environment variable, and `-Xcc` for extra flags). Extra preprocessor flags (e.g., include dirs and manual defines) can be added via new command-line option `-P`. (#4417)
   - Windows: If `clang-cl.exe` is on `PATH`, it is preferred over Microsoft's `cl.exe` by default (e.g., to avoid printing the C source file name to stderr during preprocessing).

CMakeLists.txt

@@ -249,17 +249,6 @@ if(CMAKE_COMPILER_IS_GNUCXX OR (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang"))
     endif()
 endif()
 
-if(NOT WIN32 AND NOT CYGWIN)
-    # Unify symbol visibility with LLVM to silence linker warning "direct access in function X to global
-    # weak symbol Y means the weak symbol cannot be overridden at runtime. This was likely caused by
-    # different translation units being compiled with different visibility settings."
-    # See LLVM's cmake/modules/HandleLLVMOptions.cmake.
-    check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
-    if (${SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG})
-        append("-fvisibility-inlines-hidden" LDC_CXXFLAGS)
-    endif()
-endif()
-
 if(MSVC)
     # Remove flags here, for exceptions and RTTI.
     # CL.EXE complains to override flags like "/GR /GR-".
@@ -683,10 +672,26 @@ if(LDC_ENABLE_PLUGINS)
 
         if(LINKER_ACCEPTS_EXPORT_DYNAMIC_FLAG)
             set(LDC_LINKERFLAG_LIST "${LDC_LINKERFLAG_LIST};-Wl,--export-dynamic")
+        else()
+            message(WARNING "Linker does not accept --export-dynamic, user plugins may give missing symbol errors upon load")
         endif()
     endif()
 endif()
 message(STATUS "-- Building LDC with plugin support (LDC_ENABLE_PLUGINS): ${LDC_ENABLE_PLUGINS}")
+message(STATUS "-- Linking LDC with flags: ${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST}")
+
+if(NOT WIN32 AND NOT CYGWIN)
+    # Unify symbol visibility with LLVM to silence linker warning "direct access in function X to global
+    # weak symbol Y means the weak symbol cannot be overridden at runtime. This was likely caused by
+    # different translation units being compiled with different visibility settings."
+    # See LLVM's cmake/modules/HandleLLVMOptions.cmake.
+    check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
+    if (LDC_ENABLE_PLUGINS AND NOT APPLE)
+        # For plugins, we shouldn't apply this flag because it hides the inline methods of e.g. Visitor. On macOS it's OK to add.
+    elseif (${SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG})
+        append("-fvisibility-inlines-hidden" LDC_CXXFLAGS)
+    endif()
+endif()
 
 build_d_executable(
     "${LDC_EXE}"

cmake/Modules/BuildDExecutable.cmake

@@ -25,6 +25,7 @@ endmacro()
 # - DFLAGS_BASE
 # - LDC_LINK_MANUALLY
 # - D_LINKER_ARGS
+# - LDC_ENABLE_PLUGINS
 function(build_d_executable target_name output_exe d_src_files compiler_args linker_args extra_compile_deps link_deps compile_separately)
     set(dflags "${D_COMPILER_FLAGS} ${DFLAGS_BASE} ${compiler_args}")
     if(UNIX)
@@ -40,7 +41,9 @@ function(build_d_executable target_name output_exe d_src_files compiler_args lin
         # Compile all D modules to a single object.
         set(object_file ${PROJECT_BINARY_DIR}/obj/${target_name}${CMAKE_CXX_OUTPUT_EXTENSION})
         # Default to -linkonce-templates with LDMD host compiler, to speed-up optimization.
-        if("${D_COMPILER_ID}" STREQUAL "LDMD")
+        if("${target_name}" STREQUAL "ldc2" AND LDC_ENABLE_PLUGINS)
+            # For plugin support we need ldc2's symbols to be global, don't use -linkonce-templates.
+        elseif("${D_COMPILER_ID}" STREQUAL "LDMD")
             set(dflags -linkonce-templates ${dflags})
         endif()
         add_custom_command(

driver/plugins.cpp

@@ -9,6 +9,9 @@
 //
 // Implements functionality related to plugins (`-plugin=...`).
 //
+// Note: plugins can be LLVM-plugins (to be registered with the pass manager)
+// or dlang-plugins for semantic analysis.
+//
 //===----------------------------------------------------------------------===//
 
 #include "driver/plugins.h"
@@ -17,6 +20,7 @@
 
 #include "dmd/errors.h"
 #include "dmd/globals.h"
+#include "dmd/module.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DynamicLibrary.h"
@@ -36,63 +40,120 @@ cl::list<std::string> pluginFiles("plugin", cl::CommaSeparated,
                                   cl::desc("Pass plugins to load."),
                                   cl::value_desc("dynamic_library.so,lib2.so"));
 
+struct SemaPlugin {
+  llvm::sys::DynamicLibrary library;
+  void (*runSemanticAnalysis)(Module *);
+
+  SemaPlugin(const llvm::sys::DynamicLibrary &library,
+             void (*runSemanticAnalysis)(Module *))
+      : library(library), runSemanticAnalysis(runSemanticAnalysis) {}
+};
+
+llvm::SmallVector<SemaPlugin, 1> sema_plugins;
+
 } // anonymous namespace
 
-#if LDC_LLVM_VER >= 1400
+// Tries to load plugin as SemanticAnalysis. Returns true on 'success', i.e. no
+// further attempts needed.
+bool loadSemanticAnalysisPlugin(const std::string &filename) {
+  std::string errorString;
+  auto library = llvm::sys::DynamicLibrary::getPermanentLibrary(
+      filename.c_str(), &errorString);
+  if (!library.isValid()) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          errorString.c_str());
+    return true; // No success, but no need to try loading again as LLVM plugin.
+  }
 
-namespace {
-llvm::SmallVector<llvm::PassPlugin, 1> plugins;
+  // SemanticAnalysis plugins need to export the `runSemanticAnalysis` function.
+  void *runSemanticAnalysisFnPtr =
+      library.getAddressOfSymbol("runSemanticAnalysis");
+
+  // If the symbol isn't found, this is probably an LLVM plugin.
+  if (!runSemanticAnalysisFnPtr)
+    return false;
+
+  sema_plugins.emplace_back(
+      library, reinterpret_cast<void (*)(Module *)>(runSemanticAnalysisFnPtr));
+  return true;
 }
-/// Loads all plugins for the new pass manager. These plugins will need to be
-/// added When building the optimization pipeline.
-void loadAllPluginsNewPM() {
-  for (auto &filename : pluginFiles) {
-    auto plugin = llvm::PassPlugin::Load(filename);
-    if (!plugin) {
-      error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
-            llvm::toString(plugin.takeError()).c_str());
-      continue;
-    }
-    plugins.emplace_back(plugin.get());
+
+/// Loads plugin for the legacy pass manager. The static constructor of
+/// the plugin should take care of the plugins registering themself with the
+/// rest of LDC/LLVM.
+void loadLLVMPluginLegacyPM(const std::string &filename) {
+  std::string errorString;
+  if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(filename.c_str(),
+                                                        &errorString)) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          errorString.c_str());
   }
 }
-void registerAllPluginsWithPassBuilder(llvm::PassBuilder &PB) {
-  for (auto &plugin : plugins) {
-    plugin.registerPassBuilderCallbacks(PB);
+
+#if LDC_LLVM_VER >= 1400
+
+namespace {
+llvm::SmallVector<llvm::PassPlugin, 1> llvm_plugins;
+
+/// Loads plugin for the new pass manager. The plugin will need to be
+/// added explicitly when building the optimization pipeline.
+void loadLLVMPluginNewPM(const std::string &filename) {
+
+  auto plugin = llvm::PassPlugin::Load(filename);
+  if (!plugin) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          llvm::toString(plugin.takeError()).c_str());
+    return;
   }
+  llvm_plugins.emplace_back(plugin.get());
 }
 
+} // anonymous namespace
+
 #endif // LDC_LLVM_VER >= 1400
 
-/// Loads all plugins for the legacy pass manaager. The static constructor of
-/// each plugin should take care of the plugins registering themself with the
-/// rest of LDC/LLVM.
-void loadAllPluginsLegacyPM() {
+void loadLLVMPlugin(const std::string &filename) {
+#if LDC_LLVM_VER >= 1400
+  if (opts::isUsingLegacyPassManager())
+    loadLLVMPluginLegacyPM(filename);
+  else
+    loadLLVMPluginNewPM(filename);
+#else
+  loadLLVMPluginLegacyPM(filename);
+#endif
+}
+
+void loadAllPlugins() {
   for (auto &filename : pluginFiles) {
-    std::string errorString;
-    if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(filename.c_str(),
-                                                          &errorString)) {
-      error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
-            errorString.c_str());
-    }
+    // First attempt to load plugin as SemanticAnalysis plugin. If unsuccesfull,
+    // load as LLVM plugin.
+    auto success = loadSemanticAnalysisPlugin(filename);
+    if (!success)
+      loadLLVMPlugin(filename);
   }
 }
 
+void registerAllPluginsWithPassBuilder(llvm::PassBuilder &PB) {
 #if LDC_LLVM_VER >= 1400
-void loadAllPlugins() {
-  if (opts::isUsingLegacyPassManager())
-    loadAllPluginsLegacyPM();
-  else
-    loadAllPluginsNewPM();
-}
-#else
-void loadAllPlugins() { loadAllPluginsLegacyPM(); }
-void registerAllPluginsWithPassBuilder(llvm::PassBuilder &) {}
+  for (auto &plugin : llvm_plugins) {
+    plugin.registerPassBuilderCallbacks(PB);
+  }
 #endif
+}
+
+void runAllSemanticAnalysisPlugins(Module *m) {
+  for (auto &plugin : sema_plugins) {
+    assert(plugin.runSemanticAnalysis);
+    plugin.runSemanticAnalysis(m);
+  }
+}
 
 #else // #if LDC_ENABLE_PLUGINS
 
+class Module;
+
 void loadAllPlugins() {}
 void registerAllPluginsWithPassBuilder(llvm::PassBuilder &) {}
+void runAllSemanticAnalysisPlugins(Module *m) {}
 
 #endif // LDC_ENABLE_PLUGINS

gen/semantic.d

@@ -15,13 +15,20 @@ import dmd.dmodule;
 
 extern(C++) void dcomputeSemanticAnalysis(Module m);
 extern(C) int hasComputeAttr(Dsymbol m);
+extern(C++) void runAllSemanticAnalysisPlugins(Module m);
 
 extern(C++) void extraLDCSpecificSemanticAnalysis(ref Modules modules)
 {
+    // First finish DCompute SemA for all modules, before calling plugins.
     foreach(m; modules[])
     {
-        if (hasComputeAttr(m))
+        if (hasComputeAttr(m)) {
             dcomputeSemanticAnalysis(m);
+        }
+    }
+
+    foreach(m; modules[])
+    {
+        runAllSemanticAnalysisPlugins(m);
     }
-
 }

tests/CMakeLists.txt

@@ -1,6 +1,7 @@
 set( LDC2_BIN          ${PROJECT_BINARY_DIR}/bin/${LDC_EXE} )
 set( LDCPROFDATA_BIN   ${PROJECT_BINARY_DIR}/bin/ldc-profdata )
 set( LDCPRUNECACHE_BIN ${PROJECT_BINARY_DIR}/bin/${LDCPRUNECACHE_EXE} )
+set( LDCBUILDPLUGIN_BIN ${PROJECT_BINARY_DIR}/bin/${LDC_BUILD_PLUGIN_EXE} )
 set( TIMETRACE2TXT_BIN ${PROJECT_BINARY_DIR}/bin/${TIMETRACE2TXT_EXE} )
 set( LLVM_TOOLS_DIR    ${LLVM_ROOT_DIR}/bin )
 set( LDC2_BIN_DIR      ${PROJECT_BINARY_DIR}/bin )

tests/lit.site.cfg.in

@@ -17,10 +17,12 @@ OFF = False
 config.ldc2_bin            = "@LDC2_BIN@"
 config.ldcprofdata_bin     = "@LDCPROFDATA_BIN@"
 config.ldcprunecache_bin   = "@LDCPRUNECACHE_BIN@"
+config.ldcbuildplugin_bin  = "@LDCBUILDPLUGIN_BIN@"
 config.timetrace2txt_bin   = "@TIMETRACE2TXT_BIN@"
 config.ldc2_bin_dir        = "@LDC2_BIN_DIR@"
 config.ldc2_lib_dir        = "@LDC2_LIB_DIR@"
 config.ldc2_runtime_dir    = "@RUNTIME_DIR@"
+config.ldc2_source_dir     = "@PROJECT_SOURCE_DIR@"
 config.test_source_root    = "@TESTS_IR_DIR@"
 config.llvm_tools_dir      = "@LLVM_TOOLS_DIR@"
 config.llvm_version        = @LDC_LLVM_VER@
@@ -156,6 +158,7 @@ config.substitutions.append( ('%ldc', config.ldc2_bin) )
 config.substitutions.append( ('%gnu_make', config.gnu_make_bin) )
 config.substitutions.append( ('%profdata', config.ldcprofdata_bin) )
 config.substitutions.append( ('%prunecache', config.ldcprunecache_bin) )
+config.substitutions.append( ('%buildplugin', config.ldcbuildplugin_bin + " --ldcSrcDir=" + config.ldc2_source_dir ) )
 config.substitutions.append( ('%timetrace2txt', config.timetrace2txt_bin) )
 config.substitutions.append( ('%llvm-spirv', os.path.join(config.llvm_tools_dir, 'llvm-spirv')) )
 config.substitutions.append( ('%llc', os.path.join(config.llvm_tools_dir, 'llc')) )

tests/plugins/basic_sema_plugin.d

@@ -0,0 +1,26 @@
+// REQUIRES: Plugins
+
+// For some reason this test fails with missing symbol linking issues (or crash) with macOS on Intel x86 (but not for all CI testers...)
+// UNSUPPORTED: Darwin && host_X86
+
+// RUN: split-file %s %t --leading-lines
+// RUN: %buildplugin %t/plugin.d -of=%t/plugin%so --buildDir=%t/build
+// RUN: %ldc -wi -c -o- --plugin=%t/plugin%so %t/testcase.d 2>&1 | FileCheck %t/testcase.d
+
+//--- plugin.d
+import dmd.dmodule : Module;
+import dmd.errors;
+import dmd.location;
+
+extern(C) void runSemanticAnalysis(Module m) {
+    if (m.md) {
+        warning(m.md.loc, "It works!");
+    }
+}
+
+//--- testcase.d
+// CHECK: testcase.d([[@LINE+1]]): Warning: It works!
+module testcase;
+int testfunction(int i) {
+    return i * 2;
+}

tests/plugins/lit.local.cfg

@@ -1,8 +1,28 @@
+import lit.formats
+import lit.util
 import os
+import sys
 import platform
+import string
 import re
+import subprocess
+import glob
 
 if (config.plugins_supported):
     config.available_features.add('Plugins')
     config.environment['LLVM_CONFIG'] = os.path.join(config.llvm_tools_dir, 'llvm-config')
     config.environment['LLVM_VERSION'] = str(config.llvm_version)
+
+    # Set feature that tells us that the just-built LDC is ABI compatible with the host D compiler
+    # For our tets, the required ABI compatibility seems OK since at least LDC 1.30.
+    # If the compiler is built not by LDC but another compiler, then assume the ABI to be incompatible.
+    command = [config.ldc2_bin, '--version']
+    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    text1 = p.stdout.readline() # Ex.: "LDC - the LLVM D compiler (1.33.0-git-716f627)"
+    text2 = p.stdout.readline() # Ex.: "  based on DMD v2.103.1 and LLVM 14.0.0"
+    text3 = p.stdout.readline() # Ex.: "  built with LDC - the LLVM D compiler (1.33.0-beta2)"
+    host_version = re.compile('  built with LDC.* \(1\.([0-9]+).*\)').match(text3)
+    if (host_version and int(host_version.group(1)) >= 30): # 30 = LDC 1.30
+        config.available_features.add('ABI_compatible_with_host_D')
+
+