Add th_deps attribute for compile-time dependencies

Implements the proposal in tweag#1382. A new `th_deps` attribute
is added to `haskell_(library|binary|test)` that is similar
to `deps` but makes only the libraries in the formaer available
at compile-time.

As discussed in the issue, this reduces the number of
dependencies for the build action, creating more
opportunities for parallel work.

In addition, adding all cc deps at compile time was
preventing some configurations from working: ghc's dynamic linker
apparently tries to load the .so files in the order in which they
are specified so if a.so declares symbols referenced by b.so,
a.so needs to be given first. We don't seem to control the order
in which these are passed from the rules, though. E.g., without
`th_deps` defaulting to `[]` I'm unable to package llvm-hs
to use a bazel-built llvm, since the former uses TH and FFI
(but no FFI during TH) and the latter splits things in several
several packages.

haskell/BUILD.bazel

@@ -176,6 +176,13 @@ config_setting(
     },
 )
 
+config_setting(
+    name = "use_deps_on_empty_th_deps",
+    flag_values = {
+        "@rules_haskell//flags:incompatible_use_deps_on_empty_th_deps": "true"
+    },
+)
+
 haskell_toolchain_info(
     name = "toolchain_info",
 )

haskell/cc.bzl

@@ -33,6 +33,8 @@ CcInteropInfo = provider(
         "cc_libraries_info": "HaskellCcLibrariesInfo",
         "cc_libraries": "depset, C libraries from direct linking dependencies.",
         "transitive_libraries": "depset, C and Haskell libraries from transitive linking dependencies.",
+        "cc_th_libraries": "depset, C libraries from TemplateHaskell dependencies.",
+        "transitive_th_libraries": "depset, C and Haskell libraries from transitive TemplateHaskell dependencies.",
         "plugin_libraries": "depset, C and Haskell libraries from transitive plugin dependencies.",
         "setup_libraries": "depset, C and Haskell libraries from Cabal setup dependencies.",
     },
@@ -51,6 +53,15 @@ def cc_interop_info(ctx, override_cc_toolchain = None):
     """
     ccs = [dep[CcInfo] for dep in ctx.attr.deps if CcInfo in dep and HaskellInfo not in dep]
 
+    if hasattr(ctx.attr, "th_deps"):
+        th_deps = ctx.attr.th_deps
+        if not th_deps and ctx.attr.use_deps_on_empty_th_deps:
+            th_deps = ctx.attr.deps
+    else:
+        th_deps = ctx.attr.deps
+
+    th_ccs = [dep[CcInfo] for dep in th_deps if CcInfo in dep and HaskellInfo not in dep]
+
     hdrs = []
     include_args = []
     cpp_flags = []
@@ -151,7 +162,7 @@ def cc_interop_info(ctx, override_cc_toolchain = None):
     env["CC_WRAPPER_CPU"] = cc_toolchain.cpu
 
     cc_libraries_info = deps_HaskellCcLibrariesInfo(
-        ctx.attr.deps + getattr(ctx.attr, "plugins", []) + getattr(ctx.attr, "setup_deps", []),
+        ctx.attr.deps + th_deps + getattr(ctx.attr, "plugins", []) + getattr(ctx.attr, "setup_deps", []),
     )
     return CcInteropInfo(
         tools = struct(**tools),
@@ -168,11 +179,21 @@ def cc_interop_info(ctx, override_cc_toolchain = None):
         linker_flags = linker_flags,
         cc_libraries_info = cc_libraries_info,
         cc_libraries = get_cc_libraries(cc_libraries_info, [lib for li in cc_common.merge_cc_infos(cc_infos = ccs).linking_context.linker_inputs.to_list() for lib in li.libraries]),
+        cc_th_libraries = get_cc_libraries(cc_libraries_info, [
+            lib
+            for li in cc_common.merge_cc_infos(cc_infos = th_ccs).linking_context.linker_inputs.to_list()
+            for lib in li.libraries
+        ]),
         transitive_libraries = [lib for li in cc_common.merge_cc_infos(cc_infos = [
             dep[CcInfo]
             for dep in ctx.attr.deps
             if CcInfo in dep
         ]).linking_context.linker_inputs.to_list() for lib in li.libraries],
+        transitive_th_libraries = [lib for li in cc_common.merge_cc_infos(cc_infos = [
+            dep[CcInfo]
+            for dep in th_deps
+            if CcInfo in dep
+        ]).linking_context.linker_inputs.to_list() for lib in li.libraries],
         plugin_libraries = [lib for li in cc_common.merge_cc_infos(cc_infos = [
             dep[CcInfo]
             for plugin in getattr(ctx.attr, "plugins", [])

haskell/defs.bzl

@@ -61,6 +61,9 @@ _haskell_common_attrs = {
     "modules": attr.label_list(
         providers = [HaskellModuleInfo],
     ),
+    "th_deps": attr.label_list(
+        aspects = [haskell_cc_libraries_aspect],
+    ),
     # a proxy for ctx.label so that the transition can access it
     "label_string": attr.string(),
     "data": attr.label_list(
@@ -100,6 +103,10 @@ _haskell_common_attrs = {
         cfg = "host",
         default = Label("@rules_haskell//haskell:ghc_wrapper"),
     ),
+    "use_deps_on_empty_th_deps": attr.bool(
+        # Should be provided by the wrappers, not the user
+        mandatory=True,
+    ),
     "worker": attr.label(
         default = None,
         executable = True,
@@ -233,6 +240,10 @@ def _haskell_worker_wrapper(rule_type, **kwargs):
     kwargs.pop("compiler_flags")
 
     defaults = dict(
+        use_deps_on_empty_th_deps = select({
+            "@rules_haskell//haskell:use_deps_on_empty_th_deps": True,
+            "//conditions:default": False,
+        }),
         worker = select({
             "@rules_haskell//haskell:use_worker": Label("@rules_haskell//tools/worker:bin"),
             "//conditions:default": None,
@@ -254,6 +265,7 @@ def haskell_binary(
         extra_srcs = [],
         deps = [],
         narrowed_deps = [],
+        th_deps = [],
         data = [],
         compiler_flags = [],
         ghcopts = [],
@@ -322,6 +334,8 @@ def haskell_binary(
           Note: This attribute is experimental and not ready for production, yet.
       modules: List of extra haskell_module() dependencies to be linked into this binary.
           Note: This attribute is experimental and not ready for production, yet.
+      th_deps: List of libraries needed to run TemplateHaskell code. These are needed
+          only at compile time.
       data: See [Bazel documentation](https://docs.bazel.build/versions/master/be/common-definitions.html#common.data).,
       compiler_flags: DEPRECATED. Use new name ghcopts.
       ghcopts: Flags to pass to Haskell compiler. Subject to Make variable substitution.
@@ -345,6 +359,7 @@ def haskell_binary(
         extra_srcs = extra_srcs,
         deps = deps,
         narrowed_deps = narrowed_deps,
+        th_deps = th_deps,
         data = data,
         compiler_flags = compiler_flags,
         ghcopts = ghcopts,
@@ -405,6 +420,7 @@ def haskell_test(
         extra_srcs = [],
         deps = [],
         narrowed_deps = [],
+        th_deps = [],
         data = [],
         compiler_flags = [],
         ghcopts = [],
@@ -463,6 +479,8 @@ def haskell_test(
           Note: This attribute is experimental and not ready for production, yet.
       modules: List of extra haskell_module() dependencies to be linked into this test.
           Note: This attribute is experimental and not ready for production, yet.
+      th_deps: List of libraries needed to run TemplateHaskell code. These are needed
+          only at compile time.
       data: See [Bazel documentation](https://docs.bazel.build/versions/master/be/common-definitions.html#common.data).,
       compiler_flags: DEPRECATED. Use new name ghcopts.
       ghcopts: Flags to pass to Haskell compiler. Subject to Make variable substitution.
@@ -499,6 +517,7 @@ def haskell_test(
         extra_srcs = extra_srcs,
         deps = deps,
         narrowed_deps = narrowed_deps,
+        th_deps = th_deps,
         data = data,
         compiler_flags = compiler_flags,
         ghcopts = ghcopts,
@@ -546,6 +565,7 @@ def haskell_library(
         deps = [],
         narrowed_deps = [],
         modules = [],
+        th_deps = [],
         data = [],
         compiler_flags = [],
         ghcopts = [],
@@ -606,6 +626,8 @@ def haskell_library(
           Note: This attribute is experimental and not ready for production, yet.
       modules: List of extra haskell_module() dependencies to be linked into this library.
           Note: This attribute is experimental and not ready for production, yet.
+      th_deps: List of libraries needed to run TemplateHaskell code. These are needed
+          only at compile time.
       data: See [Bazel documentation](https://docs.bazel.build/versions/master/be/common-definitions.html#common.data).,
       compiler_flags: DEPRECATED. Use new name ghcopts.
       ghcopts: Flags to pass to Haskell compiler. Subject to Make variable substitution.
@@ -636,6 +658,7 @@ def haskell_library(
         deps = deps,
         narrowed_deps = narrowed_deps,
         modules = modules,
+        th_deps = th_deps,
         data = data,
         compiler_flags = compiler_flags,
         ghcopts = ghcopts,

haskell/private/actions/compile.bzl

@@ -104,7 +104,28 @@ def _process_hsc_file(hs, cc, hsc_flags, hsc_inputs, hsc_file):
 
     return hs_out, idir
 
-def _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs, module_map, import_dir_map, extra_srcs, user_compile_flags, output_mode, with_profiling, interfaces_dir, objects_dir, my_pkg_id, version, plugins, non_default_plugins, preprocessors):
+def _compilation_defaults(
+    hs,
+    cc,
+    java,
+    posix,
+    dep_info,
+    th_dep_info,
+    plugin_dep_info,
+    srcs,
+    module_map,
+    import_dir_map,
+    extra_srcs,
+    user_compile_flags,
+    output_mode,
+    with_profiling,
+    interfaces_dir,
+    objects_dir,
+    my_pkg_id,
+    version,
+    plugins,
+    non_default_plugins,
+    preprocessors):
     """Compute variables common to all compilation targets (binary and library).
 
     Returns:
@@ -192,8 +213,15 @@ def _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs,
     (pkg_info_inputs, pkg_info_args) = pkg_info_to_compile_flags(
         hs,
         pkg_info = expose_packages(
-            package_ids = hs.package_ids,
-            package_databases = dep_info.package_databases,
+            package_ids = set.to_list(
+                set.from_list(
+                    hs.package_ids + hs.th_package_ids
+                )
+            ),
+            package_databases = depset(transitive=[
+                dep_info.package_databases,
+                th_dep_info.package_databases,
+            ]),
             version = version,
         ),
         plugin_pkg_info = expose_packages(
@@ -212,9 +240,10 @@ def _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs,
 
     hsc_inputs = []
     if version:
-        (version_macro_headers, version_macro_flags) = version_macro_includes(dep_info)
-        hsc_flags += ["--cflag=" + x for x in version_macro_flags]
-        hsc_inputs += set.to_list(version_macro_headers)
+        for info in [dep_info, th_dep_info]:
+            (version_macro_headers, version_macro_flags) = version_macro_includes(info)
+            hsc_flags += ["--cflag=" + x for x in version_macro_flags]
+            hsc_inputs += set.to_list(version_macro_headers)
 
     # Add import hierarchy root.
     # Note that this is not perfect, since GHC requires hs-boot files
@@ -349,7 +378,7 @@ def _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs,
 
     # Transitive library dependencies for runtime.
     link_libraries(
-        get_ghci_library_files(hs, cc.cc_libraries_info, cc.cc_libraries, for_th_only = True),
+        get_ghci_library_files(hs, cc.cc_libraries_info, cc.cc_th_libraries, for_th_only = True),
         args,
     )
 
@@ -363,11 +392,13 @@ def _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs,
             depset(cc.hdrs),
             dep_info.package_databases,
             dep_info.interface_dirs,
-            dep_info.hs_libraries,
+            th_dep_info.package_databases,
+            th_dep_info.interface_dirs,
+            th_dep_info.hs_libraries,
             plugin_dep_info.package_databases,
             plugin_dep_info.interface_dirs,
             plugin_dep_info.hs_libraries,
-            depset(get_ghci_library_files(hs, cc.cc_libraries_info, cc.transitive_libraries + cc.plugin_libraries)),
+            depset(get_ghci_library_files(hs, cc.cc_libraries_info, cc.transitive_th_libraries + cc.plugin_libraries)),
             java.inputs,
             preprocessors.inputs,
             plugin_tool_inputs,
@@ -405,6 +436,7 @@ def compile_binary(
         java,
         posix,
         dep_info,
+        th_dep_info,
         plugin_dep_info,
         srcs,
         module_map,
@@ -431,7 +463,29 @@ def compile_binary(
         source_files: set of Haskell source files
         boot_files: set of Haskell boot files
     """
-    c = _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs, module_map, import_dir_map, extra_srcs, user_compile_flags, "dynamic" if dynamic else "static", with_profiling, interfaces_dir, objects_dir, my_pkg_id = None, version = version, plugins = plugins, non_default_plugins = non_default_plugins, preprocessors = preprocessors)
+    c = _compilation_defaults(
+        hs,
+        cc,
+        java,
+        posix,
+        dep_info,
+        th_dep_info,
+        plugin_dep_info,
+        srcs,
+        module_map,
+        import_dir_map,
+        extra_srcs,
+        user_compile_flags,
+        "dynamic" if dynamic else "static",
+        with_profiling,
+        interfaces_dir,
+        objects_dir,
+        my_pkg_id = None,
+        version = version,
+        plugins = plugins,
+        non_default_plugins = non_default_plugins,
+        preprocessors = preprocessors,
+    )
     c.args.add_all(["-main-is", main_function])
     if dynamic:
         # For binaries, GHC creates .o files even for code to be
@@ -477,6 +531,7 @@ def compile_library(
         java,
         posix,
         dep_info,
+        th_dep_info,
         plugin_dep_info,
         srcs,
         module_map,
@@ -505,7 +560,29 @@ def compile_library(
         boot_files: set of Haskell boot files
         import_dirs: import directories that should make all modules visible (for GHCi)
     """
-    c = _compilation_defaults(hs, cc, java, posix, dep_info, plugin_dep_info, srcs, module_map, import_dir_map, extra_srcs, user_compile_flags, "dynamic-too" if with_shared else "static", with_profiling, interfaces_dir, objects_dir, my_pkg_id = my_pkg_id, version = my_pkg_id.version, plugins = plugins, non_default_plugins = non_default_plugins, preprocessors = preprocessors)
+    c = _compilation_defaults(
+        hs,
+        cc,
+        java,
+        posix,
+        dep_info,
+        th_dep_info,
+        plugin_dep_info,
+        srcs,
+        module_map,
+        import_dir_map,
+        extra_srcs,
+        user_compile_flags,
+        "dynamic-too" if with_shared else "static",
+        with_profiling,
+        interfaces_dir,
+        objects_dir,
+        my_pkg_id = my_pkg_id,
+        version = my_pkg_id.version,
+        plugins = plugins,
+        non_default_plugins = non_default_plugins,
+        preprocessors = preprocessors,
+    )
     if with_shared:
         c.args.add("-dynamic-too")
 

haskell/private/context.bzl

@@ -19,8 +19,21 @@ def haskell_context(ctx, attr = None):
     if not attr:
         attr = ctx.attr
 
-    deps = (attr.deps if hasattr(attr, "deps") else []) + (attr.exports if hasattr(attr, "exports") else []) + (attr.narrowed_deps if hasattr(attr, "narrowed_deps") else [])
-    package_ids = all_dependencies_package_ids(deps)
+    deps = getattr(attr, "deps", [])
+
+    if hasattr(attr, "th_deps"):
+        th_deps = attr.th_deps
+        if not th_deps and attr.use_deps_on_empty_th_deps:
+            th_deps = deps
+    else:
+        th_deps = deps
+
+    package_ids = all_dependencies_package_ids(
+        deps +
+        getattr(attr, "exports", []) +
+        getattr(attr, "narrowed_deps", [])
+    )
+    th_package_ids = all_dependencies_package_ids(th_deps)
 
     if hasattr(attr, "src_strip_prefix"):
         src_strip_prefix = attr.src_strip_prefix
@@ -63,6 +76,7 @@ def haskell_context(ctx, attr = None):
         ghc_wrapper = ghc_wrapper,
         worker = worker,
         package_ids = package_ids,
+        th_package_ids = th_package_ids,
         src_root = src_root,
         package_root = paths.join(ctx.label.workspace_root, ctx.label.package),
         env = env,