Add rule for bindgen (#102) (#108)

.bazelci/presubmit.yml

@@ -1,15 +1,24 @@
 ---
 default_targets: &default_targets
+- "--" # Allows negative patterns; hack for https://github.com/bazelbuild/continuous-integration/pull/245
 - "..."
 - "@docs//..."
 - "@examples//..."
+# Bindgen currently only has a working toolchain for 18.04
+- "-@examples//ffi/rust_calling_c/simple/..."
 platforms:
   ubuntu1404:
     build_targets: *default_targets
     test_targets: *default_targets
   ubuntu1604:
     build_targets: *default_targets
     test_targets: *default_targets
+  ubuntu1804:
+    build_targets: *default_targets
+    test_targets:
+      - "..."
+      - "@docs//..."
+      - "@examples//..."
   macos:
     osx_targets: &osx_targets
     - "--" # Allows negative patterns; hack for https://github.com/bazelbuild/continuous-integration/pull/245
@@ -19,11 +28,12 @@ platforms:
     # Skip tests for dylib support on osx, since we don't support it yet.
     - "-@examples//ffi/rust_calling_c:matrix_dylib_test"
     - "-@examples//ffi/rust_calling_c:matrix_dynamically_linked"
+    - "-@examples//ffi/rust_calling_c/simple/..."
     build_targets: *osx_targets
     test_targets: *osx_targets
   rbe_ubuntu1604:
     test_targets:
-    - "--"
+    - "--" # Allows negative patterns; hack for https://github.com/bazelbuild/continuous-integration/pull/245
     - "//test/..."
     - "@examples//..."
     - "-//test/conflicting_deps:conflicting_deps_test"
@@ -32,3 +42,4 @@ platforms:
     - "-@examples//fibonacci:fibonacci_doc_test"
     - "-@examples//hello_lib:hello_lib_doc_test"
     - "-//tools/runfiles:runfiles_doc_test"
+    - "-@examples//ffi/rust_calling_c/simple/..."

WORKSPACE

@@ -25,21 +25,22 @@ http_archive(
 )
 
 # TODO: Move this to examples/WORKSPACE when recursive repositories are enabled.
-load("//rust:repositories.bzl", "rust_repositories")
-
+load("@io_bazel_rules_rust//rust:repositories.bzl", "rust_repositories")
 rust_repositories()
 
 new_git_repository(
     name = "libc",
-    build_file = "//:libc.BUILD",
+    build_file = "@io_bazel_rules_rust//:libc.BUILD",
     remote = "https://github.com/rust-lang/libc",
     tag = "0.2.20",
 )
 
-load("//proto:repositories.bzl", "rust_proto_repositories")
-
+load("@io_bazel_rules_rust//proto:repositories.bzl", "rust_proto_repositories")
 rust_proto_repositories()
 
+load("@io_bazel_rules_rust//bindgen:repositories.bzl", "rust_bindgen_repositories")
+rust_bindgen_repositories()
+
 # Stardoc and its dependencies
 git_repository(
     name = "io_bazel_skydoc",
@@ -71,6 +72,5 @@ http_archive(
     ],
 )
 
-load(":workspace.bzl", "bazel_version")
-
+load("@io_bazel_rules_rust//:workspace.bzl", "bazel_version")
 bazel_version(name = "bazel_version")

bindgen/BUILD

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_rust//bindgen:bindgen.bzl", "rust_bindgen_toolchain")
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain_type(name = "bindgen_toolchain")
+
+bzl_library(
+    name = "rules",
+    srcs = glob(["**/*.bzl"]),
+    deps = ["@io_bazel_rules_rust//rust:rules"],
+)
+
+rust_bindgen_toolchain(
+    name = "example-bindgen-toolchain-impl",
+    bindgen = "//bindgen/raze:cargo_bin_bindgen",
+    clang = "@bindgen_clang//:clang",
+    libclang = "@bindgen_clang//:libclang.so",
+    libstdcxx = "@local_libstdcpp//:libstdc++",
+)
+
+toolchain(
+    name = "example-bindgen-toolchain",
+    toolchain = "example-bindgen-toolchain-impl",
+    toolchain_type = "@io_bazel_rules_rust//bindgen:bindgen_toolchain",
+)

bindgen/README.md

@@ -0,0 +1,82 @@
+# Rust Bindgen Rules
+
+<div class="toc">
+  <h2>Rules</h2>
+  <ul>
+    <li><a href="docs/index.md#rust_bindgen_toolchain">rust_bindgen_toolchain</a></li>
+    <li>rust_bindgen</li>
+    <li>rust_bindgen_library</li>
+  </ul>
+</div>
+
+## Overview
+
+These rules are for using [Bindgen][bindgen] to generate [Rust][rust] bindings to C (and some C++) libraries.
+
+[rust]: http://www.rust-lang.org/
+[bindgen]: https://github.com/rust-lang/rust-bindgen
+
+See the [bindgen example](../examples/ffi/rust_calling_c/simple/BUILD#L9) for a more complete example of use.
+
+### Setup
+
+To use the Rust bindgen rules, add the following to your `WORKSPACE` file to add the
+external repositories for the Rust bindgen toolchain (in addition to the [rust rules setup](..)):
+
+```python
+load("@io_bazel_rules_rust//bindgen:repositories.bzl", "rust_bindgen_repositories")
+
+rust_bindgen_repositories()
+```
+This makes the default toolchain defined in [`@io_bazel_rules_rust`](./BUILD) available.
+
+[raze]: https://github.com/google/cargo-raze
+
+It will load crate dependencies of bindgen that are generated using
+[cargo raze][raze] inside the rules_rust
+repository. However, using those dependencies might conflict with other uses
+of [cargo raze][raze]. If you need to change
+those dependencies, please see the [dedicated section below](#custom-deps).
+
+For additional information, see the [Bazel toolchains documentation](https://docs.bazel.build/versions/master/toolchains.html).
+
+## <a name="custom-deps">Customizing dependencies
+
+These rules depends on the [`bindgen`](https://crates.io/crates/bindgen) binary crate, and it 
+in turn depends on both a clang binary and the clang library. To obtain these dependencies,
+`rust_bindgen_repositories` imports bindgen and its dependencies using BUILD files generated with
+[`cargo raze`][raze], along with a tarball of clang.
+
+If you want to change the bindgen used, or use [`cargo raze`][raze] in a more
+complex scenario (with more dependencies), you must redefine those
+dependencies.
+
+To do this, once you've imported the needed dependencies (see our
+[Cargo.toml](./raze/Cargo.toml) file to see the default dependencies), you
+need to create your own toolchain. To do so you can create a BUILD
+file with your [`rust_bindgen_toolchain`](../docs/index.md#rust_bindgen_toolchain) definition, for example:
+
+```python
+load("@io_bazel_rules_rust//bindgen:bindgen.bzl", "rust_bindgen_toolchain")
+
+rust_bindgen_toolchain(
+    name = "bindgen-toolchain-impl",
+    bindgen = "//my/raze:cargo_bin_bindgen",
+    clang = "//my/clang:clang",
+    libclang = "//my/clang:libclang.so",
+    libstdcxx = "//my/cpp:libstdc++",
+)
+
+toolchain(
+    name = "bindgen-toolchain",
+    toolchain = "bindgen-toolchain-impl",
+    toolchain_type = "@io_bazel_rules_rust//bindgen:bindgen_toolchain",
+)
+```
+
+Now that you have your own toolchain, you need to register it by
+inserting the following statement in your `WORKSPACE` file:
+
+```python
+register_toolchains("//my/toolchains:bindgen-toolchain")
+```

bindgen/bindgen.bzl

@@ -0,0 +1,182 @@
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@io_bazel_rules_rust//rust:rust.bzl", "rust_library")
+
+def rust_bindgen_library(
+        name,
+        header,
+        cc_lib,
+        bindgen_flags = None,
+        clang_flags = None,
+        **kwargs):
+    """Generates a rust source file for `header`, and builds a rust_library.
+
+    Arguments are the same as `rust_bindgen`, and `kwargs` are passed directly to rust_library.
+    """
+
+    rust_bindgen(
+        name = name + "__bindgen",
+        header = header,
+        cc_lib = cc_lib,
+        bindgen_flags = bindgen_flags or [],
+        clang_flags = clang_flags or [],
+    )
+    rust_library(
+        name = name,
+        srcs = [name + "__bindgen.rs"],
+        deps = [cc_lib],
+        **kwargs
+    )
+
+def _rust_bindgen_impl(ctx):
+    # nb. We can't grab the cc_library`s direct headers, so a header must be provided.
+    cc_lib = ctx.attr.cc_lib
+    header = ctx.file.header
+    if header not in cc_lib.cc.transitive_headers:
+        fail("Header {} is not in {}'s transitive headers.".format(ctx.attr.header, cc_lib), "header")
+
+    toolchain = ctx.toolchains["@io_bazel_rules_rust//bindgen:bindgen_toolchain"]
+    bindgen_bin = toolchain.bindgen
+    rustfmt_bin = toolchain.rustfmt
+    clang_bin = toolchain.clang
+    libclang = toolchain.libclang
+
+    # TODO: This rule shouldn't need to depend on libstdc++
+    #  This rule requires an explicit dependency on a libstdc++ because
+    #    1. It is a runtime dependency of libclang.so
+    #    2. We cannot locate it in the cc_toolchain yet
+    #  Depending on how libclang.so was compiled, it may try to locate its libstdc++ dependency
+    #  in a way that makes our handling here unnecessary (eg. system /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
+    libstdcxx = toolchain.libstdcxx
+
+    # rustfmt is not where bindgen expects to find it, so we format manually
+    bindgen_args = ["--no-rustfmt-bindings"] + ctx.attr.bindgen_flags
+    clang_args = ctx.attr.clang_flags
+
+    output = ctx.outputs.out
+
+    # libclang should only have 1 output file
+    libclang_dir = libclang.cc.libs.to_list()[0].dirname
+    include_directories = depset(
+        [f.dirname for f in cc_lib.cc.transitive_headers],
+    )
+
+    # Vanilla usage of bindgen produces formatted output, here we do the same if we have `rustfmt` in our toolchain.
+    if rustfmt_bin:
+        unformatted_output = ctx.actions.declare_file(output.basename + ".unformatted")
+    else:
+        unformatted_output = output
+
+    args = ctx.actions.args()
+    args.add_all(bindgen_args)
+    args.add(header.path)
+    args.add("--output", unformatted_output.path)
+    args.add("--")
+    args.add_all(include_directories, before_each = "-I")
+    args.add_all(clang_args)
+    ctx.actions.run(
+        executable = bindgen_bin,
+        inputs = depset(
+            [header],
+            transitive = [cc_lib.cc.transitive_headers, libclang.cc.libs, libstdcxx.cc.libs],
+        ),
+        outputs = [unformatted_output],
+        mnemonic = "RustBindgen",
+        progress_message = "Generating bindings for {}..".format(header.path),
+        env = {
+            "RUST_BACKTRACE": "1",
+            "CLANG_PATH": clang_bin.path,
+            # Bindgen loads libclang at runtime, which also needs libstdc++, so we setup LD_LIBRARY_PATH
+            "LIBCLANG_PATH": libclang_dir,
+            "LD_LIBRARY_PATH": ":".join([f.dirname for f in libstdcxx.cc.libs]),
+        },
+        arguments = [args],
+        tools = [clang_bin],
+    )
+
+    if rustfmt_bin:
+        ctx.actions.run_shell(
+            inputs = depset([rustfmt_bin, unformatted_output]),
+            outputs = [output],
+            command = "{} --emit stdout --quiet {} > {}".format(rustfmt_bin.path, unformatted_output.path, output.path),
+            tools = [rustfmt_bin],
+        )
+
+rust_bindgen = rule(
+    _rust_bindgen_impl,
+    doc = "Generates a rust source file from a cc_library and a header.",
+    attrs = {
+        "header": attr.label(
+            doc = "The .h file to generate bindings for.",
+            allow_single_file = True,
+        ),
+        "cc_lib": attr.label(
+            doc = "The cc_library that contains the .h file. This is used to find the transitive includes.",
+            providers = ["cc"],
+        ),
+        "bindgen_flags": attr.string_list(
+            doc = "Flags to pass directly to the bindgen executable. See https://rust-lang.github.io/rust-bindgen/ for details.",
+        ),
+        "clang_flags": attr.string_list(
+            doc = "Flags to pass directly to the clang executable.",
+        ),
+    },
+    outputs = {"out": "%{name}.rs"},
+    toolchains = [
+        "@io_bazel_rules_rust//bindgen:bindgen_toolchain",
+    ],
+)
+
+def _rust_bindgen_toolchain_impl(ctx):
+    return platform_common.ToolchainInfo(
+        bindgen = ctx.executable.bindgen,
+        clang = ctx.executable.clang,
+        libclang = ctx.attr.libclang,
+        libstdcxx = ctx.attr.libstdcxx,
+        rustfmt = ctx.executable.rustfmt,
+    )
+
+rust_bindgen_toolchain = rule(
+    _rust_bindgen_toolchain_impl,
+    doc = "The tools required for the `rust_bindgen` rule.",
+    attrs = {
+        "bindgen": attr.label(
+            doc = "The label of a `bindgen` executable.",
+            executable = True,
+            cfg = "host",
+        ),
+        "rustfmt": attr.label(
+            doc = "The label of a `rustfmt` executable. If this is provided, generated sources will be formatted.",
+            executable = True,
+            cfg = "host",
+            mandatory = False,
+        ),
+        "clang": attr.label(
+            doc = "The label of a `clang` executable.",
+            executable = True,
+            cfg = "host",
+        ),
+        "libclang": attr.label(
+            doc = "A cc_library that provides bindgen's runtime dependency on libclang.",
+            cfg = "host",
+            providers = ["cc"],
+        ),
+        "libstdcxx": attr.label(
+            doc = "A cc_library that satisfies libclang's libstdc++ dependency.",
+            cfg = "host",
+            providers = ["cc"],
+        ),
+    },
+)

bindgen/clang.BUILD

@@ -0,0 +1,14 @@
+package(default_visibility = ["//visibility:public"])
+
+sh_binary(
+    name = "clang",
+    srcs = ["bin/clang"],
+    data = glob(["lib/**"]),
+)
+
+cc_library(
+    name = "libclang.so",
+    srcs = [
+        "lib/libclang.so",
+    ],
+)