diff --git a/bindings/pydrake/BUILD.bazel b/bindings/pydrake/BUILD.bazel index 40e2cc4e9a7c..d9bf0ef0a96e 100644 --- a/bindings/pydrake/BUILD.bazel +++ b/bindings/pydrake/BUILD.bazel @@ -49,7 +49,7 @@ drake_pybind_library( cc_srcs = ["common_py.cc"], package_info = PACKAGE_INFO, py_deps = [ - ":util_py", + "//bindings/pydrake/util:module_shim_py", ], py_srcs = [ "__init__.py", diff --git a/bindings/pydrake/util/BUILD.bazel b/bindings/pydrake/util/BUILD.bazel index fc8afdfc44fe..84bfb9feedd9 100644 --- a/bindings/pydrake/util/BUILD.bazel +++ b/bindings/pydrake/util/BUILD.bazel @@ -6,9 +6,12 @@ load( "//tools:drake.bzl", "drake_cc_googletest", "drake_cc_library", + "drake_cc_binary", ) load( "//tools/skylark:pybind.bzl", + "drake_pybind_cc_googletest", + "drake_pybind_library", "get_drake_pybind_installs", "get_pybind_package_info", ) @@ -52,7 +55,9 @@ drake_py_library( imports = PACKAGE_INFO.py_imports, ) -PYBIND_LIBRARIES = [] +PYBIND_LIBRARIES = [ + ":cpp_types_py", +] PY_LIBRARIES = [ ":module_py", @@ -66,6 +71,30 @@ drake_py_library( deps = PYBIND_LIBRARIES + PY_LIBRARIES, ) +# ODR does not matter, because the singleton will be stored in Python. +drake_cc_library( + name = "cpp_types_pybind", + srcs = ["cpp_types_pybind.cc"], + hdrs = ["cpp_types_pybind.h"], + deps = [ + ":type_pack", + "@pybind11", + ], +) + +drake_pybind_library( + name = "cpp_types_py", + cc_deps = [ + ":cpp_types_pybind", + ], + cc_srcs = ["cpp_types_py.cc"], + package_info = PACKAGE_INFO, + py_deps = [ + ":module_py", + ], + py_srcs = ["cpp_types.py"], +) + install( name = "install", targets = PY_LIBRARIES, @@ -100,4 +129,10 @@ drake_py_test( ], ) +drake_pybind_cc_googletest( + name = "cpp_types_test", + cc_deps = [":cpp_types_pybind"], + py_deps = [":cpp_types_py"], +) + add_lint_tests() diff --git a/bindings/pydrake/util/cpp_types.py b/bindings/pydrake/util/cpp_types.py new file mode 100644 index 000000000000..51ca667e351c --- /dev/null +++ b/bindings/pydrake/util/cpp_types.py @@ -0,0 +1,55 @@ +from __future__ import absolute_import, print_function + +import ctypes +import numpy as np + +""" +@file +Defines a mapping between Python and C++ types, and provides canonical Python +types as they relate to C++. +""" + +# Define these first, as they are used in `cpp_types_py.cc` +# (transitively, `cpp_types_pybind.cc`). + + +def _get_type_name(t): + # Gets type name as a string. + prefix = t.__module__ + "." + if prefix == "__builtin__.": + prefix = "" + return prefix + t.__name__ + + +class _StrictMap(object): + def __init__(self): + self._values = dict() + + def _strict_key(self, key): + # Ensures keys are strictly scoped to the values (for literals). + return (type(key), key) + + def add(self, key, value): + skey = self._strict_key(key) + assert skey not in self._values, "Already added: {}".format(skey) + self._values[skey] = value + + def get(self, key, default): + skey = self._strict_key(key) + return self._values.get(skey, default) + + +# Load and import type registry. +from ._cpp_types_py import _type_registry # noqa + + +def get_types_canonical(param): + """Gets the canonical types for a set of Python types (canonical as in + how they relate to C++ types. """ + return tuple(map(_type_registry.GetPyTypeCanonical, param)) + + +def get_type_names(param): + """Gets the canonical type names for a set of Python types (canonical as in + how they relate to C++ types. """ + return tuple(map(_type_registry.GetName, param)) diff --git a/bindings/pydrake/util/cpp_types_py.cc b/bindings/pydrake/util/cpp_types_py.cc new file mode 100644 index 000000000000..bae148fa02fe --- /dev/null +++ b/bindings/pydrake/util/cpp_types_py.cc @@ -0,0 +1,15 @@ +#include + +#include "drake/bindings/pydrake/util/cpp_types_pybind.h" + +using drake::pydrake::internal::TypeRegistry; + +PYBIND11_MODULE(_cpp_types_py, m) { + py::class_ type_registry_cls(m, "_TypeRegistry"); + type_registry_cls + .def(py::init<>()) + .def("GetPyTypeCanonical", &TypeRegistry::GetPyTypeCanonical) + .def("GetName", &TypeRegistry::GetName); + // Create singleton instance. + m.attr("_type_registry") = type_registry_cls(); +} diff --git a/bindings/pydrake/util/cpp_types_pybind.cc b/bindings/pydrake/util/cpp_types_pybind.cc new file mode 100644 index 000000000000..b6e026b5f59b --- /dev/null +++ b/bindings/pydrake/util/cpp_types_pybind.cc @@ -0,0 +1,171 @@ +#include "drake/bindings/pydrake/util/cpp_types_pybind.h" + +#include + +const char kModule[] = "pydrake.util.cpp_types"; + +namespace drake { +namespace pydrake { + +namespace internal { + +TypeRegistry::TypeRegistry() { + // Import modules into `locals_`. + globals_ = py::module::import(kModule).attr("__dict__"); + py_to_py_canonical_ = eval("_StrictMap")(); + + RegisterCommon(); + RegisterLiterals(); +} + +const TypeRegistry& TypeRegistry::GetPyInstance() { + auto m = py::module::import(kModule); + py::object type_registry_py = m.attr("_type_registry"); + const TypeRegistry* type_registry = + py::cast(type_registry_py); + return *type_registry; +} + +py::object TypeRegistry::DoGetPyType(const std::type_info& tinfo) const { + // Check if it's a custom-registered type. + size_t cpp_key = std::type_index(tinfo).hash_code(); + auto iter = cpp_to_py_.find(cpp_key); + if (iter != cpp_to_py_.end()) { + return iter->second; + } else { + // Get from pybind11-registered types. + // WARNING: Internal API :( + auto* info = py::detail::get_type_info(tinfo); + if (!info) { + throw std::runtime_error("Unknown type!"); + } + return py::reinterpret_borrow( + py::handle(reinterpret_cast(info->type))); + } +} + +py::object TypeRegistry::GetPyTypeCanonical(py::object py_type) const { + // Get registered canonical type if there is a mapping; otherwise return + // original type. + return py_to_py_canonical_.attr("get")(py_type, py_type); +} + +py::str TypeRegistry::GetName(py::object py_type) const { + py::object py_type_canonical = GetPyTypeCanonical(py_type); + py::object name = py_name_.attr("get")(py_type_canonical); + // Assume this is a Python type. + if (name.is(py::none())) { + name = eval("_get_type_name")(py_type_canonical); + } + return name; +} + +py::object TypeRegistry::eval(const std::string& expr) const { + return py::eval(expr, globals_, locals_); +} + +void TypeRegistry::Register( + const std::vector& cpp_keys, + py::tuple py_types, const std::string& name) { + py::object py_canonical = py_types[0]; + for (size_t cpp_key : cpp_keys) { + assert(cpp_to_py_.find(cpp_key) == cpp_to_py_.end()); + cpp_to_py_[cpp_key] = py_canonical; + } + for (auto py_type : py_types) { + py_to_py_canonical_.attr("add")(py_type, py_canonical); + } + py_name_[py_canonical] = name; +} + +template +void TypeRegistry::RegisterType( + py::tuple py_types) { + py::str name = eval("_get_type_name")(py_types[0]); + Register({type_hash()}, py_types, name.cast()); +} + +void TypeRegistry::RegisterCommon() { + // Make mappings for C++ RTTI to Python types. + // Unfortunately, this is hard to obtain from `pybind11`. + RegisterType(eval("bool,")); + RegisterType(eval("str,")); + RegisterType(eval("float, np.double, ctypes.c_double")); + RegisterType(eval("np.float32, ctypes.c_float")); + RegisterType(eval("int, np.int32, ctypes.c_int32")); + RegisterType(eval("np.uint32, ctypes.c_uint32")); + RegisterType(eval("np.int64, ctypes.c_int64")); + // For supporting generic Python types. + RegisterType(eval("object,")); +} + +class TypeRegistry::LiteralHelper { + public: + explicit LiteralHelper(TypeRegistry* type_registry) + : type_registry_(type_registry) {} + + void RegisterLiterals() { + RegisterSequence(Render(std::integer_sequence{})); + // Register `int` (and `uint` as an alias for positive values). + constexpr int i_max = 100; + RegisterSequence(MakeSequence()); + RegisterSequence( + MakeSequence(), + {MakeSequence()}); + } + + private: + template + struct Sequence { + std::vector keys; + std::vector values; + }; + + template + Sequence Render(std::integer_sequence) { + return Sequence{ + {type_hash>()...}, + {Values...}}; + } + + template + Sequence MakeSequence() { + constexpr T Count = End - Start + 1; + auto seq = sequence_transform( + constant_add{}, std::make_integer_sequence{}); + return Render(seq); + } + + template + void RegisterSequence( + const Sequence& seq, + std::vector> alias_set = {}) { + for (int i = 0; i < seq.keys.size(); ++i) { + // Get alias types. + std::vector cpp_keys{seq.keys[i]}; + for (const auto& alias : alias_set) { + assert(seq.values[i] == alias.values[i]); + cpp_keys.push_back(alias.keys[i]); + } + // Register. + T value = seq.values[i]; + py::object py_value = py::cast(value); + type_registry_->Register( + cpp_keys, + py::make_tuple(py_value), + py::str(py_value).cast()); + } + } + + TypeRegistry* type_registry_; +}; + +void TypeRegistry::RegisterLiterals() { + // Register a subset of literals. + LiteralHelper(this).RegisterLiterals(); +} + +} // namespace internal + +} // namespace pydrake +} // namespace drake diff --git a/bindings/pydrake/util/cpp_types_pybind.h b/bindings/pydrake/util/cpp_types_pybind.h new file mode 100644 index 000000000000..9702dce31fda --- /dev/null +++ b/bindings/pydrake/util/cpp_types_pybind.h @@ -0,0 +1,87 @@ +#pragma once + +/// @file +/// Provides a mechanism to map C++ types to canonical Python types. + +#include +#include +#include + +#include + +#include "drake/bindings/pydrake/util/type_pack.h" + +namespace py = pybind11; + +namespace drake { +namespace pydrake { + +namespace internal { + +// Provides a mechanism to map between Python and C++ types. +class TypeRegistry { + public: + TypeRegistry(); + + // Gets the singleton instance which is stored in Python. + // @note Storing the singleton in Python permits this module avoid + // significant ODR violations until we have shared library linking. + static const TypeRegistry& GetPyInstance(); + + // Gets the Python type for a given C++ type. + template + py::object GetPyType() const { + return DoGetPyType(typeid(T)); + } + + // Gets the canonical Python type for a given Python type. + py::object GetPyTypeCanonical(py::object py_type) const; + + // Gets the canonical string name for a given Python type. + py::str GetName(py::object py_type) const; + + private: + py::object eval(const std::string& expr) const; + + void exec(const std::string& expr); + + py::object DoGetPyType(const std::type_info& tinfo) const; + + void Register( + const std::vector& cpp_keys, + py::tuple py_types, const std::string& name); + + template + void RegisterType(py::tuple py_types); + + void RegisterCommon(); + void RegisterLiterals(); + + class LiteralHelper; + friend class LiteralHelper; + + py::object globals_; + py::object locals_; + std::map cpp_to_py_; + py::object py_to_py_canonical_; + py::dict py_name_; +}; + +} // namespace internal + +/// Gets the canonical Python type for a given C++ type. +template +inline py::object GetPyType(type_pack = {}) { + const internal::TypeRegistry& type_registry = + internal::TypeRegistry::GetPyInstance(); + return type_registry.GetPyType(); +} + +/// Gets the canonical Python types for each C++ type. +template +inline py::tuple GetPyTypes(type_pack = {}) { + return py::make_tuple(GetPyType()...); +} + +} // namespace pydrake +} // namespace drake diff --git a/bindings/pydrake/util/test/cpp_types_test.cc b/bindings/pydrake/util/test/cpp_types_test.cc new file mode 100644 index 000000000000..77fd49081434 --- /dev/null +++ b/bindings/pydrake/util/test/cpp_types_test.cc @@ -0,0 +1,114 @@ +#include "drake/bindings/pydrake/util/cpp_types_pybind.h" + +// @file +// Tests the public interfaces in `cpp_types.py` and `cpp_types_pybind.h`. + +#include + +#include +#include +#include +#include + +using std::string; + +namespace drake { +namespace pydrake { + +struct CustomType {}; + +class CppTypesTest : public ::testing::Test { + protected: + void SetUp() override { + py::exec(R"""( +import numpy as np +import ctypes + +from pydrake.util.cpp_types import get_types_canonical, get_type_names +)"""); + } + + template + bool CheckPyType(const string& py_expr_expected) { + return GetPyType().is(py::eval(py_expr_expected.c_str())); + } + + bool PyEquals(py::object lhs, py::object rhs) { + return lhs.attr("__eq__")(rhs).cast(); + } +}; + +TEST_F(CppTypesTest, InPython) { + // Check pure-Python behavior. + py::dict locals; + py::exec(R"""( +pairs = ( + # One-to-one. + (bool, bool), + (str, str), + (int, int), + (float, float), + (object, object), + # - Custom Types. + (CustomType, CustomType), + # - Literals. + (1, 1), + # Aliases: + (float, np.double), + (int, ctypes.c_int32), +) + +for canonical, alias in pairs: + pair_str = "{}, {}".format(alias, canonical) + assert get_types_canonical([alias])[0] is canonical, "Bad pair: " + pair_str + +assert get_type_names([int])[0] == "int" +assert get_type_names([ctypes.c_int32])[0] == "int" +assert get_type_names([CustomType])[0] == "__main__.CustomType" +)""", py::globals(), locals); + // Sanity check to ensure we've executed our Python code. + ASSERT_TRUE(!locals["pairs"].is_none()); +} + +template +using int_constant = std::integral_constant; + +template +using uint_constant = std::integral_constant; + +TEST_F(CppTypesTest, InCpp) { + // Check C++ behavior. + ASSERT_TRUE(CheckPyType("bool")); + ASSERT_TRUE(CheckPyType("str")); + ASSERT_TRUE(CheckPyType("float")); + ASSERT_TRUE(CheckPyType("np.float32")); + ASSERT_TRUE(CheckPyType("int")); + ASSERT_TRUE(CheckPyType("object")); + // Custom types. + ASSERT_TRUE(CheckPyType("CustomType")); + ASSERT_TRUE(PyEquals(GetPyTypes(), py::eval("int, bool"))); + // Literals parameters. + ASSERT_TRUE(CheckPyType("True")); + ASSERT_TRUE(CheckPyType>("-1")); + ASSERT_TRUE(CheckPyType>("1")); +} + +int main(int argc, char** argv) { + // Reconstructing `scoped_interpreter` mutliple times (e.g. via `SetUp()`) + // while *also* importing `numpy` wreaks havoc. + py::scoped_interpreter guard; + + // Define custom class only once here. + py::module m("__main__"); + py::class_(m, "CustomType"); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +} // namespace pydrake +} // namespace drake + +int main(int argc, char** argv) { + return drake::pydrake::main(argc, argv); +} diff --git a/tools/skylark/BUILD.bazel b/tools/skylark/BUILD.bazel index 8cd08f4220eb..42f319ca774b 100644 --- a/tools/skylark/BUILD.bazel +++ b/tools/skylark/BUILD.bazel @@ -4,4 +4,8 @@ package(default_visibility = ["//visibility:public"]) load("//tools/lint:lint.bzl", "add_lint_tests") +exports_files([ + "py_env_runner.py", +]) + add_lint_tests() diff --git a/tools/skylark/py_env_runner.py b/tools/skylark/py_env_runner.py new file mode 100644 index 000000000000..0414483f0ebf --- /dev/null +++ b/tools/skylark/py_env_runner.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +""" +Wrapper Python script to ensure we can execute a C++ binary with access to +Python libraries using an environment established by Bazel. +""" + +# TODO(eric.cousineau): See if there is a way to do this in pure C++, such +# that it is easier to debug. + +import subprocess +import sys + +assert len(sys.argv) == 2 +filepath = sys.argv[1] + +subprocess.check_call([filepath]) diff --git a/tools/skylark/pybind.bzl b/tools/skylark/pybind.bzl index e66be9579cab..c3ac4a0b20a5 100644 --- a/tools/skylark/pybind.bzl +++ b/tools/skylark/pybind.bzl @@ -5,10 +5,12 @@ load("@drake//tools/install:install.bzl", "install") load( "//tools:drake.bzl", "drake_cc_binary", + "drake_cc_googletest", ) load( "//tools/skylark:drake_py.bzl", "drake_py_library", + "drake_py_test", ) load("//tools/skylark:6996.bzl", "adjust_label_for_drake_hoist") @@ -197,3 +199,52 @@ def _get_package_info(base_package, sub_package = None): base_path_rel = base_path_rel, # Sub-package's path relative to base package's path. sub_path_rel = sub_path_rel) + +def drake_pybind_cc_googletest( + name, + cc_srcs = [], + py_deps = [], + cc_deps = [], + args = [], + visibility = None, + tags = []): + """Defines a C++ test (using `pybind`) which has access to Python + libraries. """ + cc_name = name + "_cc" + if not cc_srcs: + cc_srcs = ["test/{}.cc".format(name)] + drake_cc_googletest( + name = cc_name, + srcs = cc_srcs, + deps = cc_deps + [ + "//tools/install/libdrake:drake_shared_library", + "@pybind11", + ], + # Add 'manual', because we only want to run it with Python present. + tags = ["manual"], + visibility = visibility, + ) + + py_name = name + "_py" + # Expose as library, to make it easier to expose Bazel environment for + # external tools. + drake_py_library( + name = py_name, + deps = py_deps, + testonly = 1, + visibility = visibility, + ) + + # Use this Python test as the glue for Bazel to expose the appropriate + # environment for the C++ binary. + py_main = "//tools/skylark:py_env_runner.py" + drake_py_test( + name = name, + srcs = [py_main], + main = py_main, + data = [cc_name], + args = ["$(location {})".format(cc_name)] + args, + deps = [py_name], + tags = tags, + visibility = visibility, + )