From 3c379c82d134bf0bc37a91fef1e1391eac9d4542 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Wed, 17 Jan 2018 11:45:18 -0500 Subject: [PATCH] cpp_param: Add ability to map C++ template paramters to Python to enable simple templating. --- bindings/pydrake/BUILD.bazel | 2 +- bindings/pydrake/util/BUILD.bazel | 34 +++++++ bindings/pydrake/util/cpp_param.py | 87 ++++++++++++++++ bindings/pydrake/util/cpp_param_pybind.cc | 81 +++++++++++++++ bindings/pydrake/util/cpp_param_pybind.h | 56 +++++++++++ .../util/test/cpp_param_pybind_test.cc | 98 +++++++++++++++++++ bindings/pydrake/util/test/cpp_param_test.py | 75 ++++++++++++++ tools/skylark/BUILD.bazel | 4 + tools/skylark/py_env_runner.py | 15 +++ tools/skylark/pybind.bzl | 51 ++++++++++ 10 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 bindings/pydrake/util/cpp_param.py create mode 100644 bindings/pydrake/util/cpp_param_pybind.cc create mode 100644 bindings/pydrake/util/cpp_param_pybind.h create mode 100644 bindings/pydrake/util/test/cpp_param_pybind_test.cc create mode 100644 bindings/pydrake/util/test/cpp_param_test.py create mode 100644 tools/skylark/py_env_runner.py diff --git a/bindings/pydrake/BUILD.bazel b/bindings/pydrake/BUILD.bazel index 40e2cc4e9a7c..d9bf0ef0a96e 100644 --- a/bindings/pydrake/BUILD.bazel +++ b/bindings/pydrake/BUILD.bazel @@ -49,7 +49,7 @@ drake_pybind_library( cc_srcs = ["common_py.cc"], package_info = PACKAGE_INFO, py_deps = [ - ":util_py", + "//bindings/pydrake/util:module_shim_py", ], py_srcs = [ "__init__.py", diff --git a/bindings/pydrake/util/BUILD.bazel b/bindings/pydrake/util/BUILD.bazel index fc8afdfc44fe..56adbbfc470a 100644 --- a/bindings/pydrake/util/BUILD.bazel +++ b/bindings/pydrake/util/BUILD.bazel @@ -9,6 +9,7 @@ load( ) load( "//tools/skylark:pybind.bzl", + "drake_pybind_cc_googletest", "get_drake_pybind_installs", "get_pybind_package_info", ) @@ -55,6 +56,7 @@ drake_py_library( PYBIND_LIBRARIES = [] PY_LIBRARIES = [ + ":cpp_param_py", ":module_py", ":module_shim_py", ] @@ -66,6 +68,25 @@ drake_py_library( deps = PYBIND_LIBRARIES + PY_LIBRARIES, ) +# ODR does not matter, because the singleton will be stored in Python. +drake_cc_library( + name = "cpp_param_pybind", + srcs = ["cpp_param_pybind.cc"], + hdrs = ["cpp_param_pybind.h"], + deps = [ + ":type_pack", + "@pybind11", + ], +) + +drake_py_library( + name = "cpp_param_py", + srcs = ["cpp_param.py"], + deps = [ + ":module_py", + ], +) + install( name = "install", targets = PY_LIBRARIES, @@ -100,4 +121,17 @@ drake_py_test( ], ) +drake_py_test( + name = "cpp_param_test", + deps = [ + ":cpp_param_py", + ], +) + +drake_pybind_cc_googletest( + name = "cpp_param_pybind_test", + cc_deps = [":cpp_param_pybind"], + py_deps = [":cpp_param_py"], +) + add_lint_tests() diff --git a/bindings/pydrake/util/cpp_param.py b/bindings/pydrake/util/cpp_param.py new file mode 100644 index 000000000000..36c47a6050ee --- /dev/null +++ b/bindings/pydrake/util/cpp_param.py @@ -0,0 +1,87 @@ +from __future__ import absolute_import, print_function + +import ctypes +import numpy as np + +""" +@file +Defines a mapping between Python and alias types, and provides canonical Python +types as they relate to C++. +""" + + +def _get_type_name(t, verbose): + # Gets type name as a string. + # Defaults to just returning the name to shorten template names. + if verbose and t.__module__ != "__builtin__": + return t.__module__ + "." + t.__name__ + else: + return t.__name__ + + +class _StrictMap(object): + # Provides a map which may only add a key once. + def __init__(self): + self._values = dict() + + def add(self, key, value): + assert key not in self._values, "Already added: {}".format(key) + self._values[key] = value + + def get(self, key, default): + return self._values.get(key, default) + + +class _ParamAliases(object): + # Registers aliases for a set of objects. This will be used for template + # parameters. + def __init__(self): + self._to_canonical = _StrictMap() + self._register_common() + + def _register_common(self): + # Register common Python aliases relevant for C++. + self.register(float, [np.double, ctypes.c_double]) + self.register(np.float32, [ctypes.c_float]) + self.register(int, [np.int32, ctypes.c_int32]) + self.register(np.uint32, [ctypes.c_uint32]) + self.register(np.int64, [ctypes.c_int64]) + + def register(self, canonical, aliases): + # Registers a set of aliases to a canonical value. + for alias in aliases: + self._to_canonical.add(alias, canonical) + + def is_aliased(self, alias): + # Determines if a parameter is aliased / registered. + return self._to_canonical.get(alias, None) is not None + + def get_canonical(self, alias): + # Gets registered canonical parameter if it is aliased; otherwise + # return the same parameter. + return self._to_canonical.get(alias, alias) + + def get_name(self, alias): + # Gets string for an alias. + canonical = self.get_canonical(alias) + if isinstance(canonical, type): + return _get_type_name(canonical, False) + else: + # For literals. + return str(canonical) + + +# Create singleton instance. +_param_aliases = _ParamAliases() + + +def get_param_canonical(param): + """Gets the canonical types for a set of Python types (canonical as in + how they relate to C++ types. """ + return tuple(map(_param_aliases.get_canonical, param)) + + +def get_param_names(param): + """Gets the canonical type names for a set of Python types (canonical as in + how they relate to C++ types. """ + return tuple(map(_param_aliases.get_name, param)) diff --git a/bindings/pydrake/util/cpp_param_pybind.cc b/bindings/pydrake/util/cpp_param_pybind.cc new file mode 100644 index 000000000000..11365cb634b6 --- /dev/null +++ b/bindings/pydrake/util/cpp_param_pybind.cc @@ -0,0 +1,81 @@ +#include "drake/bindings/pydrake/util/cpp_param_pybind.h" + +#include + +namespace drake { +namespace pydrake { +namespace internal { +namespace { + +// Creates a Python object that should uniquely hash for a primitive C++ +// type. +py::object GetPyHash(const std::type_info& tinfo) { + return py::make_tuple("cpp_type", tinfo.hash_code()); +} + +// Registers C++ type. +template +void RegisterType( + py::module m, py::object param_aliases, const std::string& canonical_str) { + // Create an object that is a unique hash. + py::object canonical = py::eval(canonical_str, m.attr("__dict__")); + py::list aliases(1); + aliases[0] = GetPyHash(typeid(T)); + param_aliases.attr("register")(canonical, aliases); +} + +// Registers common C++ types. +void RegisterCommon(py::module m, py::object param_aliases) { + // Make mappings for C++ RTTI to Python types. + // Unfortunately, this is hard to obtain from `pybind11`. + RegisterType(m, param_aliases, "bool"); + RegisterType(m, param_aliases, "str"); + RegisterType(m, param_aliases, "float"); + RegisterType(m, param_aliases, "np.float32"); + RegisterType(m, param_aliases, "int"); + RegisterType(m, param_aliases, "np.uint32"); + RegisterType(m, param_aliases, "np.int64"); + // For supporting generic Python types. + RegisterType(m, param_aliases, "object"); +} + +} // namespace + +py::object GetParamAliases() { + py::module m = py::module::import("pydrake.util.cpp_param"); + py::object param_aliases = m.attr("_param_aliases"); + const char registered_check[] = "_register_common_cpp"; + if (!py::hasattr(m, registered_check)) { + RegisterCommon(m, param_aliases); + m.attr(registered_check) = true; + } + return param_aliases; +} + +py::object GetPyParamScalarImpl(const std::type_info& tinfo) { + py::object param_aliases = GetParamAliases(); + py::object py_hash = GetPyHash(tinfo); + if (param_aliases.attr("is_aliased")(py_hash).cast()) { + // If it's an alias, return the canonical type. + return param_aliases.attr("get_canonical")(py_hash); + } else { + // This type is not aliased. Get the pybind-registered type, + // erroring out if it's not registered. + // WARNING: Internal API :( + auto* info = py::detail::get_type_info(tinfo); + if (!info) { + // TODO(eric.cousineau): Use NiceTypeName::Canonicalize(...Demangle(...)) + // once simpler dependencies are used (or something else is used to + // justify linking in `libdrake.so`). + const std::string name = tinfo.name(); + throw std::runtime_error( + "C++ type is not registered in pybind: " + name); + } + py::handle h(reinterpret_cast(info->type)); + return py::reinterpret_borrow(h); + } +} + +} // namespace internal +} // namespace pydrake +} // namespace drake diff --git a/bindings/pydrake/util/cpp_param_pybind.h b/bindings/pydrake/util/cpp_param_pybind.h new file mode 100644 index 000000000000..e74e8423aef7 --- /dev/null +++ b/bindings/pydrake/util/cpp_param_pybind.h @@ -0,0 +1,56 @@ +#pragma once + +/// @file +/// Provides a mechanism to map C++ types to canonical Python types. + +#include +#include +#include + +#include + +#include "drake/bindings/pydrake/util/type_pack.h" + +namespace drake { +namespace pydrake { + +// This alias is intended to be part of the public API, as it follows +// `pybind11` conventions. +namespace py = pybind11; + +namespace internal { + +// Gets singleton for type aliases from `cpp_param`. +py::object GetParamAliases(); + +// Gets Python type object given `std::type_info`. +// @throws std::runtime_error if type is neither aliased nor registered in +// `pybind11`. +py::object GetPyParamScalarImpl(const std::type_info& tinfo); + +// Gets Python type for a C++ type (base case). +template +inline py::object GetPyParamScalarImpl(type_pack = {}) { + return GetPyParamScalarImpl(typeid(T)); +} + +// Gets Python literal for a C++ literal (specialization). +template +inline py::object GetPyParamScalarImpl( + type_pack> = {}) { + return py::cast(Value); +} + +} // namespace internal + +/// Gets the canonical Python parameters for each C++ type. +/// @returns Python tuple of canonical parameters. +/// @throws std::runtime_error on the first type it encounters that is neither +/// aliased nor registered in `pybind11`. +template +inline py::tuple GetPyParam(type_pack = {}) { + return py::make_tuple(internal::GetPyParamScalarImpl(type_pack{})...); +} + +} // namespace pydrake +} // namespace drake diff --git a/bindings/pydrake/util/test/cpp_param_pybind_test.cc b/bindings/pydrake/util/test/cpp_param_pybind_test.cc new file mode 100644 index 000000000000..22c04cb44ba9 --- /dev/null +++ b/bindings/pydrake/util/test/cpp_param_pybind_test.cc @@ -0,0 +1,98 @@ +#include "drake/bindings/pydrake/util/cpp_param_pybind.h" + +// @file +// Tests the public interfaces in `cpp_param.py` and `cpp_param_pybind.h`. + +#include +#include + +#include +#include +#include +#include + +using std::string; + +namespace drake { +namespace pydrake { + +// Compare two Python objects directly. +bool PyEquals(py::object lhs, py::object rhs) { + return lhs.attr("__eq__")(rhs).cast(); +} + +// Ensures that the type `T` maps to the expression in `py_expr_expected`. +template +bool CheckPyParam(const string& py_expr_expected, type_pack param = {}) { + py::object actual = GetPyParam(param); + py::object expected = py::eval(py_expr_expected.c_str()); + return PyEquals(actual, expected); +} + +GTEST_TEST(CppParamTest, PrimitiveTypes) { + // Tests primitive types that are not expose directly via `pybind11`, thus + // needing custom registration. + // This follows the ordering in `cpp_param_pybind.cc`, + // `RegisterCommon`. + ASSERT_TRUE(CheckPyParam("bool,")); + ASSERT_TRUE(CheckPyParam("str,")); + ASSERT_TRUE(CheckPyParam("float,")); + ASSERT_TRUE(CheckPyParam("np.float32,")); + ASSERT_TRUE(CheckPyParam("int,")); + ASSERT_TRUE(CheckPyParam("np.uint32,")); + ASSERT_TRUE(CheckPyParam("np.int64,")); + ASSERT_TRUE(CheckPyParam("object,")); +} + +// Dummy type. +// - Registered. +struct CustomCppType {}; +// - Unregistered. +struct CustomCppTypeUnregistered {}; + +GTEST_TEST(CppParamTest, CustomTypes) { + // Tests types that are C++ types registered with `pybind11`. + ASSERT_TRUE(CheckPyParam("CustomCppType,")); + EXPECT_THROW( + CheckPyParam("CustomCppTypeUnregistered"), + std::runtime_error); +} + +template +using constant = std::integral_constant; + +GTEST_TEST(CppParamTest, LiteralTypes) { + // Tests that literal types are mapped to literals in Python. + ASSERT_TRUE(CheckPyParam("True,")); + ASSERT_TRUE((CheckPyParam>("-1,"))); + ASSERT_TRUE((CheckPyParam>("1,"))); +} + +GTEST_TEST(CppParamTest, Packs) { + // Tests that type packs are properly interpreted. + ASSERT_TRUE((CheckPyParam("int, bool"))); + ASSERT_TRUE((CheckPyParam>("bool, False"))); +} + +int main(int argc, char** argv) { + // Reconstructing `scoped_interpreter` multiple times (e.g. via `SetUp()`) + // while *also* importing `numpy` wreaks havoc. + py::scoped_interpreter guard; + + // Define common scope, import numpy for use in `eval`. + py::module m("__main__"); + py::globals()["np"] = py::module::import("numpy"); + + // Define custom class only once here. + py::class_(m, "CustomCppType"); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +} // namespace pydrake +} // namespace drake + +int main(int argc, char** argv) { + return drake::pydrake::main(argc, argv); +} diff --git a/bindings/pydrake/util/test/cpp_param_test.py b/bindings/pydrake/util/test/cpp_param_test.py new file mode 100644 index 000000000000..4ab0da085a9a --- /dev/null +++ b/bindings/pydrake/util/test/cpp_param_test.py @@ -0,0 +1,75 @@ +from __future__ import print_function + +""" +Tests the pure Python functionality of `cpp_param`: (a) idempotent mapping for +unaliased types, and (b) correct mapping for aliased types (as the aliases +relate in C++). + +N.B. The C++ types are not registered in this test. They are registered and +tested in `cpp_param_pybind_test`. +""" + +import unittest +import ctypes +import numpy as np + + +from pydrake.util.cpp_param import get_param_canonical, get_param_names + + +class CustomPyType(object): + pass + + +class TestCppParam(unittest.TestCase): + def _check_alias(self, canonical, alias): + actual = get_param_canonical([alias])[0] + self.assertTrue(actual is canonical) + + def _check_idempotent(self, canonical): + self._check_alias(canonical, canonical) + + def _check_aliases(self, canonical, aliases): + for alias in aliases: + self._check_alias(canonical, alias) + + def _check_names(self, name_canonical, aliases): + for alias in aliases: + actual = get_param_names([alias])[0] + self.assertEquals(actual, name_canonical) + + def test_idempotent(self): + # Check idempotent mapping for unaliased types. + # This follows the ordering in `cpp_param_pybind.cc`, + # `RegisterCommon`. + self._check_idempotent(bool) + self._check_idempotent(str) + self._check_idempotent(float) + self._check_idempotent(np.float32) + self._check_idempotent(int) + self._check_idempotent(np.uint32) + self._check_idempotent(np.int64) + self._check_idempotent(object) + # - Custom Types. + self._check_idempotent(CustomPyType) + # - Literals. + self._check_idempotent(1) + + def test_aliases(self): + # Aliases: + # This follows the ordering in `cpp_param.py`, + # `_ParamAliases._register_common`. + self._check_aliases(float, [np.double, ctypes.c_double]) + self._check_aliases(np.float32, [ctypes.c_float]) + self._check_aliases(int, [np.int32, ctypes.c_int32]) + self._check_aliases(np.uint32, [ctypes.c_uint32]) + self._check_aliases(np.int64, [ctypes.c_int64]) + + def test_names(self): + self._check_names("int", [int, np.int32, ctypes.c_int32]) + self._check_names("CustomPyType", [CustomPyType]) + self._check_names("1", [1]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/skylark/BUILD.bazel b/tools/skylark/BUILD.bazel index 8cd08f4220eb..42f319ca774b 100644 --- a/tools/skylark/BUILD.bazel +++ b/tools/skylark/BUILD.bazel @@ -4,4 +4,8 @@ package(default_visibility = ["//visibility:public"]) load("//tools/lint:lint.bzl", "add_lint_tests") +exports_files([ + "py_env_runner.py", +]) + add_lint_tests() diff --git a/tools/skylark/py_env_runner.py b/tools/skylark/py_env_runner.py new file mode 100644 index 000000000000..2fe16e748bfd --- /dev/null +++ b/tools/skylark/py_env_runner.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +""" +Wrapper Python script to ensure we can execute a C++ binary with access to +Python libraries using an environment established by Bazel. +""" + +# TODO(eric.cousineau): See if there is a way to do this in pure C++, such +# that it is easier to debug. + +import subprocess +import sys + +assert len(sys.argv) >= 2 +subprocess.check_call(sys.argv[1:]) diff --git a/tools/skylark/pybind.bzl b/tools/skylark/pybind.bzl index e66be9579cab..c3ac4a0b20a5 100644 --- a/tools/skylark/pybind.bzl +++ b/tools/skylark/pybind.bzl @@ -5,10 +5,12 @@ load("@drake//tools/install:install.bzl", "install") load( "//tools:drake.bzl", "drake_cc_binary", + "drake_cc_googletest", ) load( "//tools/skylark:drake_py.bzl", "drake_py_library", + "drake_py_test", ) load("//tools/skylark:6996.bzl", "adjust_label_for_drake_hoist") @@ -197,3 +199,52 @@ def _get_package_info(base_package, sub_package = None): base_path_rel = base_path_rel, # Sub-package's path relative to base package's path. sub_path_rel = sub_path_rel) + +def drake_pybind_cc_googletest( + name, + cc_srcs = [], + py_deps = [], + cc_deps = [], + args = [], + visibility = None, + tags = []): + """Defines a C++ test (using `pybind`) which has access to Python + libraries. """ + cc_name = name + "_cc" + if not cc_srcs: + cc_srcs = ["test/{}.cc".format(name)] + drake_cc_googletest( + name = cc_name, + srcs = cc_srcs, + deps = cc_deps + [ + "//tools/install/libdrake:drake_shared_library", + "@pybind11", + ], + # Add 'manual', because we only want to run it with Python present. + tags = ["manual"], + visibility = visibility, + ) + + py_name = name + "_py" + # Expose as library, to make it easier to expose Bazel environment for + # external tools. + drake_py_library( + name = py_name, + deps = py_deps, + testonly = 1, + visibility = visibility, + ) + + # Use this Python test as the glue for Bazel to expose the appropriate + # environment for the C++ binary. + py_main = "//tools/skylark:py_env_runner.py" + drake_py_test( + name = name, + srcs = [py_main], + main = py_main, + data = [cc_name], + args = ["$(location {})".format(cc_name)] + args, + deps = [py_name], + tags = tags, + visibility = visibility, + )