Skip to content

Commit

Permalink
scoped_interpreter. overloaded constructor: PyConfig param (#4330)
Browse files Browse the repository at this point in the history 
* scoped_interpreter overloaded ctor: PyConfig param

* style: pre-commit fixes

* refact: some logics extracted into funcs (precheck_interpreter, _initialize_interpreter); config_guard

* style: pre-commit fixes

* refact: scoped_config, some funcs hidden in detail ns

* refact: macro PYBIND11_PYCONFIG_SUPPORT_PY_VERSION + undef

* feat: PYBIND11_PYCONFIG_SUPPORT_PY_VERSION set to 3.8

* tests: Custom PyConfig

* ci: python 3.6 -> 3.8

* ci: reverted py 38 back  to 36; refact: initialize_interpreter overloads

* style: pre-commit fixes

* fix: readability-implicit-bool-conversion

* refact: each initialize_interpreter overloads in pybind11 ns

* Move `initialize_interpreter_pre_pyconfig()` into the `detail` namespace.

Move the `PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX` define down to where it is used for the first time, and check if it is defined already, so that it is possible to customize from the compilation command line, just in case there is some unforeseen issue for Python 3.8, 3.9, 3.10.

* tests: Add program dir to path, Custom PyConfig with argv

* refact: clang-formatted

* tests: Add-program-dir-to-path covers both scoped_interpreter overloads

* tests: Add-program-dir-to-path fixed

* tests: Add-program-dir-to-path py_version dependant validation

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <[email protected]>
  • Loading branch information
@arman-novikov @pre-commit-ci
3 people authored Dec 1, 2022
1 parent 06003e8 commit 8869984
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 42 deletions.
119 changes: 77 additions & 42 deletions include/pybind11/embed.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,37 +86,22 @@ inline wchar_t *widen_chars(const char *safe_arg) {
return widened_arg;
}

PYBIND11_NAMESPACE_END(detail)

/** \rst
Initialize the Python interpreter. No other pybind11 or CPython API functions can be
called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
optional `init_signal_handlers` parameter can be used to skip the registration of
signal handlers (see the `Python documentation`_ for details). Calling this function
again after the interpreter has already been initialized is a fatal error.
If initializing the Python interpreter fails, then the program is terminated. (This
is controlled by the CPython runtime and is an exception to pybind11's normal behavior
of throwing exceptions on errors.)
The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
used to populate ``sys.argv`` and ``sys.path``.
See the |PySys_SetArgvEx documentation|_ for details.
.. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
.. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
.. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
\endrst */
inline void initialize_interpreter(bool init_signal_handlers = true,
int argc = 0,
const char *const *argv = nullptr,
bool add_program_dir_to_path = true) {
inline void precheck_interpreter() {
if (Py_IsInitialized() != 0) {
pybind11_fail("The interpreter is already running");
}
}

#if PY_VERSION_HEX < 0x030B0000
#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX)
# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000)
#endif

#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
int argc,
const char *const *argv,
bool add_program_dir_to_path) {
detail::precheck_interpreter();
Py_InitializeEx(init_signal_handlers ? 1 : 0);
# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000
PyEval_InitThreads();
Expand Down Expand Up @@ -150,33 +135,74 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
auto *pysys_argv = widened_argv.get();

PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path));
#else
PyConfig config;
PyConfig_InitIsolatedConfig(&config);
config.isolated = 0;
config.use_environment = 1;
config.install_signal_handlers = init_signal_handlers ? 1 : 0;
}
#endif

PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast<char *const *>(argv));
if (PyStatus_Exception(status)) {
PYBIND11_NAMESPACE_END(detail)

#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
inline void initialize_interpreter(PyConfig *config,
int argc = 0,
const char *const *argv = nullptr,
bool add_program_dir_to_path = true) {
detail::precheck_interpreter();
PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv));
if (PyStatus_Exception(status) != 0) {
// A failure here indicates a character-encoding failure or the python
// interpreter out of memory. Give up.
PyConfig_Clear(&config);
throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg
: "Failed to prepare CPython");
PyConfig_Clear(config);
throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
: "Failed to prepare CPython");
}
status = Py_InitializeFromConfig(&config);
PyConfig_Clear(&config);
if (PyStatus_Exception(status)) {
throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg
: "Failed to init CPython");
status = Py_InitializeFromConfig(config);
if (PyStatus_Exception(status) != 0) {
PyConfig_Clear(config);
throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
: "Failed to init CPython");
}
if (add_program_dir_to_path) {
PyRun_SimpleString("import sys, os.path; "
"sys.path.insert(0, "
"os.path.abspath(os.path.dirname(sys.argv[0])) "
"if sys.argv and os.path.exists(sys.argv[0]) else '')");
}
PyConfig_Clear(config);
}
#endif

/** \rst
Initialize the Python interpreter. No other pybind11 or CPython API functions can be
called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
optional `init_signal_handlers` parameter can be used to skip the registration of
signal handlers (see the `Python documentation`_ for details). Calling this function
again after the interpreter has already been initialized is a fatal error.
If initializing the Python interpreter fails, then the program is terminated. (This
is controlled by the CPython runtime and is an exception to pybind11's normal behavior
of throwing exceptions on errors.)
The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
used to populate ``sys.argv`` and ``sys.path``.
See the |PySys_SetArgvEx documentation|_ for details.
.. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
.. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
.. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
\endrst */
inline void initialize_interpreter(bool init_signal_handlers = true,
int argc = 0,
const char *const *argv = nullptr,
bool add_program_dir_to_path = true) {
#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
detail::initialize_interpreter_pre_pyconfig(
init_signal_handlers, argc, argv, add_program_dir_to_path);
#else
PyConfig config;
PyConfig_InitIsolatedConfig(&config);
config.isolated = 0;
config.use_environment = 1;
config.install_signal_handlers = init_signal_handlers ? 1 : 0;
initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
#endif
}

Expand Down Expand Up @@ -264,6 +290,15 @@ class scoped_interpreter {
initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path);
}

#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
explicit scoped_interpreter(PyConfig *config,
int argc = 0,
const char *const *argv = nullptr,
bool add_program_dir_to_path = true) {
initialize_interpreter(config, argc, argv, add_program_dir_to_path);
}
#endif

scoped_interpreter(const scoped_interpreter &) = delete;
scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
scoped_interpreter &operator=(const scoped_interpreter &) = delete;
Expand Down
66 changes: 66 additions & 0 deletions tests/test_embed/test_interpreter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,72 @@ TEST_CASE("There can be only one interpreter") {
py::initialize_interpreter();
}

#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
TEST_CASE("Custom PyConfig") {
py::finalize_interpreter();
PyConfig config;
PyConfig_InitPythonConfig(&config);
REQUIRE_NOTHROW(py::scoped_interpreter{&config});
{
py::scoped_interpreter p{&config};
REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast<int>() == 42);
}
py::initialize_interpreter();
}

TEST_CASE("Custom PyConfig with argv") {
py::finalize_interpreter();
{
PyConfig config;
PyConfig_InitIsolatedConfig(&config);
char *argv[] = {strdup("a.out")};
py::scoped_interpreter argv_scope{&config, 1, argv};
std::free(argv[0]);
auto module = py::module::import("test_interpreter");
auto py_widget = module.attr("DerivedWidget")("The question");
const auto &cpp_widget = py_widget.cast<const Widget &>();
REQUIRE(cpp_widget.argv0() == "a.out");
}
py::initialize_interpreter();
}
#endif

TEST_CASE("Add program dir to path") {
static auto get_sys_path_size = []() -> size_t {
auto sys_path = py::module::import("sys").attr("path");
return py::len(sys_path);
};
static auto validate_path_len = [](size_t default_len) {
#if PY_VERSION_HEX < 0x030A0000
// It seems a value remains in sys.path
// left by the previous call of scoped_interpreter ctor.
REQUIRE(get_sys_path_size() > default_len);
#else
REQUIRE(get_sys_path_size() == default_len + 1);
#endif
};
py::finalize_interpreter();

size_t sys_path_default_size = 0;
{
py::scoped_interpreter scoped_interp{true, 0, nullptr, false};
sys_path_default_size = get_sys_path_size();
}
{
py::scoped_interpreter scoped_interp{}; // expected to append some to sys.path
validate_path_len(sys_path_default_size);
}
#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
{
PyConfig config;
PyConfig_InitPythonConfig(&config);
py::scoped_interpreter scoped_interp{&config}; // expected to append some to sys.path
validate_path_len(sys_path_default_size);
}
#endif
py::initialize_interpreter();
}

bool has_pybind11_internals_builtin() {
auto builtins = py::handle(PyEval_GetBuiltins());
return builtins.contains(PYBIND11_INTERNALS_ID);
Expand Down

0 comments on commit 8869984

Please sign in to comment.