Implement PEP-587 initialization APIs

Python 3.8 implements several new APIs around interpreter configuration
and initialization as defined by PEP-587.

I will want to use many of these APIs in PyOxidizer.

This commit defines the new APIs.

The new APIs are split in CPython source between Include/initconfig.h
and Include/cpython/pylifecycle.h. All of the APIs are outside the
Py_LIMITED_API scope.

We didn't have a pylifecycle.rs for the existing Include/pylifecycle.h.
Assuming we don't want to have different Rust modules distinguish between
the Include/<foo>.h and Include/cpython/<foo>.h variants (which appear
to have been introduced in CPython 3.8 - to help split Py_LIMITED_API I
think), I went ahead and created pylifecycle.rs. Ideally the Rust
modules would reflect the current state of the symbols in the latest
CPython sources. But I left this as a TODO.

To be honest, given the number of introduced symbols, I will be shocked
if there isn't a typo or bug lingering in here somewhere. I attempted
to port every symbol from Include/initconfig.h and
Include/cpython/pylifecycle.h. I attempted to preserve pointer
const/mut: if the source didn't use const, I didn't either. Although I
initially got some of this wrong during implementation and this is an
area that deserves extra review scrutiny.

python3-sys/src/initconfig.rs

@@ -0,0 +1,143 @@
+// This entire module is Python 3.8+ and !Py_LIMITED_API only.
+
+use libc::{c_char, c_int, wchar_t, c_ulong};
+use crate::pyport::Py_ssize_t;
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub enum PyStatusType {
+    _PyStatus_TYPE_OK = 0,
+    _PyStatus_TYPE_ERROR = 1,
+    _PyStatus_TYPE_EXIT = 2,
+}
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct PyStatus {
+    _type: PyStatusType,
+    func: *const c_char,
+    err_msg: *const c_char,
+    exitcode: c_int,
+}
+
+#[cfg_attr(windows, link(name = "pythonXY"))]
+extern "C" {
+    pub fn PyStatus_Ok() -> PyStatus;
+    pub fn PyStatus_Error(err_msg: *const c_char) -> PyStatus;
+    pub fn PyStatus_NoMemory() -> PyStatus;
+    pub fn PyStatus_Exit(exitcode: c_int) -> PyStatus;
+
+    pub fn PyStatus_IsError(err: PyStatus) -> c_int;
+    pub fn PyStatus_IsExit(err: PyStatus) -> c_int;
+    pub fn PyStatus_Exception(err: PyStatus) -> c_int;
+}
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct PyWideStringList {
+    length: Py_ssize_t,
+    items: *mut *mut wchar_t,
+}
+
+#[cfg_attr(windows, link(name = "pythonXY"))]
+extern "C" {
+    pub fn PyWideStringList_Append(list: *mut PyWideStringList, item: *const wchar_t) -> PyStatus;
+    pub fn PyWideStringList_Insert(list: *mut PyWideStringList, index: Py_ssize_t, item: *const wchar_t) -> PyStatus;
+}
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct PyPreConfig {
+    _config_init: c_int,
+    parse_argv: c_int,
+    isolated: c_int,
+    use_environment: c_int,
+    configure_locale: c_int,
+    coerce_c_locale: c_int,
+    coerce_c_locale_warn: c_int,
+    #[cfg(windows)]
+    legacy_windows_fs_encoding: c_int,
+    utf8_mode: c_int,
+    dev_mode: c_int,
+    allocator: c_int,
+}
+
+#[cfg_attr(windows, link(name = "pythonXY"))]
+extern "C" {
+    pub fn PyPreConfig_InitPythonConfig(config: *mut PyPreConfig) -> ();
+    pub fn PyPreConfig_InitIsolatedConfig(config: *mut PyPreConfig) -> ();
+}
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct PyConfig {
+    _config_init: c_int,
+    isolated: c_int,
+    use_environment: c_int,
+    dev_mode: c_int,
+    install_signal_handlers: c_int,
+    use_hash_seed: c_int,
+    hash_seed: c_ulong,
+    faulthandler: c_int,
+    tracemalloc: c_int,
+    import_time: c_int,
+    show_ref_count: c_int,
+    show_alloc_count: c_int,
+    dump_refs: c_int,
+    malloc_stats: c_int,
+    filesystem_encoding: *mut wchar_t,
+    filesystem_errors: *mut wchar_t,
+    pycache_prefix: *mut wchar_t,
+    parse_argv: c_int,
+    argv: PyWideStringList,
+    program_name: *mut wchar_t,
+    xoptions: PyWideStringList,
+    warnoptions: PyWideStringList,
+    site_import: c_int,
+    bytes_warning: c_int,
+    inspect: c_int,
+    interactive: c_int,
+    optimization_level: c_int,
+    parser_debug: c_int,
+    write_bytecode: c_int,
+    verbose: c_int,
+    quiet: c_int,
+    user_site_directory: c_int,
+    configure_c_stdio: c_int,
+    buffered_stdio: c_int,
+    stdio_encoding: *mut wchar_t,
+    stdio_errors: *mut wchar_t,
+    #[cfg(windows)]
+    legacy_windows_stdio: c_int,
+    check_hash_pycs_mode: *mut wchar_t,
+    pathconfig_warnings: c_int,
+    pythonpath_env: *mut wchar_t,
+    home: *mut wchar_t,
+    module_search_paths_set: c_int,
+    module_search_paths: PyWideStringList,
+    executable: *mut wchar_t,
+    base_executable: *mut wchar_t,
+    prefix: *mut wchar_t,
+    base_prefix: *mut wchar_t,
+    exec_prefix: *mut wchar_t,
+    base_exec_prefix: *mut wchar_t,
+    skip_source_first_line: c_int,
+    run_command: *mut wchar_t,
+    run_module: *mut wchar_t,
+    run_filename: *mut wchar_t,
+    _install_importlib: c_int,
+    _init_main: c_int,
+}
+
+#[cfg_attr(windows, link(name = "pythonXY"))]
+extern "C" {
+    pub fn PyConfig_InitPythonConfig(config: *mut PyConfig) -> ();
+    pub fn PyConfig_InitIsolatedConfig(config: *mut PyConfig) -> ();
+    pub fn PyConfig_Clear(config: *mut PyConfig) -> ();
+    pub fn PyConfig_SetString(config: *mut PyConfig, config_str: *mut *mut wchar_t, value: *const wchar_t) -> PyStatus;
+    pub fn PyConfig_SetBytesString(config: *mut PyConfig, config_str: *mut *mut wchar_t, value: *const c_char) -> PyStatus;
+    pub fn PyConfig_Read(config: *mut PyConfig) -> PyStatus;
+    pub fn PyConfig_SetBytesArgv(config: *mut PyConfig, argc: Py_ssize_t, argv: *mut *const c_char) -> PyStatus;
+    pub fn PyConfig_SetArgv(config: *mut PyConfig, argc: Py_ssize_t, argv: *mut *const wchar_t) -> PyStatus;
+    pub fn PyConfig_SetWideStringList(config: *mut PyConfig, list: *mut PyWideStringList, length: Py_ssize_t, items: *mut *mut wchar_t) -> PyStatus;
+}

python3-sys/src/lib.rs

@@ -28,6 +28,8 @@ pub use crate::fileutils::*;
 pub use crate::floatobject::*;
 pub use crate::frameobject::PyFrameObject;
 pub use crate::import::*;
+#[cfg(all(Py_3_8, not(Py_LIMITED_API)))]
+pub use crate::initconfig::*;
 pub use crate::intrcheck::*;
 pub use crate::iterobject::*;
 pub use crate::listobject::*;
@@ -48,6 +50,7 @@ pub use crate::pydebug::*;
 pub use crate::pyerrors::*;
 #[cfg(Py_3_4)]
 pub use crate::pyhash::*;
+pub use crate::pylifecycle::*;
 pub use crate::pymem::*;
 pub use crate::pyport::*;
 pub use crate::pystate::*;
@@ -214,8 +217,7 @@ mod modsupport;
 // TODO some functions need to be moved to pylifecycle
 mod pythonrun;
 
-// TODO new in 3.5
-// mod pylifecycle;
+mod pylifecycle;
 
 // TODO supports PEP-384 only; needs adjustment for Python 3.3 and 3.5
 mod ceval;
@@ -284,3 +286,6 @@ pub mod frameobject {
 }
 
 mod marshal;
+
+#[cfg(all(Py_3_8, not(Py_LIMITED_API)))]
+mod initconfig;

python3-sys/src/pylifecycle.rs

@@ -0,0 +1,44 @@
+// There are 2 pylifecycle.h in CPython. We currently only define the Py_LIMITED_API
+// symbols because other symbols exist in their legacy locations in other modules.
+
+use libc::{c_char, c_int, c_void, wchar_t, FILE};
+use crate::initconfig::{PyPreConfig, PyStatus, PyConfig};
+use crate::object::PyObject;
+use crate::pyport::Py_ssize_t;
+
+// Symbols from Include/pylifecycle.h
+
+// TODO move these symbols from their legacy locations into this module.
+
+// Symbols from Include/cpython/pylifecycle.h
+
+#[cfg(Py_3_8)]
+#[cfg(not(Py_LIMITED_API))]
+#[cfg_attr(windows, link(name = "pythonXY"))]
+extern "C" {
+    pub fn Py_PreInitialize(src_config: *const PyPreConfig) -> PyStatus;
+    pub fn Py_PreInitializeFromBytesArgs(src_config: *const PyPreConfig, argc: Py_ssize_t, argv: *mut *mut c_char) -> PyStatus;
+    pub fn Py_PreInitializeFromArgs(src_config: *const PyPreConfig, argc: Py_ssize_t, argv: *mut *mut wchar_t) -> PyStatus;
+
+    pub fn _Py_IsCoreInitialized() -> c_int;
+
+    pub fn Py_InitializeFromConfig(config: *const PyConfig) -> PyStatus;
+    pub fn _Py_InitializeFromArgs(config: *const PyConfig, argc: Py_ssize_t, argv: *mut *const c_char) -> PyStatus;
+    pub fn _Py_InitializeFromWideArgs(config: *const PyConfig, argc: Py_ssize_t, argv: *mut *const wchar_t) -> PyStatus;
+    pub fn _Py_InitializeMain() -> PyStatus;
+    pub fn Py_RunMain() -> c_int;
+
+    pub fn Py_ExitStatusException(err: PyStatus) -> ();
+    pub fn _Py_PyAtExit(func: Option<extern "C" fn(obj: *mut PyObject) -> ()>, module: *mut PyObject) -> ();
+    pub fn _Py_RestoreSignals() -> ();
+    pub fn Py_FdIsInteractive(file: *mut FILE, filename: *const c_char) -> c_int;
+    pub fn _Py_SetProgramFullPath(path: *const wchar_t) -> ();
+    pub fn _Py_gitidentifier() -> *const c_char;
+    pub fn _Py_gitversion() -> *const c_char;
+    pub fn _Py_IsFinalizing() -> c_int;
+    pub fn _PyOS_URandom(buffer: *mut c_void, size: Py_ssize_t) -> c_int;
+    pub fn _PyOS_URandomNonblock(buffer: *mut c_void, size: Py_ssize_t) -> c_int;
+    pub fn _Py_CoerceLegacyLocale(warn: c_int) -> c_int;
+    pub fn _Py_LegacyLocaleDetected(warn: c_int) -> c_int;
+    pub fn _Py_SetLocaleFromEnv(category: c_int) -> *mut c_char;
+}