diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15cd1bd99..45198d8a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,11 +105,19 @@ jobs: rustup target add aarch64-unknown-linux-gnu rustup target add aarch64-unknown-linux-musl rustup target add aarch64-apple-darwin + + # abi3 cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-musl --zig cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-apple-darwin --zig # Check wheels with twine twine check --strict test-crates/pyo3-pure/target/wheels/*.whl + + # non-abi3 + cargo run -- build --no-sdist -i python3.9 -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-unknown-linux-gnu --zig + cargo run -- build --no-sdist -i python3.9 -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-apple-darwin --zig + # Check wheels with twine + twine check --strict test-crates/pyo3-mixed/target/wheels/*.whl - name: test cross compiling windows abi3 wheel if: matrix.os == 'ubuntu-latest' run: | diff --git a/Changelog.md b/Changelog.md index a9f3aa734..710106cbb 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,7 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] * Re-export `__all__` for pure Rust projects in [#886](https://github.com/PyO3/maturin/pull/886) -* Stop setting `RUSTFLAGS` environment variable to an empty string [#887](https://github.com/PyO3/maturin/pull/887) +* Stop setting `RUSTFLAGS` environment variable to an empty string in [#887](https://github.com/PyO3/maturin/pull/887) +* Add hardcoded well-known sysconfigs for effortless cross compiling in [#896](https://github.com/PyO3/maturin/pull/896) ## [0.12.14] - 2022-04-25 diff --git a/src/build_options.rs b/src/build_options.rs index 7588e78c6..1fd90d611 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -1,7 +1,7 @@ use crate::auditwheel::PlatformTag; use crate::build_context::{BridgeModel, ProjectLayout}; use crate::cross_compile::{find_sysconfigdata, parse_sysconfigdata}; -use crate::python_interpreter::{InterpreterKind, MINIMUM_PYTHON_MINOR}; +use crate::python_interpreter::{InterpreterConfig, InterpreterKind, MINIMUM_PYTHON_MINOR}; use crate::BuildContext; use crate::CargoToml; use crate::Metadata21; @@ -619,6 +619,30 @@ fn find_single_python_interpreter( Ok(interpreter) } +fn find_host_interpreter( + bridge: &BridgeModel, + interpreter: &[PathBuf], + target: &Target, + min_python_minor: Option, +) -> Result> { + let interpreters = if !interpreter.is_empty() { + PythonInterpreter::check_executables(interpreter, target, bridge) + .context("The given list of python interpreters is invalid")? + } else { + PythonInterpreter::find_all(target, bridge, min_python_minor) + .context("Finding python interpreters failed")? + }; + + if interpreters.is_empty() { + if let Some(minor) = min_python_minor { + bail!("Couldn't find any python interpreters with version >= 3.{}. Please specify at least one with -i", minor); + } else { + bail!("Couldn't find any python interpreters. Please specify at least one with -i"); + } + } + Ok(interpreters) +} + /// Finds the appropriate amount for python versions for each [BridgeModel]. /// /// This means all for bindings, one for cffi and zero for bin. @@ -631,26 +655,13 @@ pub fn find_interpreter( ) -> Result> { match bridge { BridgeModel::Bindings(binding_name, _) => { - let mut interpreter = if !interpreter.is_empty() { - PythonInterpreter::check_executables(interpreter, target, bridge) - .context("The given list of python interpreters is invalid")? - } else { - PythonInterpreter::find_all(target, bridge, min_python_minor) - .context("Finding python interpreters failed")? - }; - - if interpreter.is_empty() { - if let Some(minor) = min_python_minor { - bail!("Couldn't find any python interpreters with version >= 3.{}. Please specify at least one with -i", minor); - } else { - bail!("Couldn't find any python interpreters. Please specify at least one with -i"); - } - } - + let mut interpreters = Vec::new(); if binding_name.starts_with("pyo3") && target.is_unix() && target.cross_compiling() { if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") { + let host_interpreters = + find_host_interpreter(bridge, interpreter, target, min_python_minor)?; println!("⚠️ Cross-compiling is poorly supported"); - let host_python = &interpreter[0]; + let host_python = &host_interpreters[0]; println!( "🐍 Using host {} for cross-compiling preparation", host_python @@ -699,31 +710,81 @@ pub fn find_interpreter( } }) .context("unsupported Python interpreter")?; - interpreter = vec![PythonInterpreter { - major, - minor, - abiflags, + interpreters.push(PythonInterpreter { + config: InterpreterConfig { + major, + minor, + interpreter_kind, + abiflags, + ext_suffix: ext_suffix.to_string(), + abi_tag, + calcsize_pointer: None, + }, target: target.clone(), executable: PathBuf::new(), - ext_suffix: ext_suffix.to_string(), - interpreter_kind, - abi_tag, platform: None, runnable: false, - }]; + }); + } else { + if interpreter.is_empty() { + bail!("Couldn't find any python interpreters. Please specify at least one with -i"); + } + for interp in interpreter { + let python = interp + .file_name() + .context("Invalid python interpreter")? + .to_string_lossy(); + let (python_impl, python_ver) = + if let Some(ver) = python.strip_prefix("pypy") { + (InterpreterKind::PyPy, ver) + } else if let Some(ver) = python.strip_prefix("python") { + (InterpreterKind::CPython, ver) + } else { + bail!("Unsupported Python interpreter: {}", python); + }; + let (ver_major, ver_minor) = python_ver + .split_once('.') + .context("Invalid python interpreter version")?; + let ver_major = ver_major.parse::().with_context(|| { + format!( + "Invalid python interpreter major version '{}', expect a digit", + ver_major + ) + })?; + let ver_minor = ver_minor.parse::().with_context(|| { + format!( + "Invalid python interpreter minor version '{}', expect a digit", + ver_minor + ) + })?; + let sysconfig = InterpreterConfig::lookup( + target.target_os(), + target.target_arch(), + python_impl, + (ver_major, ver_minor), + ) + .context("Failed to find a python interpreter")?; + interpreters.push(PythonInterpreter::from_config( + sysconfig.clone(), + target.clone(), + )); + } } + } else { + interpreters = + find_host_interpreter(bridge, interpreter, target, min_python_minor)?; } println!( "🐍 Found {}", - interpreter + interpreters .iter() .map(ToString::to_string) .collect::>() .join(", ") ); - Ok(interpreter) + Ok(interpreters) } BridgeModel::Cffi => { let interpreter = find_single_python_interpreter(bridge, interpreter, target, "cffi")?; @@ -748,14 +809,17 @@ pub fn find_interpreter( // when cross compiling, so we fake a python interpreter matching it println!("⚠️ Cross-compiling is poorly supported"); Ok(vec![PythonInterpreter { - major: *major as usize, - minor: *minor as usize, - abiflags: "".to_string(), + config: InterpreterConfig { + major: *major as usize, + minor: *minor as usize, + interpreter_kind: InterpreterKind::CPython, + abiflags: "".to_string(), + ext_suffix: ".pyd".to_string(), + abi_tag: None, + calcsize_pointer: None, + }, target: target.clone(), executable: PathBuf::new(), - ext_suffix: ".pyd".to_string(), - interpreter_kind: InterpreterKind::CPython, - abi_tag: None, platform: None, runnable: false, }]) @@ -766,14 +830,17 @@ pub fn find_interpreter( println!("🐍 Not using a specific python interpreter (Automatically generating windows import library)"); // fake a python interpreter Ok(vec![PythonInterpreter { - major: *major as usize, - minor: *minor as usize, - abiflags: "".to_string(), + config: InterpreterConfig { + major: *major as usize, + minor: *minor as usize, + interpreter_kind: InterpreterKind::CPython, + abiflags: "".to_string(), + ext_suffix: ".pyd".to_string(), + abi_tag: None, + calcsize_pointer: None, + }, target: target.clone(), executable: PathBuf::new(), - ext_suffix: ".pyd".to_string(), - interpreter_kind: InterpreterKind::CPython, - abi_tag: None, platform: None, runnable: false, }]) diff --git a/src/compile.rs b/src/compile.rs index ef75dfd24..cb64108fe 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -110,6 +110,7 @@ fn compile_target( bindings_crate: &BridgeModel, ) -> Result> { let target = &context.target; + let mut shared_args: Vec<_> = context .cargo_extra_args .iter() @@ -238,15 +239,35 @@ fn compile_target( } } - if let Some(python_interpreter) = python_interpreter { + // Setup `PYO3_CONFIG_FILE` if we are cross compiling for pyo3 bindings + if let Some(interpreter) = python_interpreter { // Target python interpreter isn't runnable when cross compiling - if python_interpreter.runnable { + if interpreter.runnable { if bindings_crate.is_bindings("pyo3") || bindings_crate.is_bindings("pyo3-ffi") { - build_command.env("PYO3_PYTHON", &python_interpreter.executable); + build_command.env("PYO3_PYTHON", &interpreter.executable); } // rust-cpython, and legacy pyo3 versions - build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable); + build_command.env("PYTHON_SYS_EXECUTABLE", &interpreter.executable); + } else if (bindings_crate.is_bindings("pyo3") || bindings_crate.is_bindings("pyo3-ffi")) + && env::var_os("PYO3_CONFIG_FILE").is_none() + { + let pyo3_config = interpreter.pyo3_config_file(); + let maturin_target_dir = context.target_dir.join("maturin"); + let config_file = maturin_target_dir.join(format!( + "pyo3-config-{}-{}.{}.txt", + target.target_triple(), + interpreter.major, + interpreter.minor + )); + fs::create_dir_all(&maturin_target_dir)?; + fs::write(&config_file, pyo3_config).with_context(|| { + format!( + "Failed to create pyo3 config file at '{}'", + config_file.display() + ) + })?; + build_command.env("PYO3_CONFIG_FILE", config_file); } } diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs new file mode 100644 index 000000000..f18e56165 --- /dev/null +++ b/src/python_interpreter/config.rs @@ -0,0 +1,103 @@ +use super::InterpreterKind; +use crate::target::{Arch, Os}; +use once_cell::sync::Lazy; +use serde::Deserialize; +use std::collections::HashMap; + +/// Wellknown Python interpreter sysconfig values +static WELLKNOWN_SYSCONFIG: Lazy>>> = + Lazy::new(|| { + let mut sysconfig = HashMap::new(); + let sysconfig_linux = serde_json::from_slice(include_bytes!("sysconfig-linux.json")) + .expect("invalid sysconfig-linux.json"); + sysconfig.insert(Os::Linux, sysconfig_linux); + let sysconfig_macos = serde_json::from_slice(include_bytes!("sysconfig-macos.json")) + .expect("invalid sysconfig-macos.json"); + sysconfig.insert(Os::Macos, sysconfig_macos); + sysconfig + }); + +/// Some of the sysconfigdata of Python interpreter we care about +#[derive(Debug, Clone, Deserialize, Eq, PartialEq)] +pub struct InterpreterConfig { + /// Python's major version + pub major: usize, + /// Python's minor version + pub minor: usize, + /// cpython or pypy + #[serde(rename = "interpreter")] + pub interpreter_kind: InterpreterKind, + /// For linux and mac, this contains the value of the abiflags, e.g. "m" + /// for python3.7m or "dm" for python3.6dm. Since python3.8, the value is + /// empty. On windows, the value was always None. + /// + /// See PEP 261 and PEP 393 for details + pub abiflags: String, + /// Suffix to use for extension modules as given by sysconfig. + pub ext_suffix: String, + /// Part of sysconfig's SOABI specifying {major}{minor}{abiflags} + /// + /// Note that this always `None` on windows + pub abi_tag: Option, + /// Pointer width + pub calcsize_pointer: Option, +} + +impl InterpreterConfig { + /// Lookup a wellknown sysconfig for a given Python interpreter + pub fn lookup( + os: Os, + arch: Arch, + python_impl: InterpreterKind, + python_version: (usize, usize), + ) -> Option<&'static Self> { + let (major, minor) = python_version; + if let Some(os_sysconfigs) = WELLKNOWN_SYSCONFIG.get(&os) { + if let Some(sysconfigs) = os_sysconfigs.get(&arch) { + return sysconfigs.iter().find(|s| { + s.interpreter_kind == python_impl && s.major == major && s.minor == minor + }); + } + } + None + } + + /// Generate pyo3 config file content + pub fn pyo3_config_file(&self) -> String { + let mut content = format!( + r#"implementation={implementation} +version={major}.{minor} +shared=true +abi3=false +build_flags=WITH_THREAD +suppress_build_script_link_lines=false"#, + implementation = self.interpreter_kind, + major = self.major, + minor = self.minor, + ); + if let Some(pointer_width) = self.calcsize_pointer { + content.push_str(&format!("\npointer_width={}", pointer_width * 8)); + } + content + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_load_sysconfig() { + let linux_sysconfig = WELLKNOWN_SYSCONFIG.get(&Os::Linux).unwrap(); + assert!(linux_sysconfig.contains_key(&Arch::X86_64)); + } + + #[test] + fn test_pyo3_config_file() { + let sysconfig = + InterpreterConfig::lookup(Os::Linux, Arch::X86_64, InterpreterKind::CPython, (3, 10)) + .unwrap(); + let config_file = sysconfig.pyo3_config_file(); + assert_eq!(config_file, "implementation=CPython\nversion=3.10\nshared=true\nabi3=false\nbuild_flags=WITH_THREAD\nsuppress_build_script_link_lines=false\npointer_width=64"); + } +} diff --git a/src/get_interpreter_metadata.py b/src/python_interpreter/get_interpreter_metadata.py similarity index 90% rename from src/get_interpreter_metadata.py rename to src/python_interpreter/get_interpreter_metadata.py index 5c63488be..61b17acd6 100644 --- a/src/get_interpreter_metadata.py +++ b/src/python_interpreter/get_interpreter_metadata.py @@ -2,6 +2,7 @@ import platform import sys import sysconfig +import struct if platform.python_implementation() == "PyPy": # Workaround for PyPy 3.6 on windows: @@ -28,6 +29,8 @@ "platform": sysconfig.get_platform(), # This one isn't technically necessary, but still very useful for sanity checks "system": platform.system().lower(), + # This one is for generating a config file for pyo3 + "calcsize_pointer": struct.calcsize("P"), } print(json.dumps(metadata)) diff --git a/src/python_interpreter.rs b/src/python_interpreter/mod.rs similarity index 94% rename from src/python_interpreter.rs rename to src/python_interpreter/mod.rs index 75aee6e47..54826ffc7 100644 --- a/src/python_interpreter.rs +++ b/src/python_interpreter/mod.rs @@ -1,3 +1,4 @@ +pub use self::config::InterpreterConfig; use crate::auditwheel::PlatformTag; use crate::{BridgeModel, Target}; use anyhow::{bail, format_err, Context, Result}; @@ -6,10 +7,13 @@ use serde::Deserialize; use std::collections::HashSet; use std::fmt; use std::io; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::str; +mod config; + /// This snippets will give us information about the python interpreter's /// version and abi as json through stdout const GET_INTERPRETER_METADATA: &str = include_str!("get_interpreter_metadata.py"); @@ -228,7 +232,8 @@ fn find_all_windows(target: &Target, min_python_minor: usize) -> Result, /// Comes from `sysconfig.get_platform()` /// /// Note that this can be `None` when cross compiling @@ -297,6 +287,14 @@ pub struct PythonInterpreter { pub runnable: bool, } +impl Deref for PythonInterpreter { + type Target = InterpreterConfig; + + fn deref(&self) -> &Self::Target { + &self.config + } +} + /// Returns the abiflags that are assembled through the message, with some /// additional sanity checks. /// @@ -515,21 +513,35 @@ impl PythonInterpreter { }; Ok(Some(PythonInterpreter { - major: message.major, - minor: message.minor, - abiflags, + config: InterpreterConfig { + major: message.major, + minor: message.minor, + interpreter_kind: interpreter, + abiflags, + ext_suffix: message + .ext_suffix + .context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?, + abi_tag: message.abi_tag, + calcsize_pointer: None, + }, target: target.clone(), executable: executable.as_ref().to_path_buf(), - ext_suffix: message - .ext_suffix - .context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?, - interpreter_kind: interpreter, - abi_tag: message.abi_tag, platform, runnable: true, })) } + /// Construct a `PythonInterpreter` from a sysconfig and target + pub fn from_config(config: InterpreterConfig, target: Target) -> Self { + PythonInterpreter { + config, + target, + executable: PathBuf::new(), + platform: None, + runnable: false, + } + } + /// Tries to find all installed python versions using the heuristic for the /// given platform pub fn find_all( @@ -645,17 +657,20 @@ impl fmt::Display for PythonInterpreter { write!( f, "{} {}.{}{} at {}", - self.interpreter_kind, - self.major, - self.minor, - self.abiflags, + self.config.interpreter_kind, + self.config.major, + self.config.minor, + self.config.abiflags, self.executable.display() ) } else { write!( f, "cross compiling target {} {}.{}{}", - self.interpreter_kind, self.major, self.minor, self.abiflags, + self.config.interpreter_kind, + self.config.major, + self.config.minor, + self.config.abiflags, ) } } diff --git a/src/python_interpreter/sysconfig-linux.json b/src/python_interpreter/sysconfig-linux.json new file mode 100644 index 000000000..1868f59e0 --- /dev/null +++ b/src/python_interpreter/sysconfig-linux.json @@ -0,0 +1,318 @@ +{ + "x86_64": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-x86_64-linux-gnu.so", + "abi_tag": "36m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-x86_64-linux-gnu.so", + "abi_tag": "37m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-x86_64-linux-gnu.so", + "abi_tag": "38", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-x86_64-linux-gnu.so", + "abi_tag": "39", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-x86_64-linux-gnu.so", + "abi_tag": "310", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy37-pp73-x86_64-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy38-pp73-x86_64-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy39-pp73-x86_64-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + } + ], + "i686": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-i386-linux-gnu.so", + "abi_tag": "36m", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-i386-linux-gnu.so", + "abi_tag": "37m", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-i386-linux-gnu.so", + "abi_tag": "38", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-i386-linux-gnu.so", + "abi_tag": "39", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-i386-linux-gnu.so", + "abi_tag": "310", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 7, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy37-pp73-x86-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy38-pp73-x86-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 4 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy39-pp73-x86-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 4 + } + ], + "aarch64": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-aarch64-linux-gnu.so", + "abi_tag": "36m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-aarch64-linux-gnu.so", + "abi_tag": "37m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-aarch64-linux-gnu.so", + "abi_tag": "38", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-aarch64-linux-gnu.so", + "abi_tag": "39", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-aarch64-linux-gnu.so", + "abi_tag": "310", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy37-pp73-aarch64-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy38-pp73-aarch64-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy39-pp73-aarch64-linux-gnu.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + } + ], + "ppc64le": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-powerpc64le-linux-gnu.so", + "abi_tag": "36m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-powerpc64le-linux-gnu.so", + "abi_tag": "37m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-powerpc64le-linux-gnu.so", + "abi_tag": "38", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-powerpc64le-linux-gnu.so", + "abi_tag": "39", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-powerpc64le-linux-gnu.so", + "abi_tag": "310", + "calcsize_pointer": 8 + } + ], + "s390x": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-s390x-linux-gnu.so", + "abi_tag": "36m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-s390x-linux-gnu.so", + "abi_tag": "37m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-s390x-linux-gnu.so", + "abi_tag": "38", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-s390x-linux-gnu.so", + "abi_tag": "39", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-s390x-linux-gnu.so", + "abi_tag": "310", + "calcsize_pointer": 8 + } + ] +} diff --git a/src/python_interpreter/sysconfig-macos.json b/src/python_interpreter/sysconfig-macos.json new file mode 100644 index 000000000..69956ca93 --- /dev/null +++ b/src/python_interpreter/sysconfig-macos.json @@ -0,0 +1,123 @@ +{ + "x86_64": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-darwin.so", + "abi_tag": "36m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-darwin.so", + "abi_tag": "37m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-darwin.so", + "abi_tag": "38", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-darwin.so", + "abi_tag": "39", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-darwin.so", + "abi_tag": "310", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy37-pp73-darwin.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy38-pp73-darwin.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "pypy", + "ext_suffix": ".pypy39-pp73-darwin.so", + "abi_tag": "pp73", + "calcsize_pointer": 8 + } + ], + "aarch64": [ + { + "major": 3, + "minor": 6, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-36m-darwin.so", + "abi_tag": "36m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 7, + "abiflags": "m", + "interpreter": "cpython", + "ext_suffix": ".cpython-37m-darwin.so", + "abi_tag": "37m", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 8, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-38-darwin.so", + "abi_tag": "38", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 9, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-39-darwin.so", + "abi_tag": "39", + "calcsize_pointer": 8 + }, + { + "major": 3, + "minor": 10, + "abiflags": "", + "interpreter": "cpython", + "ext_suffix": ".cpython-310-darwin.so", + "abi_tag": "310", + "calcsize_pointer": 8 + } + ] +} \ No newline at end of file diff --git a/src/target.rs b/src/target.rs index cff7b5806..29752ee1b 100644 --- a/src/target.rs +++ b/src/target.rs @@ -3,6 +3,7 @@ use crate::python_interpreter::InterpreterKind; use crate::{PlatformTag, PythonInterpreter}; use anyhow::{anyhow, bail, format_err, Context, Result}; use platform_info::*; +use serde::Deserialize; use std::env; use std::fmt; use std::path::Path; @@ -11,8 +12,9 @@ use std::str; use target_lexicon::{Environment, Triple}; /// All supported operating system -#[derive(Debug, Clone, Eq, PartialEq)] -enum Os { +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Os { Linux, Windows, Macos, @@ -41,12 +43,16 @@ impl fmt::Display for Os { } /// All supported CPU architectures -#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum Arch { Aarch64, Armv7L, + #[serde(alias = "ppc64le")] Powerpc64Le, + #[serde(alias = "ppc64")] Powerpc64, + #[serde(alias = "i686")] X86, X86_64, S390X, @@ -348,6 +354,11 @@ impl Target { } } + /// Returns target operating system + pub fn target_os(&self) -> Os { + self.os + } + /// Returns target architecture pub fn target_arch(&self) -> Arch { self.arch diff --git a/sysconfig/generate_manylinux.py b/sysconfig/generate_manylinux.py new file mode 100644 index 000000000..371971610 --- /dev/null +++ b/sysconfig/generate_manylinux.py @@ -0,0 +1,58 @@ +import os +import sys +import json +from collections import defaultdict +import subprocess + + +ARCHES = ["x86_64", "i686", "aarch64", "ppc64le", "s390x"] +PY_VERS = [ + "cp36-cp36m", + "cp37-cp37m", + "cp38-cp38", + "cp39-cp39", + "cp310-cp310", + "pp37-pypy37_pp73", + "pp38-pypy38_pp73", + "pp39-pypy39_pp73", +] + + +def main(): + well_known = defaultdict(list) + cwd = os.getcwd() + for arch in ARCHES: + docker_image = f"quay.io/pypa/manylinux2014_{arch}" + for ver in PY_VERS: + # PyPy is not available on ppc64le & s390x + if arch in ["ppc64le", "s390x"] and ver.startswith("pp"): + continue + command = [ + "docker", + "run", + "--rm", + "-it", + "-v", + f"{cwd}:/io", + "-w", + "/io", + docker_image, + f"/opt/python/{ver}/bin/python", + "/io/src/python_interpreter/get_interpreter_metadata.py", + ] + try: + metadata = subprocess.check_output(command).decode().strip() + except subprocess.CalledProcessError as exc: + print(exc.output, file=sys.stderr) + raise + metadata = json.loads(metadata.splitlines()[-1]) + for key in ["system", "platform"]: + metadata.pop(key, None) + well_known[arch].append(metadata) + + with open("src/python_interpreter/sysconfig-linux.json", "w") as f: + f.write(json.dumps(well_known, indent=2)) + + +if __name__ == "__main__": + main()