Skip to content

Commit

Permalink
Add hard coded well-known sysconfigs for easier cross compiling
Browse files Browse the repository at this point in the history
  • Loading branch information
messense committed May 4, 2022
1 parent e9b8838 commit 7566a0d
Show file tree
Hide file tree
Showing 9 changed files with 672 additions and 78 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,16 @@ jobs:
rustup target add aarch64-unknown-linux-gnu
rustup target add aarch64-unknown-linux-musl
rustup target add aarch64-apple-darwin
# abi3
cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig
cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-musl --zig
cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-apple-darwin --zig
# Check wheels with twine
twine check --strict test-crates/pyo3-pure/target/wheels/*.whl
# non-abi3
cargo run -- build --no-sdist -i python3.9 -m test-crates/pyo3-mixed/Cargo.toml --target aarch64-unknown-linux-gnu --zig
- name: test cross compiling windows abi3 wheel
if: matrix.os == 'ubuntu-latest'
run: |
Expand Down
143 changes: 103 additions & 40 deletions src/build_options.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::auditwheel::PlatformTag;
use crate::build_context::{BridgeModel, ProjectLayout};
use crate::cross_compile::{find_sysconfigdata, parse_sysconfigdata};
use crate::python_interpreter::{InterpreterKind, MINIMUM_PYTHON_MINOR};
use crate::python_interpreter::{InterpreterConfig, InterpreterKind, MINIMUM_PYTHON_MINOR};
use crate::BuildContext;
use crate::CargoToml;
use crate::Metadata21;
Expand Down Expand Up @@ -619,6 +619,30 @@ fn find_single_python_interpreter(
Ok(interpreter)
}

fn find_host_interpreter(
bridge: &BridgeModel,
interpreter: &[PathBuf],
target: &Target,
min_python_minor: Option<usize>,
) -> Result<Vec<PythonInterpreter>> {
let interpreters = if !interpreter.is_empty() {
PythonInterpreter::check_executables(interpreter, target, bridge)
.context("The given list of python interpreters is invalid")?
} else {
PythonInterpreter::find_all(target, bridge, min_python_minor)
.context("Finding python interpreters failed")?
};

if interpreters.is_empty() {
if let Some(minor) = min_python_minor {
bail!("Couldn't find any python interpreters with version >= 3.{}. Please specify at least one with -i", minor);
} else {
bail!("Couldn't find any python interpreters. Please specify at least one with -i");
}
}
Ok(interpreters)
}

/// Finds the appropriate amount for python versions for each [BridgeModel].
///
/// This means all for bindings, one for cffi and zero for bin.
Expand All @@ -631,26 +655,13 @@ pub fn find_interpreter(
) -> Result<Vec<PythonInterpreter>> {
match bridge {
BridgeModel::Bindings(binding_name, _) => {
let mut interpreter = if !interpreter.is_empty() {
PythonInterpreter::check_executables(interpreter, target, bridge)
.context("The given list of python interpreters is invalid")?
} else {
PythonInterpreter::find_all(target, bridge, min_python_minor)
.context("Finding python interpreters failed")?
};

if interpreter.is_empty() {
if let Some(minor) = min_python_minor {
bail!("Couldn't find any python interpreters with version >= 3.{}. Please specify at least one with -i", minor);
} else {
bail!("Couldn't find any python interpreters. Please specify at least one with -i");
}
}

let mut interpreters = Vec::new();
if binding_name.starts_with("pyo3") && target.is_unix() && target.cross_compiling() {
if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") {
let host_interpreters =
find_host_interpreter(bridge, interpreter, target, min_python_minor)?;
println!("⚠️ Cross-compiling is poorly supported");
let host_python = &interpreter[0];
let host_python = &host_interpreters[0];
println!(
"🐍 Using host {} for cross-compiling preparation",
host_python
Expand Down Expand Up @@ -699,31 +710,77 @@ pub fn find_interpreter(
}
})
.context("unsupported Python interpreter")?;
interpreter = vec![PythonInterpreter {
major,
minor,
abiflags,
interpreters.push(PythonInterpreter {
config: InterpreterConfig {
major,
minor,
interpreter_kind,
abiflags,
ext_suffix: ext_suffix.to_string(),
abi_tag,
calcsize_pointer: None,
},
target: target.clone(),
executable: PathBuf::new(),
ext_suffix: ext_suffix.to_string(),
interpreter_kind,
abi_tag,
platform: None,
runnable: false,
}];
});
} else {
if interpreter.is_empty() {
bail!("Couldn't find any python interpreters. Please specify at least one with -i");
}
for interp in interpreter {
let python = interp
.file_name()
.context("Invalid python interpreter")?
.to_string_lossy();
let (python_impl, python_ver) =
if let Some(ver) = python.strip_prefix("pypy") {
(InterpreterKind::PyPy, ver)
} else if let Some(ver) = python.strip_prefix("python") {
(InterpreterKind::CPython, ver)
} else {
bail!("Unsupported Python interpreter: {}", python);
};
let mut ver_parts = python_ver.split('.');
let ver_major = ver_parts
.next()
.context("Couldn't parse major version")?
.parse::<usize>()
.context("Expect a digit")?;
let ver_minor = ver_parts
.next()
.context("Couldn't parse minor version")?
.parse::<usize>()
.context("Expect a digit")?;
let sysconfig = InterpreterConfig::lookup(
target.target_os(),
target.target_arch(),
python_impl,
(ver_major, ver_minor),
)
.context("Failed to find a python interpreter")?;
interpreters.push(PythonInterpreter::from_config(
sysconfig.clone(),
target.clone(),
));
}
}
} else {
interpreters =
find_host_interpreter(bridge, interpreter, target, min_python_minor)?;
}

println!(
"🐍 Found {}",
interpreter
interpreters
.iter()
.map(ToString::to_string)
.collect::<Vec<String>>()
.join(", ")
);

Ok(interpreter)
Ok(interpreters)
}
BridgeModel::Cffi => {
let interpreter = find_single_python_interpreter(bridge, interpreter, target, "cffi")?;
Expand All @@ -748,14 +805,17 @@ pub fn find_interpreter(
// when cross compiling, so we fake a python interpreter matching it
println!("⚠️ Cross-compiling is poorly supported");
Ok(vec![PythonInterpreter {
major: *major as usize,
minor: *minor as usize,
abiflags: "".to_string(),
config: InterpreterConfig {
major: *major as usize,
minor: *minor as usize,
interpreter_kind: InterpreterKind::CPython,
abiflags: "".to_string(),
ext_suffix: ".pyd".to_string(),
abi_tag: None,
calcsize_pointer: None,
},
target: target.clone(),
executable: PathBuf::new(),
ext_suffix: ".pyd".to_string(),
interpreter_kind: InterpreterKind::CPython,
abi_tag: None,
platform: None,
runnable: false,
}])
Expand All @@ -766,14 +826,17 @@ pub fn find_interpreter(
println!("🐍 Not using a specific python interpreter (Automatically generating windows import library)");
// fake a python interpreter
Ok(vec![PythonInterpreter {
major: *major as usize,
minor: *minor as usize,
abiflags: "".to_string(),
config: InterpreterConfig {
major: *major as usize,
minor: *minor as usize,
interpreter_kind: InterpreterKind::CPython,
abiflags: "".to_string(),
ext_suffix: ".pyd".to_string(),
abi_tag: None,
calcsize_pointer: None,
},
target: target.clone(),
executable: PathBuf::new(),
ext_suffix: ".pyd".to_string(),
interpreter_kind: InterpreterKind::CPython,
abi_tag: None,
platform: None,
runnable: false,
}])
Expand Down
29 changes: 25 additions & 4 deletions src/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ fn compile_target(
bindings_crate: &BridgeModel,
) -> Result<HashMap<String, PathBuf>> {
let target = &context.target;

let mut shared_args: Vec<_> = context
.cargo_extra_args
.iter()
Expand Down Expand Up @@ -238,15 +239,35 @@ fn compile_target(
}
}

if let Some(python_interpreter) = python_interpreter {
// Setup `PYO3_CONFIG_FILE` if we are cross compiling for pyo3 bindings
if let Some(interpreter) = python_interpreter {
// Target python interpreter isn't runnable when cross compiling
if python_interpreter.runnable {
if interpreter.runnable {
if bindings_crate.is_bindings("pyo3") || bindings_crate.is_bindings("pyo3-ffi") {
build_command.env("PYO3_PYTHON", &python_interpreter.executable);
build_command.env("PYO3_PYTHON", &interpreter.executable);
}

// rust-cpython, and legacy pyo3 versions
build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable);
build_command.env("PYTHON_SYS_EXECUTABLE", &interpreter.executable);
} else if (bindings_crate.is_bindings("pyo3") || bindings_crate.is_bindings("pyo3-ffi"))
&& env::var_os("PYO3_CONFIG_FILE").is_none()
{
let pyo3_config = interpreter.pyo3_config_file();
let maturin_target_dir = context.target_dir.join("maturin");
let config_file = maturin_target_dir.join(format!(
"pyo3-config-{}-{}.{}.txt",
target.target_triple(),
interpreter.major,
interpreter.minor
));
fs::create_dir_all(&maturin_target_dir)?;
fs::write(&config_file, pyo3_config).with_context(|| {
format!(
"Failed to create pyo3 config file at '{}'",
config_file.display()
)
})?;
build_command.env("PYO3_CONFIG_FILE", config_file);
}
}

Expand Down
100 changes: 100 additions & 0 deletions src/python_interpreter/config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
use super::InterpreterKind;
use crate::target::{Arch, Os};
use once_cell::sync::Lazy;
use serde::Deserialize;
use std::collections::HashMap;

/// Wellknown Python interpreter sysconfig values
static WELLKNOWN_SYSCONFIG: Lazy<HashMap<Os, HashMap<Arch, Vec<InterpreterConfig>>>> =
Lazy::new(|| {
let mut sysconfig = HashMap::new();
let sysconfig_linux = serde_json::from_slice(include_bytes!("sysconfig-linux.json"))
.expect("invalid sysconfig.json");
sysconfig.insert(Os::Linux, sysconfig_linux);
sysconfig
});

/// Some of the sysconfigdata of Python interpreter we care about
#[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
pub struct InterpreterConfig {
/// Python's major version
pub major: usize,
/// Python's minor version
pub minor: usize,
/// cpython or pypy
#[serde(rename = "interpreter")]
pub interpreter_kind: InterpreterKind,
/// For linux and mac, this contains the value of the abiflags, e.g. "m"
/// for python3.7m or "dm" for python3.6dm. Since python3.8, the value is
/// empty. On windows, the value was always None.
///
/// See PEP 261 and PEP 393 for details
pub abiflags: String,
/// Suffix to use for extension modules as given by sysconfig.
pub ext_suffix: String,
/// Part of sysconfig's SOABI specifying {major}{minor}{abiflags}
///
/// Note that this always `None` on windows
pub abi_tag: Option<String>,
/// Pointer width
pub calcsize_pointer: Option<usize>,
}

impl InterpreterConfig {
/// Lookup a wellknown sysconfig for a given Python interpreter
pub fn lookup(
os: Os,
arch: Arch,
python_impl: InterpreterKind,
python_version: (usize, usize),
) -> Option<&'static Self> {
let (major, minor) = python_version;
if let Some(os_sysconfigs) = WELLKNOWN_SYSCONFIG.get(&os) {
if let Some(sysconfigs) = os_sysconfigs.get(&arch) {
return sysconfigs.iter().find(|s| {
s.interpreter_kind == python_impl && s.major == major && s.minor == minor
});
}
}
None
}

/// Generate pyo3 config file content
pub fn pyo3_config_file(&self) -> String {
let mut content = format!(
r#"implementation={implementation}
version={major}.{minor}
shared=true
abi3=false
build_flags=WITH_THREAD
suppress_build_script_link_lines=false"#,
implementation = self.interpreter_kind,
major = self.major,
minor = self.minor,
);
if let Some(pointer_width) = self.calcsize_pointer {
content.push_str(&format!("\npointer_width={}", pointer_width * 8));
}
content
}
}

#[cfg(test)]
mod test {
use super::*;

#[test]
fn test_load_sysconfig() {
let linux_sysconfig = WELLKNOWN_SYSCONFIG.get(&Os::Linux).unwrap();
assert!(linux_sysconfig.contains_key(&Arch::X86_64));
}

#[test]
fn test_pyo3_config_file() {
let sysconfig =
InterpreterConfig::lookup(Os::Linux, Arch::X86_64, InterpreterKind::CPython, (3, 10))
.unwrap();
let config_file = sysconfig.pyo3_config_file();
assert_eq!(config_file, "implementation=CPython\nversion=3.10\nshared=true\nabi3=false\nbuild_flags=WITH_THREAD\nsuppress_build_script_link_lines=false\npointer_width=64");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import platform
import sys
import sysconfig
import struct

if platform.python_implementation() == "PyPy":
# Workaround for PyPy 3.6 on windows:
Expand All @@ -28,6 +29,8 @@
"platform": sysconfig.get_platform(),
# This one isn't technically necessary, but still very useful for sanity checks
"system": platform.system().lower(),
# This one is for generating a config file for pyo3
"calcsize_pointer": struct.calcsize("P"),
}

print(json.dumps(metadata))
Loading

0 comments on commit 7566a0d

Please sign in to comment.