diff --git a/src/build_options.rs b/src/build_options.rs index e03864f7b..e28347232 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -70,6 +70,10 @@ pub struct BuildOptions { #[clap(short, long, multiple_values = true, multiple_occurrences = true)] pub interpreter: Vec, + /// Find interpreters from the host machine + #[clap(short = 'F', long, conflicts_with = "interpreter")] + pub find_interpreter: bool, + /// Which kind of bindings to use. Possible values are pyo3, rust-cpython, cffi and bin #[clap(short, long)] pub bindings: Option, @@ -160,6 +164,257 @@ impl BuildOptions { )) } } + + /// Finds the appropriate amount for python versions for each [BridgeModel]. + fn find_interpreters( + &self, + bridge: &BridgeModel, + interpreter: &[PathBuf], + min_python_minor: Option, + generate_import_lib: bool, + ) -> Result> { + let target = &Target::from_target_triple(self.target.clone())?; + match bridge { + BridgeModel::Bindings(binding_name, _) | BridgeModel::Bin(Some((binding_name, _))) => { + let mut native_interpreters = false; + let mut interpreters = Vec::new(); + if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") { + if !binding_name.starts_with("pyo3") { + bail!("Only pyo3 bindings can be configured with PYO3_CONFIG_FILE"); + } + let interpreter_config = + InterpreterConfig::from_pyo3_config(config_file.as_ref(), target) + .context("Invalid PYO3_CONFIG_FILE")?; + interpreters.push(PythonInterpreter::from_config(interpreter_config)); + } else if binding_name.starts_with("pyo3") && target.cross_compiling() { + if let Some(cross_lib_dir) = env::var_os("PYO3_CROSS_LIB_DIR") { + let host_interpreters = find_interpreter_in_host( + bridge, + interpreter, + target, + min_python_minor, + )?; + let host_python = &host_interpreters[0]; + println!( + "🐍 Using host {} for cross-compiling preparation", + host_python + ); + // pyo3 + env::set_var("PYO3_PYTHON", &host_python.executable); + // rust-cpython, and legacy pyo3 versions + env::set_var("PYTHON_SYS_EXECUTABLE", &host_python.executable); + + let sysconfig_path = find_sysconfigdata(cross_lib_dir.as_ref(), target)?; + env::set_var( + "MATURIN_PYTHON_SYSCONFIGDATA_DIR", + sysconfig_path.parent().unwrap(), + ); + + let sysconfig_data = parse_sysconfigdata(host_python, sysconfig_path)?; + let major = sysconfig_data + .get("version_major") + .context("version_major is not defined")? + .parse::() + .context("Could not parse value of version_major")?; + let minor = sysconfig_data + .get("version_minor") + .context("version_minor is not defined")? + .parse::() + .context("Could not parse value of version_minor")?; + let abiflags = sysconfig_data + .get("ABIFLAGS") + .map(ToString::to_string) + .unwrap_or_default(); + let ext_suffix = sysconfig_data + .get("EXT_SUFFIX") + .context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?; + let abi_tag = sysconfig_data + .get("SOABI") + .and_then(|abi| abi.split('-').nth(1).map(ToString::to_string)); + let interpreter_kind = sysconfig_data + .get("SOABI") + .and_then(|tag| { + if tag.starts_with("pypy") { + Some(InterpreterKind::PyPy) + } else if tag.starts_with("cpython") { + Some(InterpreterKind::CPython) + } else { + None + } + }) + .context("unsupported Python interpreter")?; + interpreters.push(PythonInterpreter { + config: InterpreterConfig { + major, + minor, + interpreter_kind, + abiflags, + ext_suffix: ext_suffix.to_string(), + abi_tag, + pointer_width: None, + }, + executable: PathBuf::new(), + platform: None, + runnable: false, + }); + } else { + if interpreter.is_empty() && !self.find_interpreter { + bail!("Couldn't find any python interpreters. Please specify at least one with -i"); + } + interpreters = + find_interpreter_in_sysconfig(interpreter, target, min_python_minor)?; + } + } else { + match find_interpreter_in_host(bridge, interpreter, target, min_python_minor) { + Ok(host_interps) => { + interpreters = host_interps; + native_interpreters = true; + } + Err(err) => { + if binding_name.starts_with("pyo3") && target.is_unix() { + interpreters = find_interpreter_in_sysconfig( + interpreter, + target, + min_python_minor, + ) + .map_err(|_| err)?; + } else { + return Err(err); + } + } + } + } + + let interpreters_str = interpreters + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "); + if native_interpreters { + println!("🐍 Found {}", interpreters_str); + } else { + println!("🐍 Found cross compiling target {}", interpreters_str); + } + + Ok(interpreters) + } + BridgeModel::Cffi => { + let interpreter = + find_single_python_interpreter(bridge, interpreter, target, "cffi")?; + println!("🐍 Using {} to generate the cffi bindings", interpreter); + Ok(vec![interpreter]) + } + BridgeModel::Bin(None) => Ok(vec![]), + BridgeModel::BindingsAbi3(major, minor) => { + let interpreters = + find_interpreter_in_host(bridge, interpreter, target, Some(*minor as usize)) + .unwrap_or_else(|_| { + find_interpreter_in_sysconfig( + interpreter, + target, + Some(*minor as usize), + ) + .unwrap_or_default() + }); + // Ideally, we wouldn't want to use any python interpreter without abi3 at all. + // Unfortunately, on windows we need one to figure out base_prefix for a linker + // argument. + if target.is_windows() { + if env::var_os("PYO3_CROSS_LIB_DIR").is_some() { + // PYO3_CROSS_LIB_DIR should point to the `libs` directory inside base_prefix + // when cross compiling, so we fake a python interpreter matching it + println!("⚠️ Cross-compiling is poorly supported"); + Ok(vec![PythonInterpreter { + config: InterpreterConfig { + major: *major as usize, + minor: *minor as usize, + interpreter_kind: InterpreterKind::CPython, + abiflags: "".to_string(), + ext_suffix: ".pyd".to_string(), + abi_tag: None, + pointer_width: None, + }, + executable: PathBuf::new(), + platform: None, + runnable: false, + }]) + } else if let Some(interp) = interpreters.get(0) { + println!("🐍 Using {} to generate to link bindings (With abi3, an interpreter is only required on windows)", interp); + Ok(interpreters) + } else if generate_import_lib { + println!("🐍 Not using a specific python interpreter (Automatically generating windows import library)"); + // fake a python interpreter + Ok(vec![PythonInterpreter { + config: InterpreterConfig { + major: *major as usize, + minor: *minor as usize, + interpreter_kind: InterpreterKind::CPython, + abiflags: "".to_string(), + ext_suffix: ".pyd".to_string(), + abi_tag: None, + pointer_width: None, + }, + executable: PathBuf::new(), + platform: None, + runnable: false, + }]) + } else { + bail!("Failed to find a python interpreter"); + } + } else if interpreter.is_empty() { + println!("🐍 Not using a specific python interpreter"); + // Fake one to make `BuildContext::build_wheels` happy for abi3 when no cpython/pypy found on host + // The python interpreter config doesn't matter, as it's not used for anything + Ok(vec![PythonInterpreter { + config: InterpreterConfig { + major: *major as usize, + minor: *minor as usize, + interpreter_kind: InterpreterKind::CPython, + abiflags: "".to_string(), + ext_suffix: "".to_string(), + abi_tag: None, + pointer_width: None, + }, + executable: PathBuf::new(), + platform: None, + runnable: false, + }]) + } else if target.cross_compiling() { + let mut interps = Vec::with_capacity(interpreters.len()); + let mut pypys = Vec::new(); + for interp in interpreters { + if matches!(interp.interpreter_kind, InterpreterKind::PyPy) { + pypys.push(PathBuf::from(format!( + "pypy{}.{}", + interp.major, interp.minor + ))); + } else { + interps.push(interp); + } + } + // cross compiling to PyPy with abi3 feature enabled, + // we cannot use host pypy so switch to bundled sysconfig instead + if !pypys.is_empty() { + interps.extend(find_interpreter_in_sysconfig( + &pypys, + target, + min_python_minor, + )?) + } + if interps.is_empty() { + bail!("Failed to find any python interpreter"); + } + Ok(interps) + } else { + if interpreters.is_empty() { + bail!("Failed to find any python interpreter"); + } + Ok(interpreters) + } + } + } + } + /// Tries to fill the missing metadata for a BuildContext by querying cargo and python pub fn into_build_context( self, @@ -277,24 +532,22 @@ impl BuildOptions { }; let generate_import_lib = is_generating_import_lib(&cargo_metadata)?; - let interpreter = if self.interpreter.is_empty() { + let interpreter = if self.find_interpreter { // Auto-detect interpreters - find_interpreter( + self.find_interpreters( &bridge, &[], - &target, get_min_python_minor(&metadata21), generate_import_lib, )? } else { // User given list of interpreters - find_interpreter( - &bridge, - &self.interpreter, - &target, - None, - generate_import_lib, - )? + let interpreter = if self.interpreter.is_empty() && !target.cross_compiling() { + vec![PathBuf::from("python3")] + } else { + self.interpreter.clone() + }; + self.find_interpreters(&bridge, &interpreter, None, generate_import_lib)? }; let mut rustc_extra_args = self.rustc_extra_args.clone(); @@ -721,7 +974,11 @@ fn find_interpreter_in_host( fn find_interpreter_in_sysconfig( interpreter: &[PathBuf], target: &Target, + min_python_minor: Option, ) -> Result> { + if interpreter.is_empty() { + return Ok(PythonInterpreter::find_by_target(target, min_python_minor)); + } let mut interpreters = Vec::new(); for interp in interpreter { let python = interp @@ -773,247 +1030,6 @@ fn find_interpreter_in_sysconfig( Ok(interpreters) } -/// Finds the appropriate amount for python versions for each [BridgeModel]. -/// -/// This means all for bindings, one for cffi and zero for bin. -pub fn find_interpreter( - bridge: &BridgeModel, - interpreter: &[PathBuf], - target: &Target, - min_python_minor: Option, - generate_import_lib: bool, -) -> Result> { - match bridge { - BridgeModel::Bindings(binding_name, _) | BridgeModel::Bin(Some((binding_name, _))) => { - let mut native_interpreters = false; - let mut interpreters = Vec::new(); - if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") { - if !binding_name.starts_with("pyo3") { - bail!("Only pyo3 bindings can be configured with PYO3_CONFIG_FILE"); - } - let interpreter_config = - InterpreterConfig::from_pyo3_config(config_file.as_ref(), target) - .context("Invalid PYO3_CONFIG_FILE")?; - interpreters.push(PythonInterpreter::from_config(interpreter_config)); - } else if binding_name.starts_with("pyo3") && target.cross_compiling() { - if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") { - let host_interpreters = - find_interpreter_in_host(bridge, interpreter, target, min_python_minor)?; - let host_python = &host_interpreters[0]; - println!( - "🐍 Using host {} for cross-compiling preparation", - host_python - ); - // pyo3 - env::set_var("PYO3_PYTHON", &host_python.executable); - // rust-cpython, and legacy pyo3 versions - env::set_var("PYTHON_SYS_EXECUTABLE", &host_python.executable); - - let sysconfig_path = find_sysconfigdata(cross_lib_dir.as_ref(), target)?; - env::set_var( - "MATURIN_PYTHON_SYSCONFIGDATA_DIR", - sysconfig_path.parent().unwrap(), - ); - - let sysconfig_data = parse_sysconfigdata(host_python, sysconfig_path)?; - let major = sysconfig_data - .get("version_major") - .context("version_major is not defined")? - .parse::() - .context("Could not parse value of version_major")?; - let minor = sysconfig_data - .get("version_minor") - .context("version_minor is not defined")? - .parse::() - .context("Could not parse value of version_minor")?; - let abiflags = sysconfig_data - .get("ABIFLAGS") - .map(ToString::to_string) - .unwrap_or_default(); - let ext_suffix = sysconfig_data - .get("EXT_SUFFIX") - .context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?; - let abi_tag = sysconfig_data - .get("SOABI") - .and_then(|abi| abi.split('-').nth(1).map(ToString::to_string)); - let interpreter_kind = sysconfig_data - .get("SOABI") - .and_then(|tag| { - if tag.starts_with("pypy") { - Some(InterpreterKind::PyPy) - } else if tag.starts_with("cpython") { - Some(InterpreterKind::CPython) - } else { - None - } - }) - .context("unsupported Python interpreter")?; - interpreters.push(PythonInterpreter { - config: InterpreterConfig { - major, - minor, - interpreter_kind, - abiflags, - ext_suffix: ext_suffix.to_string(), - abi_tag, - pointer_width: None, - }, - executable: PathBuf::new(), - platform: None, - runnable: false, - }); - } else { - if interpreter.is_empty() { - bail!("Couldn't find any python interpreters. Please specify at least one with -i"); - } - interpreters = find_interpreter_in_sysconfig(interpreter, target)?; - } - } else { - match find_interpreter_in_host(bridge, interpreter, target, min_python_minor) { - Ok(host_interps) => { - interpreters = host_interps; - native_interpreters = true; - } - Err(err) => { - if !interpreter.is_empty() - && binding_name.starts_with("pyo3") - && target.is_unix() - { - interpreters = find_interpreter_in_sysconfig(interpreter, target) - .map_err(|_| err)?; - } else { - return Err(err); - } - } - } - } - - let interpreters_str = interpreters - .iter() - .map(ToString::to_string) - .collect::>() - .join(", "); - if native_interpreters { - println!("🐍 Found {}", interpreters_str); - } else { - println!("🐍 Found cross compiling target {}", interpreters_str); - } - - Ok(interpreters) - } - BridgeModel::Cffi => { - let interpreter = find_single_python_interpreter(bridge, interpreter, target, "cffi")?; - println!("🐍 Using {} to generate the cffi bindings", interpreter); - Ok(vec![interpreter]) - } - BridgeModel::Bin(None) => Ok(vec![]), - BridgeModel::BindingsAbi3(major, minor) => { - let interpreters = if !interpreter.is_empty() { - match find_interpreter_in_host(bridge, interpreter, target, Some(*minor as usize)) { - Ok(host_interps) => host_interps, - Err(err) => { - if !interpreter.is_empty() { - find_interpreter_in_sysconfig(interpreter, target).map_err(|_| err)? - } else { - Vec::new() - } - } - } - } else { - find_interpreter_in_host(bridge, interpreter, target, Some(*minor as usize)) - .unwrap_or_default() - }; - // Ideally, we wouldn't want to use any python interpreter without abi3 at all. - // Unfortunately, on windows we need one to figure out base_prefix for a linker - // argument. - if target.is_windows() { - if env::var_os("PYO3_CROSS_LIB_DIR").is_some() { - // PYO3_CROSS_LIB_DIR should point to the `libs` directory inside base_prefix - // when cross compiling, so we fake a python interpreter matching it - println!("⚠️ Cross-compiling is poorly supported"); - Ok(vec![PythonInterpreter { - config: InterpreterConfig { - major: *major as usize, - minor: *minor as usize, - interpreter_kind: InterpreterKind::CPython, - abiflags: "".to_string(), - ext_suffix: ".pyd".to_string(), - abi_tag: None, - pointer_width: None, - }, - executable: PathBuf::new(), - platform: None, - runnable: false, - }]) - } else if let Some(interp) = interpreters.get(0) { - println!("🐍 Using {} to generate to link bindings (With abi3, an interpreter is only required on windows)", interp); - Ok(interpreters) - } else if generate_import_lib { - println!("🐍 Not using a specific python interpreter (Automatically generating windows import library)"); - // fake a python interpreter - Ok(vec![PythonInterpreter { - config: InterpreterConfig { - major: *major as usize, - minor: *minor as usize, - interpreter_kind: InterpreterKind::CPython, - abiflags: "".to_string(), - ext_suffix: ".pyd".to_string(), - abi_tag: None, - pointer_width: None, - }, - executable: PathBuf::new(), - platform: None, - runnable: false, - }]) - } else { - bail!("Failed to find a python interpreter"); - } - } else { - println!("🐍 Not using a specific python interpreter"); - if interpreter.is_empty() { - // Fake one to make `BuildContext::build_wheels` happy for abi3 when no cpython/pypy found on host - // The python interpreter config doesn't matter, as it's not used for anything - Ok(vec![PythonInterpreter { - config: InterpreterConfig { - major: *major as usize, - minor: *minor as usize, - interpreter_kind: InterpreterKind::CPython, - abiflags: "".to_string(), - ext_suffix: "".to_string(), - abi_tag: None, - pointer_width: None, - }, - executable: PathBuf::new(), - platform: None, - runnable: false, - }]) - } else if target.cross_compiling() { - let mut interps = Vec::with_capacity(interpreters.len()); - let mut pypys = Vec::new(); - for interp in interpreters { - if matches!(interp.interpreter_kind, InterpreterKind::PyPy) { - pypys.push(PathBuf::from(format!( - "pypy{}.{}", - interp.major, interp.minor - ))); - } else { - interps.push(interp); - } - } - // cross compiling to PyPy with abi3 feature enabled, - // we cannot use host pypy so switch to bundled sysconfig instead - if !pypys.is_empty() { - interps.extend(find_interpreter_in_sysconfig(&pypys, target)?) - } - Ok(interps) - } else { - Ok(interpreters) - } - } - } - } -} - /// Helper function that calls shlex on all extra args given fn split_extra_args(given_args: &[String]) -> Result> { let mut splitted_args = vec![]; diff --git a/src/develop.rs b/src/develop.rs index d3549c952..9f00154af 100644 --- a/src/develop.rs +++ b/src/develop.rs @@ -30,6 +30,7 @@ pub fn develop( let build_options = BuildOptions { platform_tag: vec![PlatformTag::Linux], interpreter: vec![python.clone()], + find_interpreter: false, bindings, manifest_path: Some(manifest_file.to_path_buf()), out: Some(wheel_dir.path().to_path_buf()), diff --git a/src/main.rs b/src/main.rs index 7097ff0e9..8a0a2e984 100644 --- a/src/main.rs +++ b/src/main.rs @@ -367,7 +367,7 @@ fn run() -> Result<()> { Opt::ListPython { target } => { let found = if target.is_some() { let target = Target::from_target_triple(target)?; - PythonInterpreter::find_by_target(&target) + PythonInterpreter::find_by_target(&target, None) } else { let target = Target::from_target_triple(None)?; // We don't know the targeted bindings yet, so we use the most lenient diff --git a/src/python_interpreter/mod.rs b/src/python_interpreter/mod.rs index 91f3de7a8..62bdf5794 100644 --- a/src/python_interpreter/mod.rs +++ b/src/python_interpreter/mod.rs @@ -620,10 +620,22 @@ impl PythonInterpreter { } /// Find all available python interpreters for a given target - pub fn find_by_target(target: &Target) -> Vec { + pub fn find_by_target( + target: &Target, + min_python_minor: Option, + ) -> Vec { InterpreterConfig::lookup_target(target) .into_iter() - .map(Self::from_config) + .filter_map(|config| match min_python_minor { + Some(min_python_minor) => { + if config.minor < min_python_minor { + None + } else { + Some(Self::from_config(config)) + } + } + None => Some(Self::from_config(config)), + }) .collect() }