Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for building with multiple binary targets #948

Merged
merged 5 commits into from
Jun 7, 2022
Merged

Add support for building with multiple binary targets #948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
32ee134
Add support for building with multiple binary targets
messense Jun 1, 2022
69e040d
auditwheel: only graft duplicated shared library dependency once
messense Jun 1, 2022
2988963
Add another binary to `hello-world` crate
messense Jun 1, 2022
4e5ca52
auditwheel: only print grafted libraries once
messense Jun 7, 2022
c51fc62
Don't graft `/lib/ld-musl-*.so.1` on musllinux
messense Jun 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Changelog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]

* **Breaking Change**: Drop support for python 3.6, which is end of life
* Add support for building with multiple binary targets in [#948](https://github.com/PyO3/maturin/pull/948)
* Fix incompatibility with cibuildwheel for 32-bit Windows in [#951](https://github.com/PyO3/maturin/pull/951)
* Don't require `pip` error messages to be utf-8 encoding in [#953](https://github.com/PyO3/maturin/pull/953)
* Compare minimum python version requirement between `requires-python` and bindings crate in [#954](https://github.com/PyO3/maturin/pull/954)
Expand Down
6 changes: 4 additions & 2 deletions src/auditwheel/repair.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,16 @@ pub fn find_external_libs(
.analyze(artifact)
.map_err(AuditWheelError::DependencyAnalysisError)?;
let mut ext_libs = Vec::new();
for (name, lib) in deps.libraries {
for (_, lib) in deps.libraries {
let name = &lib.name;
// Skip dynamic linker/loader and white-listed libs
if name.starts_with("ld-linux")
|| name == "ld64.so.2"
|| name == "ld64.so.1"
// musl libc, eg: libc.musl-aarch64.so.1
|| name.starts_with("ld-musl")
|| name.starts_with("libc.")
|| policy.lib_whitelist.contains(&name)
|| policy.lib_whitelist.contains(name)
{
continue;
}
Expand Down
120 changes: 72 additions & 48 deletions src/build_context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use fs_err as fs;
use lddtree::Library;
use sha2::{Digest, Sha256};
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::fmt::{Display, Formatter};
use std::io;
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -330,10 +330,10 @@ impl BuildContext {
fn add_external_libs(
&self,
writer: &mut WheelWriter,
artifact: &Path,
ext_libs: &[Library],
artifacts: &[PathBuf],
ext_libs: &[Vec<Library>],
) -> Result<()> {
if ext_libs.is_empty() {
if ext_libs.iter().all(|libs| libs.is_empty()) {
return Ok(());
}
// Put external libs to ${module_name}.libs directory
Expand All @@ -343,8 +343,8 @@ impl BuildContext {

let temp_dir = tempfile::tempdir()?;
let mut soname_map = HashMap::new();
let mut libs_copied = Vec::new();
for lib in ext_libs {
let mut libs_copied = HashSet::new();
for lib in ext_libs.iter().flatten() {
let lib_path = lib.realpath.clone().with_context(|| {
format!(
"Cannot repair wheel, because required library {} could not be located.",
Expand All @@ -364,7 +364,7 @@ impl BuildContext {
// for example soname and rpath
let dest_path = temp_dir.path().join(&new_soname);
fs::copy(&lib_path, &dest_path)?;
libs_copied.push(lib_path);
libs_copied.insert(lib_path);

patchelf::set_soname(&dest_path, &new_soname)?;
if !lib.rpath.is_empty() || !lib.runpath.is_empty() {
Expand All @@ -376,12 +376,21 @@ impl BuildContext {
);
}

let replacements = soname_map
.iter()
.map(|(k, v)| (k, v.0.clone()))
.collect::<Vec<_>>();
if !replacements.is_empty() {
patchelf::replace_needed(artifact, &replacements[..])?;
for (artifact, artifact_ext_libs) in artifacts.iter().zip(ext_libs) {
let artifact_deps: HashSet<_> = artifact_ext_libs.iter().map(|lib| &lib.name).collect();
let replacements = soname_map
.iter()
.filter_map(|(k, v)| {
if artifact_deps.contains(k) {
Some((k, v.0.clone()))
} else {
None
}
})
.collect::<Vec<_>>();
if !replacements.is_empty() {
patchelf::replace_needed(artifact, &replacements[..])?;
}
}

// we grafted in a bunch of libraries and modified their sonames, but
Expand Down Expand Up @@ -411,14 +420,16 @@ impl BuildContext {

// Currently artifact .so file always resides at ${module_name}/${module_name}.so
let artifact_dir = Path::new(&self.module_name);
let old_rpaths = patchelf::get_rpath(artifact)?;
// TODO: clean existing rpath entries if it's not pointed to a location within the wheel
// See https://github.com/pypa/auditwheel/blob/353c24250d66951d5ac7e60b97471a6da76c123f/src/auditwheel/repair.py#L160
let mut new_rpaths: Vec<&str> = old_rpaths.split(':').collect();
let new_rpath = Path::new("$ORIGIN").join(relpath(&libs_dir, artifact_dir));
new_rpaths.push(new_rpath.to_str().unwrap());
let new_rpath = new_rpaths.join(":");
patchelf::set_rpath(artifact, &new_rpath)?;
for artifact in artifacts {
let old_rpaths = patchelf::get_rpath(artifact)?;
// TODO: clean existing rpath entries if it's not pointed to a location within the wheel
// See https://github.com/pypa/auditwheel/blob/353c24250d66951d5ac7e60b97471a6da76c123f/src/auditwheel/repair.py#L160
let mut new_rpaths: Vec<&str> = old_rpaths.split(':').collect();
let new_rpath = Path::new("$ORIGIN").join(relpath(&libs_dir, artifact_dir));
new_rpaths.push(new_rpath.to_str().unwrap());
let new_rpath = new_rpaths.join(":");
patchelf::set_rpath(artifact, &new_rpath)?;
}
Ok(())
}

Expand All @@ -433,7 +444,7 @@ impl BuildContext {
&self,
artifact: &Path,
platform_tags: &[PlatformTag],
ext_libs: &[Library],
ext_libs: Vec<Library>,
major: u8,
min_minor: u8,
) -> Result<BuiltWheelMetadata> {
Expand All @@ -443,7 +454,7 @@ impl BuildContext {
let tag = format!("cp{}{}-abi3-{}", major, min_minor, platform);

let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &[tag.clone()])?;
self.add_external_libs(&mut writer, artifact, ext_libs)?;
self.add_external_libs(&mut writer, &[artifact.to_path_buf()], &[ext_libs])?;

write_bindings_module(
&mut writer,
Expand Down Expand Up @@ -488,7 +499,7 @@ impl BuildContext {
let (wheel_path, tag) = self.write_binding_wheel_abi3(
&artifact,
&platform_tags,
&external_libs,
external_libs,
major,
min_minor,
)?;
Expand All @@ -509,12 +520,12 @@ impl BuildContext {
python_interpreter: &PythonInterpreter,
artifact: &Path,
platform_tags: &[PlatformTag],
ext_libs: &[Library],
ext_libs: Vec<Library>,
) -> Result<BuiltWheelMetadata> {
let tag = python_interpreter.get_tag(&self.target, platform_tags, self.universal2)?;

let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &[tag.clone()])?;
self.add_external_libs(&mut writer, artifact, ext_libs)?;
self.add_external_libs(&mut writer, &[artifact.to_path_buf()], &[ext_libs])?;

write_bindings_module(
&mut writer,
Expand Down Expand Up @@ -564,7 +575,7 @@ impl BuildContext {
python_interpreter,
&artifact,
&platform_tags,
&external_libs,
external_libs,
)?;
println!(
"📦 Built wheel for {} {}.{}{} to {}",
Expand Down Expand Up @@ -592,6 +603,9 @@ impl BuildContext {
) -> Result<PathBuf> {
let artifacts = compile(self, python_interpreter, &self.bridge)
.context("Failed to build a native library through cargo")?;
let artifacts = artifacts
.get(0)
.context("Failed to build a native library through cargo")?;

let artifact = artifacts.get("cdylib").cloned().ok_or_else(|| {
anyhow!(
Expand Down Expand Up @@ -620,14 +634,14 @@ impl BuildContext {
&self,
artifact: &Path,
platform_tags: &[PlatformTag],
ext_libs: &[Library],
ext_libs: Vec<Library>,
) -> Result<BuiltWheelMetadata> {
let (tag, tags) = self
.target
.get_universal_tags(platform_tags, self.universal2)?;

let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &tags)?;
self.add_external_libs(&mut writer, artifact, ext_libs)?;
self.add_external_libs(&mut writer, &[artifact.to_path_buf()], &[ext_libs])?;

write_cffi_module(
&mut writer,
Expand Down Expand Up @@ -656,7 +670,7 @@ impl BuildContext {
} else {
self.platform_tag.clone()
};
let (wheel_path, tag) = self.write_cffi_wheel(&artifact, &platform_tags, &external_libs)?;
let (wheel_path, tag) = self.write_cffi_wheel(&artifact, &platform_tags, external_libs)?;

// Warn if cffi isn't specified in the requirements
if !self
Expand All @@ -680,9 +694,9 @@ impl BuildContext {
fn write_bin_wheel(
&self,
python_interpreter: Option<&PythonInterpreter>,
artifact: &Path,
artifacts: &[PathBuf],
platform_tags: &[PlatformTag],
ext_libs: &[Library],
ext_libs: &[Vec<Library>],
) -> Result<BuiltWheelMetadata> {
let (tag, tags) = match (&self.bridge, python_interpreter) {
(BridgeModel::Bin(None), _) => self
Expand All @@ -709,14 +723,15 @@ impl BuildContext {
}
}

// I wouldn't know of any case where this would be the wrong (and neither do
// I know a better alternative)
let bin_name = artifact
.file_name()
.expect("Couldn't get the filename from the binary produced by cargo");
self.add_external_libs(&mut writer, artifact, ext_libs)?;

write_bin(&mut writer, artifact, &self.metadata21, bin_name)?;
for artifact in artifacts {
// I wouldn't know of any case where this would be the wrong (and neither do
// I know a better alternative)
let bin_name = artifact
.file_name()
.expect("Couldn't get the filename from the binary produced by cargo");
write_bin(&mut writer, artifact, &self.metadata21, bin_name)?;
}
self.add_external_libs(&mut writer, artifacts, ext_libs)?;

self.add_pth(&mut writer)?;
add_data(&mut writer, self.project_layout.data.as_deref())?;
Expand All @@ -735,12 +750,21 @@ impl BuildContext {
let artifacts = compile(self, python_interpreter, &self.bridge)
.context("Failed to build a native library through cargo")?;

let artifact = artifacts
.get("bin")
.cloned()
.ok_or_else(|| anyhow!("Cargo didn't build a binary"))?;

let (policy, external_libs) = self.auditwheel(&artifact, &self.platform_tag, None)?;
let mut policies = Vec::with_capacity(artifacts.len());
let mut ext_libs = Vec::new();
let mut artifact_paths = Vec::with_capacity(artifacts.len());
for artifact in artifacts {
let artifact = artifact
.get("bin")
.cloned()
.ok_or_else(|| anyhow!("Cargo didn't build a binary"))?;

let (policy, external_libs) = self.auditwheel(&artifact, &self.platform_tag, None)?;
policies.push(policy);
ext_libs.push(external_libs);
artifact_paths.push(artifact);
}
let policy = policies.iter().min_by_key(|p| p.priority).unwrap();
let platform_tags = if self.platform_tag.is_empty() {
vec![policy.platform_tag()]
} else {
Expand All @@ -749,9 +773,9 @@ impl BuildContext {

let (wheel_path, tag) = self.write_bin_wheel(
python_interpreter,
&artifact,
&artifact_paths,
&platform_tags,
&external_libs,
&ext_libs,
)?;
println!("📦 Built wheel to {}", wheel_path.display());
wheels.push((wheel_path, tag));
Expand Down
Loading