Refactor mostly auditwheel

.github/workflows/lint.yml

@@ -37,7 +37,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --features password-storage -- -D warnings
+          args: --tests --all-features -- -D warnings
 
   black:
     runs-on: ubuntu-latest

Cargo.toml

@@ -32,7 +32,6 @@ keyring = { version = "1.0.0", optional = true }
 platform-info = "0.2.0"
 pretty_env_logger = { version = "0.4.0", optional = true }
 regex = "1.4.5"
-rpassword = { version = "5.0.1", optional = true }
 serde = { version = "1.0.131", features = ["derive"] }
 serde_json = "1.0.70"
 sha2 = "0.10.0"
@@ -42,8 +41,7 @@ tempfile = "3.2.0"
 toml = "0.5.8"
 zip = "0.5.5"
 thiserror = "1.0.24"
-dirs = { version = "4.0.0", optional = true }
-configparser = { version = "3.0.0", optional = true }
+dirs = "4.0.0"
 fs-err = "2.5.0"
 fat-macho = { version = "0.4.4", default-features = false }
 once_cell = "1.7.2"
@@ -61,15 +59,18 @@ clap = { version = "3.0.0", features = ["derive", "env", "wrap_help"] }
 clap_complete = "3.0.0"
 clap_complete_fig = "3.0.0"
 semver = "1.0.4"
+# upload
+configparser = { version = "3.0.0", optional = true }
+multipart = { version = "0.18.0", features = ["client"], default-features = false, optional = true }
+rpassword = { version = "5.0.1", optional = true }
 ureq = { version = "2.3.1", optional = true }
-multipart = { version = "0.18.0", features = ["client"],default-features = false, optional = true }
 
 [dev-dependencies]
 indoc = "1.0.3"
 
 [features]
 default = ["log", "upload", "human-panic"]
-upload = ["ureq", "multipart", "rpassword", "configparser", "dirs"]
+upload = ["ureq", "multipart", "rpassword", "configparser"]
 password-storage = ["upload", "keyring"]
 log = ["pretty_env_logger"]
 # Internal feature to speed up the tests significantly

src/auditwheel/audit.rs

@@ -1,15 +1,16 @@
 use super::musllinux::{find_musl_libc, get_musl_version};
 use super::policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES};
-use crate::auditwheel::PlatformTag;
+use crate::auditwheel::{find_external_libs, PlatformTag};
 use crate::target::Target;
-use anyhow::Result;
+use anyhow::{bail, Context, Result};
 use fs_err::File;
 use goblin::elf::{sym::STT_FUNC, Elf};
+use lddtree::Library;
 use regex::Regex;
 use std::collections::{HashMap, HashSet};
 use std::io;
 use std::io::Read;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use thiserror::Error;
 
 /// Error raised during auditing an elf file for manylinux/musllinux compatibility
@@ -354,3 +355,127 @@ pub fn auditwheel_rs(
     }?;
     Ok((policy, should_repair))
 }
+
+/// Get sysroot path from target C compiler
+///
+/// Currently only gcc is supported, clang doesn't have a `--print-sysroot` option
+pub fn get_sysroot_path(target: &Target) -> Result<PathBuf> {
+    use crate::target::get_host_target;
+    use std::process::{Command, Stdio};
+
+    if let Some(sysroot) = std::env::var_os("TARGET_SYSROOT") {
+        return Ok(PathBuf::from(sysroot));
+    }
+
+    let host_triple = get_host_target()?;
+    let target_triple = target.target_triple();
+    if host_triple != target_triple {
+        let mut build = cc::Build::new();
+        build
+            // Suppress cargo metadata for example env vars printing
+            .cargo_metadata(false)
+            // opt_level, host and target are required
+            .opt_level(0)
+            .host(&host_triple)
+            .target(target_triple);
+        let compiler = build
+            .try_get_compiler()
+            .with_context(|| format!("Failed to get compiler for {}", target_triple))?;
+        // Only GNU like compilers support `--print-sysroot`
+        if !compiler.is_like_gnu() {
+            return Ok(PathBuf::from("/"));
+        }
+        let path = compiler.path();
+        let out = Command::new(path)
+            .arg("--print-sysroot")
+            .stdout(Stdio::piped())
+            .stderr(Stdio::null())
+            .output()
+            .with_context(|| format!("Failed to run `{} --print-sysroot`", path.display()))?;
+        if out.status.success() {
+            let sysroot = String::from_utf8(out.stdout)
+                .context("Failed to read the sysroot path")?
+                .trim()
+                .to_owned();
+            return Ok(PathBuf::from(sysroot));
+        } else {
+            bail!(
+                "Failed to get the sysroot path: {}",
+                String::from_utf8(out.stderr)?
+            );
+        }
+    }
+    Ok(PathBuf::from("/"))
+}
+
+/// For the given compilation result, return the manylinux platform and the external libs
+/// we need to add to repair it
+pub fn get_policy_and_libs(
+    artifact: &Path,
+    platform_tag: Option<PlatformTag>,
+    target: &Target,
+) -> Result<(Policy, Vec<Library>)> {
+    let (policy, should_repair) =
+        auditwheel_rs(artifact, target, platform_tag).with_context(|| {
+            if let Some(platform_tag) = platform_tag {
+                format!("Error ensuring {} compliance", platform_tag)
+            } else {
+                "Error checking for manylinux/musllinux compliance".to_string()
+            }
+        })?;
+    let external_libs = if should_repair {
+        let sysroot = get_sysroot_path(target).unwrap_or_else(|_| PathBuf::from("/"));
+        find_external_libs(&artifact, &policy, sysroot).with_context(|| {
+            if let Some(platform_tag) = platform_tag {
+                format!("Error repairing wheel for {} compliance", platform_tag)
+            } else {
+                "Error repairing wheel for manylinux/musllinux compliance".to_string()
+            }
+        })?
+    } else {
+        Vec::new()
+    };
+    Ok((policy, external_libs))
+}
+
+pub fn relpath(to: &Path, from: &Path) -> PathBuf {
+    let mut suffix_pos = 0;
+    for (f, t) in from.components().zip(to.components()) {
+        if f == t {
+            suffix_pos += 1;
+        } else {
+            break;
+        }
+    }
+    let mut result = PathBuf::new();
+    from.components()
+        .skip(suffix_pos)
+        .map(|_| result.push(".."))
+        .last();
+    to.components()
+        .skip(suffix_pos)
+        .map(|x| result.push(x.as_os_str()))
+        .last();
+    result
+}
+
+#[cfg(test)]
+mod test {
+    use crate::auditwheel::audit::relpath;
+    use std::path::Path;
+
+    #[test]
+    fn test_relpath() {
+        let cases = [
+            ("", "", ""),
+            ("/", "/usr", ".."),
+            ("/", "/usr/lib", "../.."),
+        ];
+        for (from, to, expected) in cases {
+            let from = Path::new(from);
+            let to = Path::new(to);
+            let result = relpath(from, to);
+            assert_eq!(result, Path::new(expected));
+        }
+    }
+}

src/auditwheel/mod.rs

@@ -8,4 +8,4 @@ mod repair;
 pub use audit::*;
 pub use platform_tag::PlatformTag;
 pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES};
-pub use repair::{find_external_libs, hash_file};
+pub use repair::find_external_libs;

src/auditwheel/policy.rs

@@ -74,16 +74,6 @@ impl PartialOrd for Policy {
 }
 
 impl Policy {
-    /// Get highest priority policy than self
-    pub fn higher_priority_policies(&self) -> impl Iterator<Item = &Policy> {
-        let policies = if self.name.starts_with("musllinux") {
-            &MUSLLINUX_POLICIES
-        } else {
-            &MANYLINUX_POLICIES
-        };
-        policies.iter().filter(move |p| p.priority > self.priority)
-    }
-
     /// Get platform tag from this policy
     pub fn platform_tag(&self) -> PlatformTag {
         self.name.parse().expect("unknown platform tag")

src/auditwheel/repair.rs

@@ -1,10 +1,7 @@
 use super::audit::AuditWheelError;
 use crate::auditwheel::Policy;
 use anyhow::Result;
-use fs_err as fs;
 use lddtree::DependencyAnalyzer;
-use sha2::{Digest, Sha256};
-use std::io;
 use std::path::{Path, PathBuf};
 
 pub fn find_external_libs(
@@ -30,12 +27,3 @@ pub fn find_external_libs(
     }
     Ok(ext_libs)
 }
-
-/// Calculate the sha256 of a file
-pub fn hash_file(path: impl AsRef<Path>) -> Result<String, AuditWheelError> {
-    let mut file = fs::File::open(path.as_ref()).map_err(AuditWheelError::IoError)?;
-    let mut hasher = Sha256::new();
-    io::copy(&mut file, &mut hasher).map_err(AuditWheelError::IoError)?;
-    let hex = format!("{:x}", hasher.finalize());
-    Ok(hex)
-}