From 8a8ba2672bd5c506c4249864e4a9638b6db2f973 Mon Sep 17 00:00:00 2001 From: messense Date: Fri, 4 Jun 2021 00:09:55 +0800 Subject: [PATCH 1/2] Replace `read_distribution` mod with `python-pkginfo` crate --- Cargo.lock | 134 ++++++++------------- Cargo.toml | 2 +- src/lib.rs | 4 - src/main.rs | 72 ++--------- src/read_distribution.rs | 254 --------------------------------------- src/upload.rs | 97 ++++++++++----- 6 files changed, 123 insertions(+), 440 deletions(-) delete mode 100644 src/read_distribution.rs diff --git a/Cargo.lock b/Cargo.lock index e99781972..923e33ed8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03345e98af8f3d786b6d9f656ccfa6ac316d954e92bc4841f0bba20789d5fb5a" +checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a" dependencies = [ "gimli", ] @@ -70,12 +70,6 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" -[[package]] -name = "ascii" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" - [[package]] name = "atty" version = "0.2.14" @@ -95,9 +89,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4717cfcbfaa661a0fd48f8453951837ae7e8f81e481fbb136e3202d72805a744" +checksum = "b7815ea54e4d821e791162e078acbebfd6d8c8939cd559c9335dceb1c8ca7282" dependencies = [ "addr2line", "cc", @@ -180,9 +174,9 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da" [[package]] name = "bzip2" -version = "0.3.3" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" dependencies = [ "bzip2-sys", "libc", @@ -190,9 +184,9 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.10+1.0.8" +version = "0.1.11+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17fa3d1ac1ca21c5c4e36a97f3c3eb25084576f6fc47bf0139c1123434216c6c" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" dependencies = [ "cc", "libc", @@ -310,22 +304,19 @@ dependencies = [ [[package]] name = "combine" -version = "3.8.1" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e" dependencies = [ - "ascii", - "byteorder", - "either", + "bytes", "memchr", - "unreachable", ] [[package]] name = "configparser" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aad39d76dbe45b809ef6d783b8c597732225b8f3d6c4d8ceb4a4f834a844ffe" +checksum = "f7201ee416d124d589a820111ba755930df8b75855321a9a1b87312a0597ec8f" [[package]] name = "core-foundation" @@ -420,12 +411,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - [[package]] name = "encoding_rs" version = "0.8.28" @@ -651,9 +636,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" [[package]] name = "heck" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] @@ -747,9 +732,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.8" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3f71a7eea53a3f8257a7b4795373ff886397178cd634430ea94e12d7fe4fe34" +checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83" dependencies = [ "bytes", "futures-channel", @@ -761,7 +746,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project", + "pin-project-lite", "socket2", "tokio", "tower-service", @@ -855,9 +840,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.95" +version = "0.2.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +checksum = "5600b4e6efc5421841a2138a6b082e07fe12f9aaa12783d50e5d13325b26b4fc" [[package]] name = "linked-hash-map" @@ -919,11 +904,11 @@ dependencies = [ "human-panic", "indoc", "keyring", - "mailparse", "once_cell", "platform-info", "pretty_env_logger", "pyproject-toml", + "python-pkginfo", "regex", "reqwest", "rpassword", @@ -1116,9 +1101,12 @@ dependencies = [ [[package]] name = "object" -version = "0.24.0" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" +checksum = "f8bc1d42047cf336f0f939c99e97183cf31551bf0f2865a2ec9c8d91fd4ffb5e" +dependencies = [ + "memchr", +] [[package]] name = "once_cell" @@ -1156,26 +1144,6 @@ dependencies = [ "ucd-trie", ] -[[package]] -name = "pin-project" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.6" @@ -1290,6 +1258,19 @@ dependencies = [ "toml", ] +[[package]] +name = "python-pkginfo" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae411f3584c4447905a0d7ee7af5c3bb5f8dff76df588a4bbff9a230545a1e3" +dependencies = [ + "flate2", + "fs-err", + "mailparse", + "tar", + "zip", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1331,7 +1312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" dependencies = [ "libc", - "rand_chacha 0.3.0", + "rand_chacha 0.3.1", "rand_core 0.6.2", "rand_hc 0.3.0", ] @@ -1348,9 +1329,9 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core 0.6.2", @@ -1742,9 +1723,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.72" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" dependencies = [ "proc-macro2", "quote", @@ -1896,9 +1877,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09391a441b373597cf0888d2b052dcf82c5be4fee05da3636ae30fb57aad8484" +checksum = "dbbdcf4f749dd33b1f1ea19b547bf789d87442ec40767d6015e5e2d39158d69a" dependencies = [ "chrono", "combine", @@ -1969,9 +1950,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" dependencies = [ "tinyvec", ] @@ -2000,15 +1981,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -2048,12 +2020,6 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "walkdir" version = "2.3.2" @@ -2235,9 +2201,9 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c83dc9b784d252127720168abd71ea82bf8c3d96b17dc565b5e2a02854f2b27" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" dependencies = [ "byteorder", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 133568e35..92b2e8f7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,6 @@ zip = "0.5.5" thiserror = "1.0.24" dirs = { version = "3.0.1", optional = true } configparser = { version = "2.0.0", optional = true } -mailparse = "0.13.2" fs-err = "2.5.0" fat-macho = "0.4.3" toml_edit = "0.2.0" @@ -58,6 +57,7 @@ once_cell = "1.7.2" scroll = "0.10.2" target-lexicon = "0.12.0" pyproject-toml = "0.1.0" +python-pkginfo = "0.4.0" [dev-dependencies] indoc = "1.0.3" diff --git a/src/lib.rs b/src/lib.rs index 5ac67ade4..aa5e7bc75 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,9 +38,6 @@ pub use crate::module_writer::{ }; pub use crate::pyproject_toml::PyProjectToml; pub use crate::python_interpreter::PythonInterpreter; -pub use crate::read_distribution::{ - get_metadata_for_distribution, get_supported_version_for_distribution, -}; pub use crate::target::Target; pub use auditwheel::PlatformTag; pub use source_distribution::source_distribution; @@ -61,7 +58,6 @@ mod metadata; mod module_writer; mod pyproject_toml; mod python_interpreter; -mod read_distribution; #[cfg(feature = "upload")] mod registry; mod source_distribution; diff --git a/src/main.rs b/src/main.rs index dffbc8d91..d1dc146fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,54 +23,12 @@ use std::path::PathBuf; use structopt::StructOpt; #[cfg(feature = "upload")] use { - maturin::{ - get_metadata_for_distribution, get_supported_version_for_distribution, upload, - BuiltWheelMetadata, Registry, UploadError, - }, + maturin::{upload, Registry, UploadError}, reqwest::Url, rpassword, std::io, }; -#[cfg(feature = "upload")] -/// Upload item descriptor used by `upload_ui()` -struct UploadItem { - /// Built wheel file path - wheel_path: PathBuf, - /// Supported Python versions tag (e.g. "cp39") - supported_versions: String, - /// Wheel metadata in the (key, value) format - metadata: Vec<(String, String)>, -} - -#[cfg(feature = "upload")] -impl UploadItem { - /// Creates a new upload item descriptor from the built wheel and its metadata. - fn from_built_wheel(wheel: BuiltWheelMetadata, metadata: Vec<(String, String)>) -> Self { - let (wheel_path, supported_versions) = wheel; - - UploadItem { - wheel_path, - supported_versions, - metadata, - } - } - - /// Attempts to create a new upload item descriptor from the third-party wheel file path. - /// - /// Fails with the wheel metadata extraction errors. - fn try_from_wheel_path(wheel_path: PathBuf) -> Result { - let supported_versions = get_supported_version_for_distribution(&wheel_path)?; - let metadata = get_metadata_for_distribution(&wheel_path)?; - - Ok(UploadItem { - wheel_path, - supported_versions, - metadata, - }) - } -} - /// Returns the password and a bool that states whether to ask for re-entering the password /// after a failed authentication /// @@ -452,13 +410,13 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { /// Handles authentication/keyring integration and retrying of the publish subcommand #[cfg(feature = "upload")] -fn upload_ui(items: &[UploadItem], publish: &PublishOpt) -> Result<()> { +fn upload_ui(items: &[PathBuf], publish: &PublishOpt) -> Result<()> { let registry = complete_registry(&publish)?; println!("🚀 Uploading {} packages", items.len()); for i in items { - let upload_result = upload(®istry, &i.wheel_path, &i.metadata, &i.supported_versions); + let upload_result = upload(®istry, &i); match upload_result { Ok(()) => (), @@ -482,10 +440,7 @@ fn upload_ui(items: &[UploadItem], publish: &PublishOpt) -> Result<()> { bail!("Username and/or password are wrong"); } Err(err) => { - let filename = i - .wheel_path - .file_name() - .unwrap_or_else(|| i.wheel_path.as_os_str()); + let filename = i.file_name().unwrap_or_else(|| i.as_os_str()); if let UploadError::FileExistsError(_) = err { if publish.skip_existing { eprintln!( @@ -495,11 +450,9 @@ fn upload_ui(items: &[UploadItem], publish: &PublishOpt) -> Result<()> { continue; } } - let filesize = fs::metadata(&i.wheel_path) + let filesize = fs::metadata(&i) .map(|x| ByteSize(x.len()).to_string()) - .unwrap_or_else(|e| { - format!("Failed to get the filesize of {:?}: {}", &i.wheel_path, e) - }); + .unwrap_or_else(|e| format!("Failed to get the filesize of {:?}: {}", &i, e)); return Err(err) .context(format!("💥 Failed to upload {:?} ({})", filename, filesize)); } @@ -559,7 +512,6 @@ fn run() -> Result<()> { eprintln!("âš  Warning: You're publishing debug wheels"); } - let metadata21 = build_context.metadata21.to_vec(); let mut wheels = build_context.build_wheels()?; if !no_sdist { if let Some(sd) = build_context.build_source_distribution()? { @@ -567,10 +519,7 @@ fn run() -> Result<()> { } } - let items = wheels - .into_iter() - .map(|wheel| UploadItem::from_built_wheel(wheel, metadata21.clone())) - .collect::>(); + let items = wheels.into_iter().map(|wheel| wheel.0).collect::>(); upload_ui(&items, &publish)? } @@ -658,12 +607,7 @@ fn run() -> Result<()> { return Ok(()); } - let items = files - .into_iter() - .map(UploadItem::try_from_wheel_path) - .collect::>>()?; - - upload_ui(&items, &publish)? + upload_ui(&files, &publish)? } } diff --git a/src/read_distribution.rs b/src/read_distribution.rs deleted file mode 100644 index d73a32211..000000000 --- a/src/read_distribution.rs +++ /dev/null @@ -1,254 +0,0 @@ -use anyhow::{bail, Context, Result}; -use flate2::read::GzDecoder; -use fs_err::File; -use mailparse::parse_mail; -use regex::Regex; -use std::io::{BufReader, Read}; -use std::path::{Path, PathBuf}; -use zip::ZipArchive; - -fn filename_from_file(path: impl AsRef) -> Result { - Ok(path - .as_ref() - .file_name() - .context("Missing filename")? - .to_str() - .context("Expected a utf-8 filename")? - .to_string()) -} - -/// Standard Python wheel filename components (tags) -/// -/// The wheel filename is "----.whl" -struct WheelFilenameParts { - name: String, - version: String, - python_tag: String, - #[allow(dead_code)] - abi_tag: String, - #[allow(dead_code)] - platform_tag: String, -} - -/// Parses the wheel filename into its components -/// -/// The wheel filename _must_ end with ".whl" -fn parse_wheel_filename(fname: &str) -> Result { - let split: Vec<_> = fname.strip_suffix(".whl").unwrap().split('-').collect(); - - let parts = match split.as_slice() { - [name, version, python_tag, abi_tag, platform_tag] => WheelFilenameParts { - name: name.to_string(), - version: version.to_string(), - python_tag: python_tag.to_string(), - abi_tag: abi_tag.to_string(), - platform_tag: platform_tag.to_string(), - }, - _ => bail!("The wheel filename is invalid: {}", fname), - }; - - Ok(parts) -} - -/// Read the email format into key value pairs -fn metadata_from_bytes(metadata_email: &mut Vec) -> Result> { - let metadata_email = parse_mail(&metadata_email).context("Failed to parse METADATA")?; - - let mut metadata = Vec::new(); - for header in &metadata_email.headers { - metadata.push((header.get_key().to_string(), header.get_value().to_string())); - } - - let body = metadata_email - .get_body() - .context("Failed to parse METADATA")?; - if !body.trim().is_empty() { - metadata.push(("Description".into(), body)); - } - Ok(metadata) -} - -/// Port of pip's `canonicalize_name` -/// https://github.com/pypa/pip/blob/b33e791742570215f15663410c3ed987d2253d5b/src/pip/_vendor/packaging/utils.py#L18-L25 -fn canonicalize_name(name: &str) -> String { - Regex::new("[-_.]+") - .unwrap() - .replace(name, "-") - .to_lowercase() -} - -/// Reads the METADATA file in the .dist-info directory of a wheel, returning -/// the metadata (https://packaging.python.org/specifications/core-metadata/) -/// as key value pairs -fn read_metadata_for_wheel(path: impl AsRef) -> Result> { - let filename = filename_from_file(path.as_ref())?; - let parts = parse_wheel_filename(&filename)?; - - let reader = BufReader::new(File::open(path.as_ref())?); - let mut archive = ZipArchive::new(reader).context("Failed to read file as zip")?; - - // The METADATA format is an email (RFC 822) - // pip's implementation: https://github.com/pypa/pip/blob/b33e791742570215f15663410c3ed987d2253d5b/src/pip/_internal/utils/wheel.py#L109-L144 - // twine's implementation: https://github.com/pypa/twine/blob/534385596820129b41cbcdcc83d34aa8788067f1/twine/wheel.py#L52-L56 - // We mostly follow pip - let mut metadata_email = Vec::new(); - - // Find the metadata file - let name = format!("{}-{}.dist-info/METADATA", parts.name, parts.version); - let metadata_files: Vec<_> = archive - .file_names() - .filter(|i| canonicalize_name(i) == canonicalize_name(&name)) - .map(ToString::to_string) - .collect(); - - match &metadata_files.as_slice() { - [] => bail!( - "This wheel does not contain a METADATA matching {}, which is mandatory for wheels", - name - ), - [metadata_file] => archive - .by_name(&metadata_file) - .context(format!("Failed to read METADATA file {}", metadata_file))? - .read_to_end(&mut metadata_email) - .context(format!("Failed to read METADATA file {}", metadata_file))?, - files => bail!( - "Found more than one metadata file matching {}: {:?}", - name, - files - ), - }; - - metadata_from_bytes(&mut metadata_email) -} - -/// Returns the metadata for a source distribution (.tar.gz). -/// Only parses the filename since dist-info is not part of source -/// distributions -fn read_metadata_for_source_distribution(path: impl AsRef) -> Result> { - // "dist/foo_ext-1.0.1.tar.gz" -> "foo_ext-1.0.1/PKG-INFO" - let mut pkginfo: PathBuf = path.as_ref().file_name().unwrap().into(); - pkginfo.set_extension(""); - pkginfo.set_extension(""); - pkginfo.push("PKG-INFO"); - - let mut reader = tar::Archive::new(GzDecoder::new(BufReader::new(File::open(path.as_ref())?))); - // Unlike for wheels, in source distributions the metadata is stored in a file called PKG-INFO - // try_find would be ideal here, but it's nightly only - let mut entry = reader - .entries()? - .map(|entry| -> Result<_> { - let entry = entry?; - if entry.path()? == pkginfo { - Ok(Some(entry)) - } else { - Ok(None) - } - }) - .find_map(|x| x.transpose()) - .context(format!( - "Source distribution {:?} does not contain a PKG-INFO, but it should", - path.as_ref() - ))? - .context(format!("Failed to read {:?}", path.as_ref()))?; - let mut metadata_email = Vec::new(); - entry - .read_to_end(&mut metadata_email) - .context(format!("Failed to read {:?}", path.as_ref()))?; - metadata_from_bytes(&mut metadata_email) -} - -/// Returns the metadata as key value pairs for a wheel or a source distribution -pub fn get_metadata_for_distribution(path: &Path) -> Result> { - let filename = filename_from_file(path)?; - if filename.ends_with(".whl") { - read_metadata_for_wheel(path) - .context(format!("Failed to read metadata from wheel at {:?}", path)) - } else if filename.ends_with(".tar.gz") { - read_metadata_for_source_distribution(path).context(format!( - "Failed to read metadata from source distribution at {:?}", - path - )) - } else { - bail!("File has an unknown extension: {:?}", path) - } -} - -/// Returns the supported Python interpreter version tag for a wheel or a source distribution -/// -/// The version tag is encoded in the wheel file name and usually looks like "py3" or "cp37". -/// For the source distributions the version tag is always "source". -pub fn get_supported_version_for_distribution(path: &Path) -> Result { - let filename = filename_from_file(path)?; - - let python_tag = if filename.ends_with(".whl") { - parse_wheel_filename(&filename)?.python_tag - } else if filename.ends_with(".tar.gz") { - "source".to_string() - } else { - bail!("File has an unknown extension: {:?}", path) - }; - - Ok(python_tag) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_source_distribution() { - let metadata = - get_metadata_for_distribution(Path::new("test-data/pyo3_mixed-2.1.1.tar.gz")).unwrap(); - let expected: Vec<_> = [ - ("Metadata-Version", "2.1"), - ("Name", "pyo3-mixed"), - ("Version", "2.1.1"), - ("Summary", "Implements a dummy function combining rust and python"), - ("Author", "konstin "), - ("Author-Email", "konstin "), - ("Description-Content-Type", "text/markdown; charset=UTF-8; variant=GFM"), - ("Description", "# pyo3-mixed\n\nA package for testing maturin with a mixed pyo3/python project.\n\n"), - ].iter().map(|(k,v)| (k.to_string(), v.to_string())).collect(); - - assert_eq!(metadata, expected); - } - - #[test] - fn test_wheel() { - let metadata = get_metadata_for_distribution(Path::new( - "test-data/pyo3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl", - )) - .unwrap(); - assert_eq!( - metadata.iter().map(|x| &x.0).collect::>(), - vec![ - "Metadata-Version", - "Name", - "Version", - "Summary", - "Author", - "Author-Email", - "Description-Content-Type", - "Description" - ] - ); - // Check the description - assert!(metadata[7].1.starts_with("# pyo3-mixed")); - assert!(metadata[7].1.ends_with("tox.ini\n\n")); - } - - #[test] - fn test_supported_version() { - let path = Path::new("test-data/pyo3_mixed-2.1.1.tar.gz"); - let supported_version = get_supported_version_for_distribution(path).unwrap(); - assert_eq!(supported_version, "source"); - - let path = Path::new("test-data/pyo3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl"); - let supported_version = get_supported_version_for_distribution(path).unwrap(); - assert_eq!(supported_version, "cp38"); - - let path = Path::new("test_data/pyo3_stubs-2.1.1-py3-none-any.whl"); - let supported_version = get_supported_version_for_distribution(path).unwrap(); - assert_eq!(supported_version, "py3"); - } -} diff --git a/src/upload.rs b/src/upload.rs index 1cd5b99f2..4be840fe7 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -3,10 +3,11 @@ use crate::Registry; use fs_err::File; +use regex::Regex; use reqwest::{self, blocking::multipart::Form, blocking::Client, StatusCode}; use sha2::{Digest, Sha256}; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use thiserror::Error; /// Error type for different types of errors that can happen when uploading a @@ -32,6 +33,9 @@ pub enum UploadError { /// File already exists #[error("File already exists: {0}")] FileExistsError(String), + /// Read package metadata error + #[error("Could not read the metadata from the package at {0}")] + PkgInfoError(PathBuf, #[source] python_pkginfo::Error), } impl From for UploadError { @@ -46,51 +50,78 @@ impl From for UploadError { } } +/// Port of pip's `canonicalize_name` +/// https://github.com/pypa/pip/blob/b33e791742570215f15663410c3ed987d2253d5b/src/pip/_vendor/packaging/utils.py#L18-L25 +fn canonicalize_name(name: &str) -> String { + Regex::new("[-_.]+") + .unwrap() + .replace(name, "-") + .to_lowercase() +} + /// Uploads a single wheel to the registry -pub fn upload( - registry: &Registry, - wheel_path: &Path, - metadata21: &[(String, String)], - supported_version: &str, -) -> Result<(), UploadError> { +pub fn upload(registry: &Registry, wheel_path: &Path) -> Result<(), UploadError> { let mut wheel = File::open(&wheel_path)?; let mut hasher = Sha256::new(); io::copy(&mut wheel, &mut hasher)?; let hash_hex = format!("{:x}", hasher.finalize()); + let dist = python_pkginfo::Distribution::new(wheel_path) + .map_err(|err| UploadError::PkgInfoError(wheel_path.to_owned(), err))?; + let metadata = dist.metadata(); + let mut api_metadata = vec![ - (":action".to_string(), "file_upload".to_string()), - ("sha256_digest".to_string(), hash_hex), - ("protocol_version".to_string(), "1".to_string()), + (":action", "file_upload".to_string()), + ("sha256_digest", hash_hex), + ("protocol_version", "1".to_string()), + ("metadata_version", metadata.metadata_version.clone()), + ("name", canonicalize_name(&metadata.name)), + ("version", metadata.version.clone()), + ("pyversion", dist.python_version().to_string()), + ("filetype", dist.r#type().to_string()), ]; - api_metadata.push(("pyversion".to_string(), supported_version.to_string())); + let mut add_option = |name, value: &Option| { + if let Some(some) = value.clone() { + api_metadata.push((name, some)); + } + }; + + // https://github.com/pypa/warehouse/blob/75061540e6ab5aae3f8758b569e926b6355abea8/warehouse/forklift/legacy.py#L424 + add_option("summary", &metadata.summary); + add_option("description", &metadata.description); + add_option( + "description_content_type", + &metadata.description_content_type, + ); + add_option("author", &metadata.author); + add_option("author_email", &metadata.author_email); + add_option("maintainer", &metadata.maintainer); + add_option("maintainer_email", &metadata.maintainer_email); + add_option("license", &metadata.license); + add_option("keywords", &metadata.keywords); + add_option("home_page", &metadata.home_page); + add_option("download_url", &metadata.download_url); + add_option("requires_path", &metadata.requires_python); + add_option("summary", &metadata.summary); - if supported_version != "source" { - api_metadata.push(("filetype".to_string(), "bdist_wheel".to_string())); - } else { - api_metadata.push(("filetype".to_string(), "sdist".to_string())); - } + let mut add_vec = |name, values: &[String]| { + for i in values { + api_metadata.push((name, i.clone())); + } + }; - let joined_metadata: Vec<(String, String)> = api_metadata - .into_iter() - // Type system shenanigans - .chain(metadata21.to_vec().into_iter()) - // All fields must be lower case and with underscores or they will be ignored by warehouse - .map(|(key, value)| { - let mut key = key.to_lowercase().replace("-", "_"); - if key == "classifier" { - // PyPI upload api expects `classifiers` instead of `classifier` - // See https://github.com/pypa/warehouse/issues/3151#issuecomment-796965735 - key = "classifiers".to_string(); - } - (key, value) - }) - .collect(); + add_vec("classifiers", &metadata.classifiers); + add_vec("platform", &metadata.platforms); + add_vec("requires_dist", &metadata.requires_dist); + add_vec("provides_dist", &metadata.provides_dist); + add_vec("obsoletes_dist", &metadata.obsoletes_dist); + add_vec("requires_external", &metadata.requires_external); + add_vec("project_urls", &metadata.project_urls); let mut form = Form::new(); - for (key, value) in joined_metadata { - form = form.text(key, value.to_owned()) + for (key, value) in api_metadata { + form = form.text(key, value); } form = form.file("content", &wheel_path)?; From 8f33c223be12c972c23e8e88431584a5275d0920 Mon Sep 17 00:00:00 2001 From: messense Date: Thu, 10 Jun 2021 13:57:43 +0800 Subject: [PATCH 2/2] Remove now unused pyo3_mixed* test files --- test-data/Readme.md | 1 - ...3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl | Bin 2061 -> 0 bytes test-data/pyo3_mixed-2.1.1.tar.gz | Bin 403 -> 0 bytes 3 files changed, 1 deletion(-) delete mode 100644 test-data/pyo3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl delete mode 100644 test-data/pyo3_mixed-2.1.1.tar.gz diff --git a/test-data/Readme.md b/test-data/Readme.md index bd415bc70..ce3ac75d2 100644 --- a/test-data/Readme.md +++ b/test-data/Readme.md @@ -1,2 +1 @@ * `py.exe`: Mock for the windows python launcher we can insert in path - * `pyo3_mixed-2.1.1.tar.gz` and `pyo3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl`: Near empty archives to test the metadata reader. \ No newline at end of file diff --git a/test-data/pyo3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl b/test-data/pyo3_mixed-2.1.1-cp38-cp38-manylinux1_x86_64.whl deleted file mode 100644 index f3aae23ad9024e581c691e30dcb7ddde208b33da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2061 zcmai#-E!hK6vx?QyVLP??|P$lZ+551!G=I!OQ$4!CgCH*0g|1L5k_EvExDGw;N0c9 z)0sX)CpYb5^d)-VC#j4>A?z;2BMg$x?;M@;KX&Kh@a0#D=<}OVzWVjx>mfQwB%<+J z&UqnAn=V=6G1OH}USq-qBy_@#niZo|E*YgJ7HEtFf<5eW!e{}=ok|P2xBwz^xH4C^ zSLOP3;JU~U3!ve84t6mW93U{3vmuxS)Dn^eEar{~C3Fh>fD3@=SQ3TJs8R~#%`bpC zqg)UQzJL1hql<_$VoR0z(;uF8!Admwe|yWATR!n3zwnGvAtQoD=*dF}yc9<=*!h?( z=sRH9$mduTZ;UEDc?UiqpCBrVwQ5sQUT>dRDFF{zb2P%#8l#EhrX2Mvn>7jS&Z@D2*q$V9Ua%-O82NtQ$~3#MZ9=Rty`jjw?&* zV;L_PMu#^yVp6?um;?)(IC7)hY%w|^)4)fovJ)QJNSZ2(uPB{o3jOjS{STGOj0V`R z(0;vAx$N5*I~#3kYD%eLilwVaKkH(=NvU}?1(D;~NY&mdUB(1#aM%fa?2u6bOpwE| zVxVaOSYG-BuD)DELx<2{shP`@teMU1#QMu1`|Wo!#`V}F;G=Gma`QveV?;KBDwg8C zObKTWHe*%tpOm4RTsvkEbM4R@nCSHGg0b=f!nU8?yY1tl~?wlSG8GlvlluXEN;VG z`R94Q9p+p*!V|NToi!+}G5ywZSmwOBtoG}-Nli=b5h?V{G>lTml-w%PqSOKUfp zJsVxqMylMcd**Q?W8LU}^ofeO4+Xe)*wuKc9zwHC?J3C9Sh0 zd;%-_z-;3>#7JMr6zkgaIM?q0)ry}qP&Pm;c({^3oYFw$Y@HH3)iQbwVmH%D1?jGB zj6s$)wcziv#uW^Y1Dt_SNU{A-e4$ zu4TOo+?xpQYco)3e;T!N@`@wN0I~QNPicDV0 StM($1_*4FSmAEG!^!dMaR(9_I diff --git a/test-data/pyo3_mixed-2.1.1.tar.gz b/test-data/pyo3_mixed-2.1.1.tar.gz deleted file mode 100644 index b5e9c02864f2de5a9961325c40ca015f1546c496..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 403 zcmV;E0c`#siwFP!000001MSk^PunmM2k<=SuQ+MXY2zkorZUhFP=pX{5=@?P0j;@(>Spqjt?|MT=XIXR#H*y0&|gU5OP zV*iUQi$qqAMUe|pavZ;iQAFt$b9m7w@Bf(!vX((6*UGor*$MGAh|i@_6YB3qvc4Pf z<|?mWbjC=(n$U@9?vznBv?QtK*%jS(whA0el`}WmYC9v(KbfShdB-q!Hs0@e@Do}% z+Xijv+tb5$BlX>lTMnF`?L9>$NB!hmv4^Uyyl(n)lLKc1XOvv78ZIKJWf*d@i`qTd z9jfNiw<_#jT~3p)J9IC-mNx7jPtW4`%Vx##J~gsh$eE&B=Xsm{&>0ySI?ri5=rHG! x*Rku*hJR|K#=GCD3WGTQ->n4z000000000000000004kb;1LYfe-Z#F000Ad#Pa|E