From ef2b09a36e559f9eb30640d43f996d0750d3a231 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 27 Dec 2022 19:47:14 +0800 Subject: [PATCH] Add support for packaging multiple pure Python packages --- Changelog.md | 1 + guide/src/metadata.md | 4 ++ src/module_writer.rs | 51 ++++++++++++++-------- src/project_layout.rs | 5 +++ src/pyproject_toml.rs | 13 ++++++ src/source_distribution.rs | 52 +++++++++++++++-------- test-crates/pyo3-mixed-src/pyproject.toml | 3 ++ tests/run.rs | 1 + 8 files changed, 94 insertions(+), 36 deletions(-) diff --git a/Changelog.md b/Changelog.md index 1dbf44b8a..58d8d5bc5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **Breaking Change**: Remove deprecated `python-source` option in `Cargo.toml` in [#1335](https://github.com/PyO3/maturin/pull/1335) * **Breaking Change**: Turn `patchelf` version warning into a hard error in [#1335](https://github.com/PyO3/maturin/pull/1335) * **Breaking Change**: [`uniffi_bindgen` CLI](https://mozilla.github.io/uniffi-rs/tutorial/Prerequisites.html#the-uniffi-bindgen-cli-tool) is required for building `uniffi` bindings wheels in [#1352](https://github.com/PyO3/maturin/pull/1352) +* Add support for packaging multiple pure Python packages in [#1378](https://github.com/PyO3/maturin/pull/1378) ## [0.14.7] - 2022-12-20 diff --git a/guide/src/metadata.md b/guide/src/metadata.md index eb61cb9be..f6333737b 100644 --- a/guide/src/metadata.md +++ b/guide/src/metadata.md @@ -117,6 +117,10 @@ bindings = "pyo3" compatibility = "manylinux2014" # Don't check for manylinux compliance skip-auditwheel = false +# Python source directory +python-source = "src" +# Python packages to include +python-packages = ["foo", "bar"] # Strip the library for minimum file size strip = true # Build artifacts with the specified Cargo profile diff --git a/src/module_writer.rs b/src/module_writer.rs index 06b5080f8..738f14af4 100644 --- a/src/module_writer.rs +++ b/src/module_writer.rs @@ -303,7 +303,7 @@ impl WheelWriter { debug!("Adding {} from {}", target, python_path); self.add_bytes(target, python_path.as_bytes())?; } else { - println!("⚠️ source code path contains non-Unicode sequences, editable installs may not work."); + eprintln!("⚠️ source code path contains non-Unicode sequences, editable installs may not work."); } } Ok(()) @@ -1098,31 +1098,46 @@ pub fn write_python_part( python_module: impl AsRef, pyproject_toml: Option<&PyProjectToml>, ) -> Result<()> { - let python_module = python_module.as_ref(); - for absolute in WalkBuilder::new(python_module).hidden(false).build() { - let absolute = absolute?.into_path(); - let relative = absolute - .strip_prefix(python_module.parent().unwrap()) - .unwrap(); - if absolute.is_dir() { - writer.add_directory(relative)?; - } else { - // Ignore native libraries from develop, if any - if let Some(extension) = relative.extension() { - if extension.to_string_lossy() == "so" { - debug!("Ignoring native library {}", relative.display()); + let python_module = python_module.as_ref().to_path_buf(); + let python_dir = python_module.parent().unwrap().to_path_buf(); + let mut python_packages = vec![python_module]; + if let Some(pyproject_toml) = pyproject_toml { + if let Some(packages) = pyproject_toml.python_packages() { + for package in packages { + let package_path = python_dir.join(package); + if python_packages.iter().any(|p| *p == package_path) { continue; } + python_packages.push(package_path); + } + } + } + + for package in python_packages { + for absolute in WalkBuilder::new(package).hidden(false).build() { + let absolute = absolute?.into_path(); + let relative = absolute.strip_prefix(&python_dir).unwrap(); + if absolute.is_dir() { + writer.add_directory(relative)?; + } else { + // Ignore native libraries from develop, if any + if let Some(extension) = relative.extension() { + if extension.to_string_lossy() == "so" { + debug!("Ignoring native library {}", relative.display()); + continue; + } + } + writer + .add_file(relative, &absolute) + .context(format!("File to add file from {}", absolute.display()))?; } - writer - .add_file(relative, &absolute) - .context(format!("File to add file from {}", absolute.display()))?; } } // Include additional files if let Some(pyproject) = pyproject_toml { - let pyproject_dir = python_module.parent().unwrap(); + // FIXME: in src-layout pyproject.toml isn't located directly in python dir + let pyproject_dir = &python_dir; if let Some(glob_patterns) = pyproject.include() { for pattern in glob_patterns .iter() diff --git a/src/project_layout.rs b/src/project_layout.rs index 30d8fd2de..1645a2969 100644 --- a/src/project_layout.rs +++ b/src/project_layout.rs @@ -13,6 +13,8 @@ const PYPROJECT_TOML: &str = "pyproject.toml"; /// Whether this project is pure rust or rust mixed with python and whether it has wheel data #[derive(Clone, Debug, PartialEq, Eq)] pub struct ProjectLayout { + /// Contains the absolute path to the python source directory + pub python_dir: PathBuf, /// Contains the canonicalized (i.e. absolute) path to the python part of the project /// If none, we have a rust crate compiled into a shared library with only some glue python for cffi /// If some, we have a python package that is extended by a native rust module. @@ -342,6 +344,7 @@ impl ProjectLayout { }; debug!( project_root = %project_root.display(), + python_dir = %python_root.display(), rust_module = %rust_module.display(), python_module = %python_module.display(), extension_name = %extension_name, @@ -369,6 +372,7 @@ impl ProjectLayout { println!("🍹 Building a mixed python/rust project"); Ok(ProjectLayout { + python_dir: python_root, python_module: Some(python_module), rust_module, extension_name, @@ -376,6 +380,7 @@ impl ProjectLayout { }) } else { Ok(ProjectLayout { + python_dir: python_root, python_module: None, rust_module: project_root.to_path_buf(), extension_name, diff --git a/src/pyproject_toml.rs b/src/pyproject_toml.rs index 7ff2087e6..d8ec33918 100644 --- a/src/pyproject_toml.rs +++ b/src/pyproject_toml.rs @@ -97,6 +97,8 @@ pub struct ToolMaturin { strip: bool, /// The directory with python module, contains `/__init__.py` python_source: Option, + /// Python packages to include + python_packages: Option>, /// Path to the wheel directory, defaults to `.data` data: Option, // Some customizable cargo options @@ -208,6 +210,12 @@ impl PyProjectToml { .and_then(|maturin| maturin.python_source.as_deref()) } + /// Returns the value of `[tool.maturin.python-packages]` in pyproject.toml + pub fn python_packages(&self) -> Option<&[String]> { + self.maturin() + .and_then(|maturin| maturin.python_packages.as_deref()) + } + /// Returns the value of `[tool.maturin.data]` in pyproject.toml pub fn data(&self) -> Option<&Path> { self.maturin().and_then(|maturin| maturin.data.as_deref()) @@ -291,6 +299,7 @@ mod tests { [tool.maturin] manylinux = "2010" + python-packages = ["foo", "bar"] manifest-path = "Cargo.toml" profile = "dev" features = ["foo", "bar"] @@ -317,6 +326,10 @@ mod tests { maturin.rustc_args, Some(vec!["-Z".to_string(), "unstable-options".to_string()]) ); + assert_eq!( + maturin.python_packages, + Some(vec!["foo".to_string(), "bar".to_string()]) + ); } #[test] diff --git a/src/source_distribution.rs b/src/source_distribution.rs index 36f6a59ac..898323ecf 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -620,26 +620,42 @@ pub fn source_distribution( let pyproject_dir = pyproject_toml_path.parent().unwrap(); // Add python source files - if let Some(python_source) = build_context.project_layout.python_module.as_ref() { - for entry in ignore::Walk::new(python_source) { - let source = entry?.into_path(); - // Technically, `ignore` crate should handle this, - // but somehow it doesn't on Alpine Linux running in GitHub Actions, - // so we do it manually here. - // See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013 - if source - .extension() - .map(|ext| ext == "pyc" || ext == "pyd" || ext == "so") - .unwrap_or_default() - { - debug!("Ignoring {}", source.display()); + if let Some(python_module) = build_context.project_layout.python_module.as_ref() { + let mut python_packages = vec![python_module.to_path_buf()]; + for package in build_context + .pyproject_toml + .as_ref() + .and_then(|toml| toml.python_packages()) + .unwrap_or_default() + { + let package_path = build_context.project_layout.python_dir.join(package); + if python_packages.iter().any(|p| *p == package_path) { continue; } - let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap()); - if source.is_dir() { - writer.add_directory(target)?; - } else { - writer.add_file(target, &source)?; + python_packages.push(package_path); + } + + for package in python_packages { + for entry in ignore::Walk::new(package) { + let source = entry?.into_path(); + // Technically, `ignore` crate should handle this, + // but somehow it doesn't on Alpine Linux running in GitHub Actions, + // so we do it manually here. + // See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013 + if source + .extension() + .map(|ext| ext == "pyc" || ext == "pyd" || ext == "so") + .unwrap_or_default() + { + debug!("Ignoring {}", source.display()); + continue; + } + let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap()); + if source.is_dir() { + writer.add_directory(target)?; + } else { + writer.add_file(target, &source)?; + } } } } diff --git a/test-crates/pyo3-mixed-src/pyproject.toml b/test-crates/pyo3-mixed-src/pyproject.toml index 73544cf8f..4294e0c07 100644 --- a/test-crates/pyo3-mixed-src/pyproject.toml +++ b/test-crates/pyo3-mixed-src/pyproject.toml @@ -12,3 +12,6 @@ requires-python = ">=3.7" [project.scripts] get_42 = "pyo3_mixed_src:get_42" + +[tool.maturin] +python-packages = ["pyo3_mixed_src", "tests"] diff --git a/tests/run.rs b/tests/run.rs index 5a5d178d2..723a9963e 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -491,6 +491,7 @@ fn pyo3_mixed_src_layout_sdist() { "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/__init__.py", "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/python_module/__init__.py", "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/python_module/double.py", + "pyo3_mixed_src-2.1.3/src/tests/test_pyo3_mixed.py", "pyo3_mixed_src-2.1.3/rust/Cargo.toml", "pyo3_mixed_src-2.1.3/rust/Cargo.lock", "pyo3_mixed_src-2.1.3/rust/src/lib.rs",