Skip to content

Commit

Permalink
Merge #1380
Browse files Browse the repository at this point in the history 
1380: Detect src-layout for pure Rust projects with multiple Python packages r=messense a=messense

Follow-up of #1378, `python-packages` actually slightly changes the meaning of mixed Rust/Python project layout.

Co-authored-by: messense <[email protected]>
  • Loading branch information
@bors @messense
bors[bot] and messense authored Dec 30, 2022
2 parents c627d88 + 79f2c17 commit 037442b
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 89 deletions.
22 changes: 12 additions & 10 deletions src/build_context.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -926,16 +926,18 @@ impl BuildContext {
self.excludes(Format::Wheel)?,
)?;

if let Some(python_module) = &self.project_layout.python_module {
if self.target.is_wasi() {
// TODO: Can we have python code and the wasm launchers coexisting
// without clashes?
bail!("Sorry, adding python code to a wasm binary is currently not supported")
}
if !self.editable {
write_python_part(&mut writer, python_module, self.pyproject_toml.as_ref())
.context("Failed to add the python module to the package")?;
}
if self.project_layout.python_module.is_some() && self.target.is_wasi() {
// TODO: Can we have python code and the wasm launchers coexisting
// without clashes?
bail!("Sorry, adding python code to a wasm binary is currently not supported")
}
if !self.editable {
write_python_part(
&mut writer,
&self.project_layout,
self.pyproject_toml.as_ref(),
)
.context("Failed to add the python module to the package")?;
}

let mut artifacts_ref = Vec::with_capacity(artifacts.len());
Expand Down
63 changes: 30 additions & 33 deletions src/module_writer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,9 +295,9 @@ impl WheelWriter {
project_layout: &ProjectLayout,
metadata21: &Metadata21,
) -> Result<()> {
if let Some(python_module) = &project_layout.python_module {
let absolute_path = python_module.normalize()?.into_path_buf();
if let Some(python_path) = absolute_path.parent().and_then(|p| p.to_str()) {
if project_layout.python_module.is_some() || !project_layout.python_packages.is_empty() {
let absolute_path = project_layout.python_dir.normalize()?.into_path_buf();
if let Some(python_path) = absolute_path.to_str() {
let name = metadata21.get_distribution_escaped();
let target = format!("{}.pth", name);
debug!("Adding {} from {}", target, python_path);
Expand Down Expand Up @@ -720,6 +720,10 @@ pub fn write_bindings_module(
}
};

if !editable {
write_python_part(writer, project_layout, pyproject_toml)
.context("Failed to add the python module to the package")?;
}
if let Some(python_module) = &project_layout.python_module {
if editable {
let target = project_layout.rust_module.join(&so_filename);
Expand All @@ -735,9 +739,6 @@ pub fn write_bindings_module(
target.display()
))?;
} else {
write_python_part(writer, python_module, pyproject_toml)
.context("Failed to add the python module to the package")?;

let relative = project_layout
.rust_module
.strip_prefix(python_module.parent().unwrap())
Expand Down Expand Up @@ -789,14 +790,13 @@ pub fn write_cffi_module(
) -> Result<()> {
let cffi_declarations = generate_cffi_declarations(crate_dir, target_dir, python)?;

let module;
if !editable {
write_python_part(writer, project_layout, pyproject_toml)
.context("Failed to add the python module to the package")?;
}

let module;
if let Some(python_module) = &project_layout.python_module {
if !editable {
write_python_part(writer, python_module, pyproject_toml)
.context("Failed to add the python module to the package")?;
}

if editable {
let base_path = python_module.join(module_name);
fs::create_dir_all(&base_path)?;
Expand Down Expand Up @@ -956,14 +956,13 @@ pub fn write_uniffi_module(
} = generate_uniffi_bindings(crate_dir, target_dir, target_os)?;
let py_init = format!("from .{} import * # NOQA\n", binding_name);

let module;
if !editable {
write_python_part(writer, project_layout, pyproject_toml)
.context("Failed to add the python module to the package")?;
}

let module;
if let Some(python_module) = &project_layout.python_module {
if !editable {
write_python_part(writer, python_module, pyproject_toml)
.context("Failed to add the python module to the package")?;
}

if editable {
let base_path = python_module.join(module_name);
fs::create_dir_all(&base_path)?;
Expand Down Expand Up @@ -1095,28 +1094,26 @@ if __name__ == '__main__':
/// Adds the python part of a mixed project to the writer,
pub fn write_python_part(
writer: &mut impl ModuleWriter,
python_module: impl AsRef<Path>,
project_layout: &ProjectLayout,
pyproject_toml: Option<&PyProjectToml>,
) -> Result<()> {
let python_module = python_module.as_ref().to_path_buf();
let python_dir = python_module.parent().unwrap().to_path_buf();
let mut python_packages = vec![python_module];
if let Some(pyproject_toml) = pyproject_toml {
if let Some(packages) = pyproject_toml.python_packages() {
for package in packages {
let package_path = python_dir.join(package);
if python_packages.iter().any(|p| *p == package_path) {
continue;
}
python_packages.push(package_path);
}
let python_dir = &project_layout.python_dir;
let mut python_packages = Vec::new();
if let Some(python_module) = project_layout.python_module.as_ref() {
python_packages.push(python_module.to_path_buf());
}
for package in &project_layout.python_packages {
let package_path = python_dir.join(package);
if python_packages.iter().any(|p| *p == package_path) {
continue;
}
python_packages.push(package_path);
}

for package in python_packages {
for absolute in WalkBuilder::new(package).hidden(false).build() {
let absolute = absolute?.into_path();
let relative = absolute.strip_prefix(&python_dir).unwrap();
let relative = absolute.strip_prefix(python_dir).unwrap();
if absolute.is_dir() {
writer.add_directory(relative)?;
} else {
Expand All @@ -1137,7 +1134,7 @@ pub fn write_python_part(
// Include additional files
if let Some(pyproject) = pyproject_toml {
// FIXME: in src-layout pyproject.toml isn't located directly in python dir
let pyproject_dir = &python_dir;
let pyproject_dir = python_dir;
if let Some(glob_patterns) = pyproject.include() {
for pattern in glob_patterns
.iter()
Expand Down
50 changes: 37 additions & 13 deletions src/project_layout.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use crate::{CargoToml, Metadata21, PyProjectToml};
use anyhow::{bail, format_err, Context, Result};
use cargo_metadata::{Metadata, MetadataCommand};
use normpath::PathExt as _;
use std::collections::HashSet;
use std::env;
use std::io;
use std::path::{Path, PathBuf};
Expand All @@ -19,6 +20,8 @@ pub struct ProjectLayout {
/// If none, we have a rust crate compiled into a shared library with only some glue python for cffi
/// If some, we have a python package that is extended by a native rust module.
pub python_module: Option<PathBuf>,
/// Python packages to include
pub python_packages: Vec<String>,
/// Contains the canonicalized (i.e. absolute) path to the rust part of the project
pub rust_module: PathBuf,
/// Rust extension name
Expand Down Expand Up @@ -137,19 +140,30 @@ impl ProjectResolver {
} else {
manifest_dir
};
let python_packages = pyproject
.and_then(|x| x.python_packages())
.unwrap_or_default()
.to_vec();
let py_root = match pyproject.and_then(|x| x.python_source()) {
Some(py_src) => project_root.join(py_src),
None => match pyproject.and_then(|x| x.project_name()) {
Some(project_name) => {
// Detect src layout
let import_name = project_name.replace('-', "_");
let rust_cargo_toml_found =
project_root.join("rust").join("Cargo.toml").is_file();
let python_src_found = project_root
.join("src")
.join(import_name)
.join("__init__.py")
.is_file();
let import_name = project_name.replace('-', "_");
let mut package_init = HashSet::new();
package_init.insert(
project_root
.join("src")
.join(import_name)
.join("__init__.py"),
);
for package in &python_packages {
package_init
.insert(project_root.join("src").join(package).join("__init__.py"));
}
let python_src_found = package_init.iter().any(|x| x.is_file());
if rust_cargo_toml_found && python_src_found {
project_root.join("src")
} else {
Expand All @@ -176,7 +190,8 @@ impl ProjectResolver {
}
}),
};
let project_layout = ProjectLayout::determine(project_root, extension_name, py_root, data)?;
let project_layout =
ProjectLayout::determine(project_root, extension_name, py_root, python_packages, data)?;
Ok(Self {
project_layout,
cargo_toml_path: manifest_file,
Expand Down Expand Up @@ -259,12 +274,18 @@ impl ProjectResolver {
} else if let Some(project_name) = pyproject.project_name() {
// Check if python source directory in `src/<project_name>`
let import_name = project_name.replace('-', "_");
if current_dir
.join("src")
.join(import_name)
.join("__init__.py")
.is_file()
{
let mut package_init = HashSet::new();
package_init.insert(
current_dir
.join("src")
.join(import_name)
.join("__init__.py"),
);
for package in pyproject.python_packages().unwrap_or_default() {
package_init
.insert(current_dir.join("src").join(package).join("__init__.py"));
}
if package_init.iter().any(|x| x.is_file()) {
return Ok((path, pyproject_file));
}
}
Expand Down Expand Up @@ -322,6 +343,7 @@ impl ProjectLayout {
project_root: impl AsRef<Path>,
module_name: &str,
python_root: PathBuf,
python_packages: Vec<String>,
data: Option<PathBuf>,
) -> Result<ProjectLayout> {
// A dot in the module name means the extension module goes into the module folder specified by the path
Expand Down Expand Up @@ -373,6 +395,7 @@ impl ProjectLayout {

Ok(ProjectLayout {
python_dir: python_root,
python_packages,
python_module: Some(python_module),
rust_module,
extension_name,
Expand All @@ -381,6 +404,7 @@ impl ProjectLayout {
} else {
Ok(ProjectLayout {
python_dir: python_root,
python_packages,
python_module: None,
rust_module: project_root.to_path_buf(),
extension_name,
Expand Down
62 changes: 29 additions & 33 deletions src/source_distribution.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -620,42 +620,38 @@ pub fn source_distribution(

let pyproject_dir = pyproject_toml_path.parent().unwrap();
// Add python source files
let mut python_packages = Vec::new();
if let Some(python_module) = build_context.project_layout.python_module.as_ref() {
let mut python_packages = vec![python_module.to_path_buf()];
for package in build_context
.pyproject_toml
.as_ref()
.and_then(|toml| toml.python_packages())
.unwrap_or_default()
{
let package_path = build_context.project_layout.python_dir.join(package);
if python_packages.iter().any(|p| *p == package_path) {
continue;
}
python_packages.push(package_path);
python_packages.push(python_module.to_path_buf());
}
for package in &build_context.project_layout.python_packages {
let package_path = build_context.project_layout.python_dir.join(package);
if python_packages.iter().any(|p| *p == package_path) {
continue;
}
python_packages.push(package_path);
}

for package in python_packages {
for entry in ignore::Walk::new(package) {
let source = entry?.into_path();
// Technically, `ignore` crate should handle this,
// but somehow it doesn't on Alpine Linux running in GitHub Actions,
// so we do it manually here.
// See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013
if source
.extension()
.map(|ext| ext == "pyc" || ext == "pyd" || ext == "so")
.unwrap_or_default()
{
debug!("Ignoring {}", source.display());
continue;
}
let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
if source.is_dir() {
writer.add_directory(target)?;
} else {
writer.add_file(target, &source)?;
}
for package in python_packages {
for entry in ignore::Walk::new(package) {
let source = entry?.into_path();
// Technically, `ignore` crate should handle this,
// but somehow it doesn't on Alpine Linux running in GitHub Actions,
// so we do it manually here.
// See https://github.com/PyO3/maturin/pull/1187#issuecomment-1273987013
if source
.extension()
.map(|ext| ext == "pyc" || ext == "pyd" || ext == "so")
.unwrap_or_default()
{
debug!("Ignoring {}", source.display());
continue;
}
let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
if source.is_dir() {
writer.add_directory(target)?;
} else {
writer.add_file(target, &source)?;
}
}
}
Expand Down

0 comments on commit 037442b

Please sign in to comment.