From 5eb0fcb3655206a81ac6eb3a7c71a9450da23223 Mon Sep 17 00:00:00 2001 From: konstin Date: Sun, 8 May 2022 21:13:43 +0200 Subject: [PATCH] Add wheel data support This also introduces tracing for logging, which is neat because you can e.g. `RUST_LOG=...` at runtime. Fixes #258 --- .github/workflows/lint.yml | 2 +- Cargo.lock | 33 +++ Cargo.toml | 1 + guide/src/project_layout.md | 14 + src/build_context.rs | 96 +++---- src/build_options.rs | 6 +- src/cargo_toml.rs | 3 + src/main.rs | 2 +- src/module_writer.rs | 252 ++++++++++-------- src/pyproject_toml.rs | 10 + src/source_distribution.rs | 4 +- test-crates/pyo3-pure/pyo3_pure.pyi | 2 +- test-crates/with-data/Cargo.lock | 7 + test-crates/with-data/Cargo.toml | 9 + .../check_installed/check_installed.py | 33 +++ test-crates/with-data/pyproject.toml | 7 + test-crates/with-data/src/lib.rs | 12 + .../with_data.data/data/data_subdir/hello.txt | 1 + .../with-data/with_data.data/headers/empty.h | 0 tests/common/integration.rs | 2 +- tests/common/mod.rs | 2 +- tests/run.rs | 9 + 22 files changed, 344 insertions(+), 163 deletions(-) create mode 100644 test-crates/with-data/Cargo.lock create mode 100644 test-crates/with-data/Cargo.toml create mode 100644 test-crates/with-data/check_installed/check_installed.py create mode 100644 test-crates/with-data/pyproject.toml create mode 100644 test-crates/with-data/src/lib.rs create mode 100644 test-crates/with-data/with_data.data/data/data_subdir/hello.txt create mode 100644 test-crates/with-data/with_data.data/headers/empty.h diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26fb5bede..a438f7530 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - uses: psf/black@22.1.0 + - uses: psf/black@22.3.0 spellcheck: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 15430cc19..7f0fd97d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1117,6 +1117,7 @@ dependencies = [ "textwrap", "thiserror", "toml_edit", + "tracing", "ureq", "zip", ] @@ -2088,6 +2089,38 @@ dependencies = [ "serde", ] +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", +] + [[package]] name = "typenum" version = "1.15.0" diff --git a/Cargo.toml b/Cargo.toml index 27a37d0f2..4c6d29948 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ multipart = { version = "0.18.0", features = ["client"], default-features = fals rpassword = { version = "6.0.1", optional = true } ureq = { version = "2.3.1", features = ["gzip"], default-features = false, optional = true } native-tls-crate = { package = "native-tls", version = "0.2.8", optional = true } +tracing = "0.1.34" [dev-dependencies] indoc = "1.0.3" diff --git a/guide/src/project_layout.md b/guide/src/project_layout.md index d1f17fd98..db1f674eb 100644 --- a/guide/src/project_layout.md +++ b/guide/src/project_layout.md @@ -144,3 +144,17 @@ my-project └── src └── lib.rs ``` + +## Data + +You can add wheel data by creating a `.data` folder or setting its location as `data` in pyproject.toml under `[tool.maturin]` or in Cargo.toml under `[project.metadata.maturin]`. + +The data folder may have the following subfolder: + + * `data`: The contents of this folder will simply be unpacked into the virtualenv + * `scripts`: Treated similar to entry points, files in there are installed as standalone executable + * `headers`: For `.h` C header files + * `purelib`: This also exists, but seems to be barely used + * `platlib`: This also exists, but seems to be barely used + +If you add a symlink in the data directory, we'll include the actual file so you more flexibility diff --git a/src/build_context.rs b/src/build_context.rs index 7cf286850..771b70359 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -2,7 +2,7 @@ use crate::auditwheel::{get_policy_and_libs, patchelf, relpath}; use crate::auditwheel::{PlatformTag, Policy}; use crate::compile::warn_missing_py_init; use crate::module_writer::{ - write_bin, write_bindings_module, write_cffi_module, write_python_part, WheelWriter, + add_data, write_bin, write_bindings_module, write_cffi_module, write_python_part, WheelWriter, }; use crate::python_interpreter::InterpreterKind; use crate::source_distribution::source_distribution; @@ -51,25 +51,19 @@ impl BridgeModel { } } -/// Whether this project is pure rust or rust mixed with python +/// Whether this project is pure rust or rust mixed with python and whether it has wheel data #[derive(Clone, Debug, PartialEq, Eq)] -pub enum ProjectLayout { - /// A rust crate compiled into a shared library with only some glue python for cffi - PureRust { - /// Contains the canonicialized (i.e. absolute) path to the rust part of the project - rust_module: PathBuf, - /// rust extension name - extension_name: String, - }, - /// A python package that is extended by a native rust module. - Mixed { - /// Contains the canonicialized (i.e. absolute) path to the python part of the project - python_module: PathBuf, - /// Contains the canonicialized (i.e. absolute) path to the rust part of the project - rust_module: PathBuf, - /// rust extension name - extension_name: String, - }, +pub struct ProjectLayout { + /// Contains the canonicalized (i.e. absolute) path to the python part of the project + /// If none, we have a rust crate compiled into a shared library with only some glue python for cffi + /// If some, we have a python package that is extended by a native rust module. + pub python_module: Option, + /// Contains the canonicalized (i.e. absolute) path to the rust part of the project + pub rust_module: PathBuf, + /// rust extension name + pub extension_name: String, + /// The location of the wheel data, if any + pub data: Option, } impl ProjectLayout { @@ -78,6 +72,7 @@ impl ProjectLayout { project_root: impl AsRef, module_name: &str, py_src: Option>, + data: Option>, ) -> Result { // A dot in the module name means the extension module goes into the module folder specified by the path let parts: Vec<&str> = module_name.split('.').collect(); @@ -100,6 +95,23 @@ impl ProjectLayout { module_name.to_string(), ) }; + + let data = if let Some(data) = data { + let data = if data.as_ref().is_absolute() { + data.as_ref().to_path_buf() + } else { + project_root.join(data) + }; + if !data.is_dir() { + bail!("No such data directory {}", data.display()); + } + Some(data) + } else if project_root.join(format!("{}.data", module_name)).is_dir() { + Some(project_root.join(format!("{}.data", module_name))) + } else { + None + }; + if python_module.is_dir() { if !python_module.join("__init__.py").is_file() { bail!("Found a directory with the module name ({}) next to Cargo.toml, which indicates a mixed python/rust project, but the directory didn't contain an __init__.py file.", module_name) @@ -107,29 +119,21 @@ impl ProjectLayout { println!("🍹 Building a mixed python/rust project"); - Ok(ProjectLayout::Mixed { - python_module, + Ok(ProjectLayout { + python_module: Some(python_module), rust_module, extension_name, + data, }) } else { - Ok(ProjectLayout::PureRust { + Ok(ProjectLayout { + python_module: None, rust_module: project_root.to_path_buf(), extension_name, + data, }) } } - - pub fn extension_name(&self) -> &str { - match *self { - ProjectLayout::PureRust { - ref extension_name, .. - } => extension_name, - ProjectLayout::Mixed { - ref extension_name, .. - } => extension_name, - } - } } /// Contains all the metadata required to build the crate @@ -246,6 +250,7 @@ impl BuildContext { &self.cargo_metadata, pyproject.sdist_include(), include_cargo_lock, + self.project_layout.data.as_deref(), ) .context("Failed to build source distribution")?; Ok(Some((sdist_path, "source".to_string()))) @@ -396,7 +401,7 @@ impl BuildContext { .context("Failed to add the files to the wheel")?; self.add_pth(&mut writer)?; - + add_data(&mut writer, self.project_layout.data.as_deref())?; let wheel_path = writer.finish()?; Ok((wheel_path, format!("cp{}{}", major, min_minor))) } @@ -415,7 +420,7 @@ impl BuildContext { let python_interpreter = interpreters.get(0); let artifact = self.compile_cdylib( python_interpreter, - Some(self.project_layout.extension_name()), + Some(&self.project_layout.extension_name), )?; let (policy, external_libs) = self.auditwheel(&artifact, self.platform_tag)?; let (wheel_path, tag) = self.write_binding_wheel_abi3( @@ -461,7 +466,7 @@ impl BuildContext { .context("Failed to add the files to the wheel")?; self.add_pth(&mut writer)?; - + add_data(&mut writer, self.project_layout.data.as_deref())?; let wheel_path = writer.finish()?; Ok(( wheel_path, @@ -484,7 +489,7 @@ impl BuildContext { for python_interpreter in interpreters { let artifact = self.compile_cdylib( Some(python_interpreter), - Some(self.project_layout.extension_name()), + Some(&self.project_layout.extension_name), )?; let (policy, external_libs) = self.auditwheel(&artifact, self.platform_tag)?; let (wheel_path, tag) = self.write_binding_wheel( @@ -568,7 +573,7 @@ impl BuildContext { )?; self.add_pth(&mut writer)?; - + add_data(&mut writer, self.project_layout.data.as_deref())?; let wheel_path = writer.finish()?; Ok((wheel_path, "py3".to_string())) } @@ -616,16 +621,11 @@ impl BuildContext { let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &tags)?; - match self.project_layout { - ProjectLayout::Mixed { - ref python_module, .. - } => { - if !self.editable { - write_python_part(&mut writer, python_module) - .context("Failed to add the python module to the package")?; - } + if let Some(python_module) = &self.project_layout.python_module { + if !self.editable { + write_python_part(&mut writer, python_module) + .context("Failed to add the python module to the package")?; } - ProjectLayout::PureRust { .. } => {} } // I wouldn't know of any case where this would be the wrong (and neither do @@ -638,7 +638,7 @@ impl BuildContext { write_bin(&mut writer, artifact, &self.metadata21, bin_name)?; self.add_pth(&mut writer)?; - + add_data(&mut writer, self.project_layout.data.as_deref())?; let wheel_path = writer.finish()?; Ok((wheel_path, "py3".to_string())) } diff --git a/src/build_options.rs b/src/build_options.rs index 5981b418e..2c1531f79 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::env; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; // This is used for BridgeModel::Bindings("pyo3-ffi") and BridgeModel::Bindings("pyo3"). // These should be treated almost identically but must be correctly identified @@ -209,10 +209,14 @@ impl BuildOptions { .filter(|name| name.contains('.')) .unwrap_or(&module_name); + let data = pyproject + .and_then(|x| x.data()) + .or_else(|| extra_metadata.data.as_ref().map(Path::new)); let project_layout = ProjectLayout::determine( manifest_dir, extension_name, extra_metadata.python_source.as_deref(), + data, )?; let mut args_from_pyproject = Vec::new(); diff --git a/src/cargo_toml.rs b/src/cargo_toml.rs index b0d6aa63f..f1949ecba 100644 --- a/src/cargo_toml.rs +++ b/src/cargo_toml.rs @@ -123,7 +123,10 @@ pub struct RemainingCoreMetadata { pub project_url: Option>, pub provides_extra: Option>, pub description_content_type: Option, + /// The directory with python module, contains `/__init__.py` pub python_source: Option, + /// The directory containing the wheel data + pub data: Option, } #[cfg(test)] diff --git a/src/main.rs b/src/main.rs index 11ea210bc..e4a6ec9e4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -256,7 +256,7 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { metadata_directory, strip, } => { - assert!(build_options.interpreter.len() == 1); + assert_eq!(build_options.interpreter.len(), 1); let context = build_options.into_build_context(true, strip, false)?; // Since afaik all other PEP 517 backends also return linux tagged wheels, we do so too diff --git a/src/module_writer.rs b/src/module_writer.rs index 8cc8f3b83..04a21271e 100644 --- a/src/module_writer.rs +++ b/src/module_writer.rs @@ -9,6 +9,7 @@ use flate2::write::GzEncoder; use flate2::Compression; use fs_err as fs; use fs_err::File; +use ignore::WalkBuilder; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::ffi::OsStr; @@ -22,6 +23,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::str; use tempfile::{tempdir, TempDir}; +use tracing::debug; use zip::{self, ZipWriter}; /// Allows writing the module to a wheel or add it directly to the virtualenv @@ -44,7 +46,7 @@ pub trait ModuleWriter { permissions: u32, ) -> Result<()>; - /// Copies the source file the the target path relative to the module base path + /// Copies the source file to the target path relative to the module base path fn add_file(&mut self, target: impl AsRef, source: impl AsRef) -> Result<()> { let read_failed_context = format!("Failed to read {}", source.as_ref().display()); let mut file = File::open(source.as_ref()).context(read_failed_context.clone())?; @@ -273,19 +275,14 @@ impl WheelWriter { project_layout: &ProjectLayout, metadata21: &Metadata21, ) -> Result<()> { - match project_layout { - ProjectLayout::Mixed { - ref python_module, .. - } => { - let absolute_path = python_module.canonicalize()?; - if let Some(python_path) = absolute_path.parent().and_then(|p| p.to_str()) { - let name = metadata21.get_distribution_escaped(); - self.add_bytes(format!("{}.pth", name), python_path.as_bytes())?; - } else { - println!("⚠️ source code path contains non-Unicode sequences, editable installs may not work."); - } + if let Some(python_module) = &project_layout.python_module { + let absolute_path = python_module.canonicalize()?; + if let Some(python_path) = absolute_path.parent().and_then(|p| p.to_str()) { + let name = metadata21.get_distribution_escaped(); + self.add_bytes(format!("{}.pth", name), python_path.as_bytes())?; + } else { + println!("⚠️ source code path contains non-Unicode sequences, editable installs may not work."); } - ProjectLayout::PureRust { .. } => {} } Ok(()) } @@ -586,7 +583,7 @@ fn handle_cffi_call_result( ); } else { // Don't swallow warnings - std::io::stderr().write_all(&output.stderr)?; + io::stderr().write_all(&output.stderr)?; let ffi_py_content = fs::read_to_string(&ffi_py)?; tempdir.close()?; @@ -605,7 +602,7 @@ pub fn write_bindings_module( target: &Target, editable: bool, ) -> Result<()> { - let ext_name = project_layout.extension_name(); + let ext_name = &project_layout.extension_name; let so_filename = match python_interpreter { Some(python_interpreter) => python_interpreter.get_library_name(ext_name), // abi3 @@ -619,63 +616,58 @@ pub fn write_bindings_module( } }; - match project_layout { - ProjectLayout::Mixed { - ref python_module, - ref rust_module, - .. - } => { - if !editable { - write_python_part(writer, python_module) - .context("Failed to add the python module to the package")?; - } + if let Some(python_module) = &project_layout.python_module { + if !editable { + write_python_part(writer, python_module) + .context("Failed to add the python module to the package")?; + } - if editable { - let target = rust_module.join(&so_filename); - // Remove existing so file to avoid triggering SIGSEV in running process - // See https://github.com/PyO3/maturin/issues/758 - let _ = fs::remove_file(&target); - fs::copy(&artifact, &target).context(format!( - "Failed to copy {} to {}", - artifact.display(), - target.display() - ))?; - } + if editable { + let target = project_layout.rust_module.join(&so_filename); + // Remove existing so file to avoid triggering SIGSEV in running process + // See https://github.com/PyO3/maturin/issues/758 + let _ = fs::remove_file(&target); + fs::copy(&artifact, &target).context(format!( + "Failed to copy {} to {}", + artifact.display(), + target.display() + ))?; + } - if !editable { - let relative = rust_module.strip_prefix(python_module.parent().unwrap())?; - writer.add_file_with_permissions(relative.join(&so_filename), &artifact, 0o755)?; - } + if !editable { + let relative = project_layout + .rust_module + .strip_prefix(python_module.parent().unwrap())?; + writer.add_file_with_permissions(relative.join(&so_filename), &artifact, 0o755)?; } - ProjectLayout::PureRust { - ref rust_module, .. - } => { - let module = PathBuf::from(module_name); - writer.add_directory(&module)?; - // Reexport the shared library as if it were the top level module - writer.add_bytes( - &module.join("__init__.py"), - format!( - r#"from .{module_name} import * + } else { + let module = PathBuf::from(module_name); + writer.add_directory(&module)?; + // Reexport the shared library as if it were the top level module + writer.add_bytes( + &module.join("__init__.py"), + format!( + r#"from .{module_name} import * __doc__ = {module_name}.__doc__ if hasattr({module_name}, "__all__"): __all__ = {module_name}.__all__"#, - module_name = module_name - ) - .as_bytes(), + module_name = module_name + ) + .as_bytes(), + )?; + let type_stub = project_layout + .rust_module + .join(format!("{}.pyi", module_name)); + if type_stub.exists() { + writer.add_bytes( + &module.join("__init__.pyi"), + &fs_err::read(type_stub).context("Failed to read type stub file")?, )?; - let type_stub = rust_module.join(format!("{}.pyi", module_name)); - if type_stub.exists() { - writer.add_bytes( - &module.join("__init__.pyi"), - &fs_err::read(type_stub).context("Failed to read type stub file")?, - )?; - writer.add_bytes(&module.join("py.typed"), b"")?; - println!("📖 Found type stub file at {}.pyi", module_name); - } - writer.add_file_with_permissions(&module.join(so_filename), &artifact, 0o755)?; + writer.add_bytes(&module.join("py.typed"), b"")?; + println!("📖 Found type stub file at {}.pyi", module_name); } + writer.add_file_with_permissions(&module.join(so_filename), &artifact, 0o755)?; } Ok(()) @@ -697,55 +689,49 @@ pub fn write_cffi_module( let module; - match project_layout { - ProjectLayout::Mixed { - ref python_module, - ref rust_module, - ref extension_name, - } => { - if !editable { - write_python_part(writer, python_module) - .context("Failed to add the python module to the package")?; - } - - if editable { - let base_path = python_module.join(&module_name); - fs::create_dir_all(&base_path)?; - let target = base_path.join("native.so"); - fs::copy(&artifact, &target).context(format!( - "Failed to copy {} to {}", - artifact.display(), - target.display() - ))?; - File::create(base_path.join("__init__.py"))? - .write_all(cffi_init_file().as_bytes())?; - File::create(base_path.join("ffi.py"))?.write_all(cffi_declarations.as_bytes())?; - } + if let Some(python_module) = &project_layout.python_module { + if !editable { + write_python_part(writer, python_module) + .context("Failed to add the python module to the package")?; + } - let relative = rust_module.strip_prefix(python_module.parent().unwrap())?; - module = relative.join(extension_name); - if !editable { - writer.add_directory(&module)?; - } + if editable { + let base_path = python_module.join(&module_name); + fs::create_dir_all(&base_path)?; + let target = base_path.join("native.so"); + fs::copy(&artifact, &target).context(format!( + "Failed to copy {} to {}", + artifact.display(), + target.display() + ))?; + File::create(base_path.join("__init__.py"))?.write_all(cffi_init_file().as_bytes())?; + File::create(base_path.join("ffi.py"))?.write_all(cffi_declarations.as_bytes())?; } - ProjectLayout::PureRust { - ref rust_module, .. - } => { - module = PathBuf::from(module_name); + + let relative = project_layout + .rust_module + .strip_prefix(python_module.parent().unwrap())?; + module = relative.join(&project_layout.extension_name); + if !editable { writer.add_directory(&module)?; - let type_stub = rust_module.join(format!("{}.pyi", module_name)); - if type_stub.exists() { - writer.add_bytes( - &module.join("__init__.pyi"), - &fs_err::read(type_stub).context("Failed to read type stub file")?, - )?; - writer.add_bytes(&module.join("py.typed"), b"")?; - println!("📖 Found type stub file at {}.pyi", module_name); - } + } + } else { + module = PathBuf::from(module_name); + writer.add_directory(&module)?; + let type_stub = project_layout + .rust_module + .join(format!("{}.pyi", module_name)); + if type_stub.exists() { + writer.add_bytes( + &module.join("__init__.pyi"), + &fs_err::read(type_stub).context("Failed to read type stub file")?, + )?; + writer.add_bytes(&module.join("py.typed"), b"")?; + println!("📖 Found type stub file at {}.pyi", module_name); } }; - if !editable || matches!(project_layout, ProjectLayout::PureRust { .. }) { + if !editable || project_layout.python_module.is_none() { writer.add_bytes(&module.join("__init__.py"), cffi_init_file().as_bytes())?; writer.add_bytes(&module.join("ffi.py"), cffi_declarations.as_bytes())?; writer.add_file_with_permissions(&module.join("native.so"), &artifact, 0o755)?; @@ -780,8 +766,6 @@ pub fn write_python_part( writer: &mut impl ModuleWriter, python_module: impl AsRef, ) -> Result<()> { - use ignore::WalkBuilder; - for absolute in WalkBuilder::new(&python_module).hidden(false).build() { let absolute = absolute?.into_path(); let relative = absolute.strip_prefix(python_module.as_ref().parent().unwrap())?; @@ -850,3 +834,55 @@ pub fn write_dist_info( Ok(()) } + +/// If any, copies the data files from the data directory, resolving symlinks to their source. +/// We resolve symlinks since we require this rather rigid structure while people might need +/// to save or generate the data in other places +/// +/// See https://peps.python.org/pep-0427/#file-contents +pub fn add_data(writer: &mut impl ModuleWriter, data: Option<&Path>) -> Result<()> { + let possible_data_dir_names = ["data", "scripts", "headers", "purelib", "platlib"]; + if let Some(data) = data { + for subdir in fs::read_dir(data).context("Failed to read data dir")? { + let subdir = subdir?; + let dir_name = subdir + .file_name() + .to_str() + .context("Invalid data dir name")? + .to_string(); + if !subdir.path().is_dir() || !possible_data_dir_names.contains(&dir_name.as_str()) { + bail!( + "Invalid data dir entry {}. Possible are directories named {}", + subdir.path().display(), + possible_data_dir_names.join(", ") + ); + } + debug!("Adding data from {}", subdir.path().display()); + (|| { + for file in WalkBuilder::new(subdir.path()) + .standard_filters(false) + .build() + { + let file = file?; + let relative = file.path().strip_prefix(data.parent().unwrap())?; + + if file.path_is_symlink() { + // Copy the actual file contents, not the link, so that you can create a + // data directory by joining different data sources + let source = fs::read_link(file.path())?; + writer.add_file(relative, source.parent().unwrap())?; + } else if file.path().is_file() { + writer.add_file(relative, file.path())?; + } else if file.path().is_dir() { + writer.add_directory(relative)?; + } else { + bail!("Can't handle data dir entry {}", file.path().display()); + } + } + Ok(()) + })() + .with_context(|| format!("Failed to include data from {}", data.display()))? + } + } + Ok(()) +} diff --git a/src/pyproject_toml.rs b/src/pyproject_toml.rs index 3004ddf71..9533344db 100644 --- a/src/pyproject_toml.rs +++ b/src/pyproject_toml.rs @@ -27,6 +27,8 @@ pub struct ToolMaturin { skip_auditwheel: bool, #[serde(default)] strip: bool, + /// Path to the wheel directory, defaults to `.data` + data: Option, } /// A pyproject.toml as specified in PEP 517 @@ -122,6 +124,14 @@ impl PyProjectToml { .unwrap_or_default() } + /// Returns the value of `[tool.maturin.data]` in pyproject.toml + pub fn data(&self) -> Option<&Path> { + self.tool + .as_ref() + .and_then(|tool| tool.maturin.as_ref()) + .and_then(|maturin| maturin.data.as_deref()) + } + /// Returns the value of `[tool.maturin.manifest-path]` in pyproject.toml pub fn manifest_path(&self) -> Option<&Path> { self.tool diff --git a/src/source_distribution.rs b/src/source_distribution.rs index 3958cf96a..9712620e9 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,4 +1,4 @@ -use crate::module_writer::ModuleWriter; +use crate::module_writer::{add_data, ModuleWriter}; use crate::{Metadata21, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::{Metadata, PackageId}; @@ -272,6 +272,7 @@ pub fn source_distribution( cargo_metadata: &Metadata, sdist_include: Option<&Vec>, include_cargo_lock: bool, + data: Option<&Path>, ) -> Result { let known_path_deps = find_path_deps(cargo_metadata)?; @@ -332,6 +333,7 @@ pub fn source_distribution( metadata21.to_file_contents().as_bytes(), )?; + add_data(&mut writer, data)?; let source_distribution_path = writer.finish()?; println!( diff --git a/test-crates/pyo3-pure/pyo3_pure.pyi b/test-crates/pyo3-pure/pyo3_pure.pyi index aad4bb417..a07176815 100644 --- a/test-crates/pyo3-pure/pyo3_pure.pyi +++ b/test-crates/pyo3-pure/pyo3_pure.pyi @@ -1,5 +1,5 @@ class DummyClass: @staticmethod - def get_42(self) -> int: ... + def get_42() -> int: ... fourtytwo: int diff --git a/test-crates/with-data/Cargo.lock b/test-crates/with-data/Cargo.lock new file mode 100644 index 000000000..4c5c30998 --- /dev/null +++ b/test-crates/with-data/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "with-data" +version = "0.1.0" diff --git a/test-crates/with-data/Cargo.toml b/test-crates/with-data/Cargo.toml new file mode 100644 index 000000000..68f41d412 --- /dev/null +++ b/test-crates/with-data/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "with-data" +version = "0.1.0" +authors = ["konstin "] +edition = "2018" + +[lib] +name = "with_data" +crate-type = ["cdylib"] diff --git a/test-crates/with-data/check_installed/check_installed.py b/test-crates/with-data/check_installed/check_installed.py new file mode 100644 index 000000000..23cfd5282 --- /dev/null +++ b/test-crates/with-data/check_installed/check_installed.py @@ -0,0 +1,33 @@ +import locale +import sys +from pathlib import Path + +import with_data + +assert with_data.lib.one() == 1 +assert with_data.ffi.string(with_data.lib.say_hello()).decode() == "hello" + +venv_root = Path(sys.prefix) + +installed_data = ( + venv_root.joinpath("data_subdir") + .joinpath("hello.txt") + # With the default encoding, python under windows fails to read the file correctly :/ + .read_text(encoding="utf-8") + .strip() +) +assert installed_data == "Hi! 😊", ( + installed_data, + "Hi! 😊", + locale.getpreferredencoding(), +) +header_file = ( + venv_root.joinpath("include") + .joinpath("site") + .joinpath(f"python{sys.version_info.major}.{sys.version_info.minor}") + .joinpath("with-data") + .joinpath("empty.h") +) +assert header_file.is_file(), header_file + +print("SUCCESS") diff --git a/test-crates/with-data/pyproject.toml b/test-crates/with-data/pyproject.toml new file mode 100644 index 000000000..e2bcaa270 --- /dev/null +++ b/test-crates/with-data/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["maturin>=0.12,<0.13"] +build-backend = "maturin" + +[project] +name = "with-data" +dependencies = ["cffi"] \ No newline at end of file diff --git a/test-crates/with-data/src/lib.rs b/test-crates/with-data/src/lib.rs new file mode 100644 index 000000000..4763084b2 --- /dev/null +++ b/test-crates/with-data/src/lib.rs @@ -0,0 +1,12 @@ +use std::ffi::CString; +use std::os::raw::{c_char, c_int}; + +#[no_mangle] +pub unsafe extern "C" fn say_hello() -> *const c_char { + CString::new("hello").unwrap().into_raw() +} + +#[no_mangle] +pub unsafe extern "C" fn one() -> c_int { + 1 +} diff --git a/test-crates/with-data/with_data.data/data/data_subdir/hello.txt b/test-crates/with-data/with_data.data/data/data_subdir/hello.txt new file mode 100644 index 000000000..2e80f4446 --- /dev/null +++ b/test-crates/with-data/with_data.data/data/data_subdir/hello.txt @@ -0,0 +1 @@ +Hi! 😊 \ No newline at end of file diff --git a/test-crates/with-data/with_data.data/headers/empty.h b/test-crates/with-data/with_data.data/headers/empty.h new file mode 100644 index 000000000..e69de29bb diff --git a/tests/common/integration.rs b/tests/common/integration.rs index 7f2d575b4..c8d1c64b6 100644 --- a/tests/common/integration.rs +++ b/tests/common/integration.rs @@ -21,7 +21,7 @@ pub fn test_integration( maybe_mock_cargo(); // Pass CARGO_BIN_EXE_maturin for testing purpose - std::env::set_var( + env::set_var( "CARGO_BIN_EXE_cargo-zigbuild", env!("CARGO_BIN_EXE_maturin"), ); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 606de6eb7..7c08baea6 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -53,7 +53,7 @@ pub fn check_installed(package: &Path, python: &Path) -> Result<()> { let message = str::from_utf8(&output.stdout).unwrap().trim(); if message != "SUCCESS" { - panic!("{}", message); + panic!("Not SUCCESS: {}", message); } Ok(()) diff --git a/tests/run.rs b/tests/run.rs index 2fb8bdfb7..5ab72f748 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -201,6 +201,15 @@ fn integration_pyo3_ffi_pure() { false, )); } +#[test] +fn integration_with_data() { + handle_result(integration::test_integration( + "test-crates/with-data", + None, + "integration_with_data", + false, + )); +} #[test] fn abi3_without_version() {