Merge pull request #906 from PyO3/wheel-data

Add wheel data support

.github/workflows/lint.yml

@@ -48,7 +48,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - uses: psf/black@22.1.0
+      - uses: psf/black@22.3.0
 
   spellcheck:
     runs-on: ubuntu-latest

Cargo.toml

@@ -66,6 +66,7 @@ multipart = { version = "0.18.0", features = ["client"], default-features = fals
 rpassword = { version = "6.0.1", optional = true }
 ureq = { version = "2.3.1", features = ["gzip"], default-features = false, optional = true }
 native-tls-crate = { package = "native-tls", version = "0.2.8", optional = true }
+tracing = "0.1.34"
 
 [dev-dependencies]
 indoc = "1.0.3"

guide/src/project_layout.md

@@ -144,3 +144,17 @@ my-project
 └── src
     └── lib.rs
 ```
+
+## Data
+
+You can add wheel data by creating a `<module_name>.data` folder or setting its location as `data` in pyproject.toml under `[tool.maturin]` or in Cargo.toml under `[project.metadata.maturin]`.
+
+The data folder may have the following subfolder:
+
+ * `data`: The contents of this folder will simply be unpacked into the virtualenv
+ * `scripts`: Treated similar to entry points, files in there are installed as standalone executable
+ * `headers`: For `.h` C header files
+ * `purelib`: This also exists, but seems to be barely used
+ * `platlib`: This also exists, but seems to be barely used
+
+If you add a symlink in the data directory, we'll include the actual file so you more flexibility 

src/build_context.rs

@@ -2,7 +2,7 @@ use crate::auditwheel::{get_policy_and_libs, patchelf, relpath};
 use crate::auditwheel::{PlatformTag, Policy};
 use crate::compile::warn_missing_py_init;
 use crate::module_writer::{
-    write_bin, write_bindings_module, write_cffi_module, write_python_part, WheelWriter,
+    add_data, write_bin, write_bindings_module, write_cffi_module, write_python_part, WheelWriter,
 };
 use crate::python_interpreter::InterpreterKind;
 use crate::source_distribution::source_distribution;
@@ -51,25 +51,19 @@ impl BridgeModel {
     }
 }
 
-/// Whether this project is pure rust or rust mixed with python
+/// Whether this project is pure rust or rust mixed with python and whether it has wheel data
 #[derive(Clone, Debug, PartialEq, Eq)]
-pub enum ProjectLayout {
-    /// A rust crate compiled into a shared library with only some glue python for cffi
-    PureRust {
-        /// Contains the canonicialized (i.e. absolute) path to the rust part of the project
-        rust_module: PathBuf,
-        /// rust extension name
-        extension_name: String,
-    },
-    /// A python package that is extended by a native rust module.
-    Mixed {
-        /// Contains the canonicialized (i.e. absolute) path to the python part of the project
-        python_module: PathBuf,
-        /// Contains the canonicialized (i.e. absolute) path to the rust part of the project
-        rust_module: PathBuf,
-        /// rust extension name
-        extension_name: String,
-    },
+pub struct ProjectLayout {
+    /// Contains the canonicalized (i.e. absolute) path to the python part of the project
+    /// If none, we have a rust crate compiled into a shared library with only some glue python for cffi
+    /// If some, we have a python package that is extended by a native rust module.
+    pub python_module: Option<PathBuf>,
+    /// Contains the canonicalized (i.e. absolute) path to the rust part of the project
+    pub rust_module: PathBuf,
+    /// rust extension name
+    pub extension_name: String,
+    /// The location of the wheel data, if any
+    pub data: Option<PathBuf>,
 }
 
 impl ProjectLayout {
@@ -78,6 +72,7 @@ impl ProjectLayout {
         project_root: impl AsRef<Path>,
         module_name: &str,
         py_src: Option<impl AsRef<Path>>,
+        data: Option<impl AsRef<Path>>,
     ) -> Result<ProjectLayout> {
         // A dot in the module name means the extension module goes into the module folder specified by the path
         let parts: Vec<&str> = module_name.split('.').collect();
@@ -100,36 +95,45 @@ impl ProjectLayout {
                 module_name.to_string(),
             )
         };
+
+        let data = if let Some(data) = data {
+            let data = if data.as_ref().is_absolute() {
+                data.as_ref().to_path_buf()
+            } else {
+                project_root.join(data)
+            };
+            if !data.is_dir() {
+                bail!("No such data directory {}", data.display());
+            }
+            Some(data)
+        } else if project_root.join(format!("{}.data", module_name)).is_dir() {
+            Some(project_root.join(format!("{}.data", module_name)))
+        } else {
+            None
+        };
+
         if python_module.is_dir() {
             if !python_module.join("__init__.py").is_file() {
                 bail!("Found a directory with the module name ({}) next to Cargo.toml, which indicates a mixed python/rust project, but the directory didn't contain an __init__.py file.", module_name)
             }
 
             println!("🍹 Building a mixed python/rust project");
 
-            Ok(ProjectLayout::Mixed {
-                python_module,
+            Ok(ProjectLayout {
+                python_module: Some(python_module),
                 rust_module,
                 extension_name,
+                data,
             })
         } else {
-            Ok(ProjectLayout::PureRust {
+            Ok(ProjectLayout {
+                python_module: None,
                 rust_module: project_root.to_path_buf(),
                 extension_name,
+                data,
             })
         }
     }
-
-    pub fn extension_name(&self) -> &str {
-        match *self {
-            ProjectLayout::PureRust {
-                ref extension_name, ..
-            } => extension_name,
-            ProjectLayout::Mixed {
-                ref extension_name, ..
-            } => extension_name,
-        }
-    }
 }
 
 /// Contains all the metadata required to build the crate
@@ -246,6 +250,7 @@ impl BuildContext {
                     &self.cargo_metadata,
                     pyproject.sdist_include(),
                     include_cargo_lock,
+                    self.project_layout.data.as_deref(),
                 )
                 .context("Failed to build source distribution")?;
                 Ok(Some((sdist_path, "source".to_string())))
@@ -396,7 +401,7 @@ impl BuildContext {
         .context("Failed to add the files to the wheel")?;
 
         self.add_pth(&mut writer)?;
-
+        add_data(&mut writer, self.project_layout.data.as_deref())?;
         let wheel_path = writer.finish()?;
         Ok((wheel_path, format!("cp{}{}", major, min_minor)))
     }
@@ -415,7 +420,7 @@ impl BuildContext {
         let python_interpreter = interpreters.get(0);
         let artifact = self.compile_cdylib(
             python_interpreter,
-            Some(self.project_layout.extension_name()),
+            Some(&self.project_layout.extension_name),
         )?;
         let (policy, external_libs) = self.auditwheel(&artifact, self.platform_tag)?;
         let (wheel_path, tag) = self.write_binding_wheel_abi3(
@@ -461,7 +466,7 @@ impl BuildContext {
         .context("Failed to add the files to the wheel")?;
 
         self.add_pth(&mut writer)?;
-
+        add_data(&mut writer, self.project_layout.data.as_deref())?;
         let wheel_path = writer.finish()?;
         Ok((
             wheel_path,
@@ -484,7 +489,7 @@ impl BuildContext {
         for python_interpreter in interpreters {
             let artifact = self.compile_cdylib(
                 Some(python_interpreter),
-                Some(self.project_layout.extension_name()),
+                Some(&self.project_layout.extension_name),
             )?;
             let (policy, external_libs) = self.auditwheel(&artifact, self.platform_tag)?;
             let (wheel_path, tag) = self.write_binding_wheel(
@@ -568,7 +573,7 @@ impl BuildContext {
         )?;
 
         self.add_pth(&mut writer)?;
-
+        add_data(&mut writer, self.project_layout.data.as_deref())?;
         let wheel_path = writer.finish()?;
         Ok((wheel_path, "py3".to_string()))
     }
@@ -616,16 +621,11 @@ impl BuildContext {
 
         let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &tags)?;
 
-        match self.project_layout {
-            ProjectLayout::Mixed {
-                ref python_module, ..
-            } => {
-                if !self.editable {
-                    write_python_part(&mut writer, python_module)
-                        .context("Failed to add the python module to the package")?;
-                }
+        if let Some(python_module) = &self.project_layout.python_module {
+            if !self.editable {
+                write_python_part(&mut writer, python_module)
+                    .context("Failed to add the python module to the package")?;
             }
-            ProjectLayout::PureRust { .. } => {}
         }
 
         // I wouldn't know of any case where this would be the wrong (and neither do
@@ -638,7 +638,7 @@ impl BuildContext {
         write_bin(&mut writer, artifact, &self.metadata21, bin_name)?;
 
         self.add_pth(&mut writer)?;
-
+        add_data(&mut writer, self.project_layout.data.as_deref())?;
         let wheel_path = writer.finish()?;
         Ok((wheel_path, "py3".to_string()))
     }

src/build_options.rs

@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::env;
 use std::io;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 // This is used for BridgeModel::Bindings("pyo3-ffi") and BridgeModel::Bindings("pyo3").
 // These should be treated almost identically but must be correctly identified
@@ -209,10 +209,14 @@ impl BuildOptions {
             .filter(|name| name.contains('.'))
             .unwrap_or(&module_name);
 
+        let data = pyproject
+            .and_then(|x| x.data())
+            .or_else(|| extra_metadata.data.as_ref().map(Path::new));
         let project_layout = ProjectLayout::determine(
             manifest_dir,
             extension_name,
             extra_metadata.python_source.as_deref(),
+            data,
         )?;
 
         let mut args_from_pyproject = Vec::new();

src/cargo_toml.rs

@@ -123,7 +123,10 @@ pub struct RemainingCoreMetadata {
     pub project_url: Option<HashMap<String, String>>,
     pub provides_extra: Option<Vec<String>>,
     pub description_content_type: Option<String>,
+    /// The directory with python module, contains `<module_name>/__init__.py`
     pub python_source: Option<String>,
+    /// The directory containing the wheel data
+    pub data: Option<String>,
 }
 
 #[cfg(test)]

src/main.rs

@@ -256,7 +256,7 @@ fn pep517(subcommand: Pep517Command) -> Result<()> {
             metadata_directory,
             strip,
         } => {
-            assert!(build_options.interpreter.len() == 1);
+            assert_eq!(build_options.interpreter.len(), 1);
             let context = build_options.into_build_context(true, strip, false)?;
 
             // Since afaik all other PEP 517 backends also return linux tagged wheels, we do so too