From 15f804507590c610aebe150780a9e58cf8189a01 Mon Sep 17 00:00:00 2001 From: Zameer Manji Date: Mon, 5 Dec 2022 14:52:19 +0000 Subject: [PATCH] Support SOURCE_DATE_EPOCH environment variable in wheel building This adds support for the `SOURCE_DATE_EPOCH` environment variable like wheel does when building wheels. If this environment variable is set, the `mtime` of the files in the wheel are set to this value to ensure reproducible output. Fixes #1320 --- Cargo.lock | 1 + Cargo.toml | 1 + Changelog.md | 1 + src/module_writer.rs | 37 ++++++++++++++++++++++++++++++++++--- tests/common/other.rs | 34 +++++++++++++++++++++++++++++----- tests/run.rs | 12 ++++++++++++ 6 files changed, 78 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 539cc9e79..190a7328c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1510,6 +1510,7 @@ dependencies = [ "tempfile", "textwrap", "thiserror", + "time", "toml_edit", "tracing", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index 44564f7d2..a199c2fad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ rpassword = { version = "7.0.0", optional = true } ureq = { version = "2.3.1", features = ["gzip", "socks-proxy"], default-features = false, optional = true } native-tls-crate = { package = "native-tls", version = "0.2.8", optional = true } keyring = { version = "1.1.1", optional = true } +time = "0.3.17" [dev-dependencies] indoc = "1.0.3" diff --git a/Changelog.md b/Changelog.md index e39a39789..952c3a2ab 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] * **Breaking Change**: Build with `--no-default-features` by default when bootstrapping from sdist in [#1333](https://github.com/PyO3/maturin/pull/1333) +* Support `SOURCE_DATE_EPOCH` when building wheels in [#1334](https://github.com/PyO3/maturin/pull/1334) ## [0.14.4] - 2022-12-05 diff --git a/src/module_writer.rs b/src/module_writer.rs index c890dbd3e..ecdabfc48 100644 --- a/src/module_writer.rs +++ b/src/module_writer.rs @@ -14,6 +14,7 @@ use ignore::WalkBuilder; use normpath::PathExt as _; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; +use std::env; use std::ffi::OsStr; use std::fmt::Write as _; #[cfg(target_family = "unix")] @@ -26,8 +27,10 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::str; use tempfile::{tempdir, TempDir}; +use time::macros::datetime; +use time::OffsetDateTime; use tracing::debug; -use zip::{self, ZipWriter}; +use zip::{self, DateTime, ZipWriter}; /// Allows writing the module to a wheel or add it directly to the virtualenv pub trait ModuleWriter { @@ -234,9 +237,15 @@ impl ModuleWriter for WheelWriter { } else { zip::CompressionMethod::Deflated }; - let options = zip::write::FileOptions::default() + + let mut options = zip::write::FileOptions::default() .unix_permissions(permissions) .compression_method(compression_method); + let mtime = self.mtime().ok(); + if let Some(mtime) = mtime { + options = options.last_modified_time(mtime); + } + self.zip.start_file(target.clone(), options)?; self.zip.write_all(bytes)?; @@ -309,6 +318,22 @@ impl WheelWriter { } } + /// Returns a DateTime representing the value SOURCE_DATE_EPOCH environment variable + /// Note that the earliest timestamp a zip file can represent is 1980-01-01 + fn mtime(&self) -> Result { + let epoch: i64 = env::var("SOURCE_DATE_EPOCH")?.parse()?; + let dt = OffsetDateTime::from_unix_timestamp(epoch)?; + let min_dt = datetime!(1980-01-01 0:00 UTC); + let dt = dt.max(min_dt); + + let dt = DateTime::from_time(dt); + + match dt { + Ok(dt) => Ok(dt), + Err(_) => Err(anyhow!("failed to build zip DateTime")), + } + } + /// Creates the record file and finishes the zip pub fn finish(mut self) -> Result { let compression_method = if cfg!(feature = "faster-tests") { @@ -316,7 +341,13 @@ impl WheelWriter { } else { zip::CompressionMethod::Deflated }; - let options = zip::write::FileOptions::default().compression_method(compression_method); + + let mut options = zip::write::FileOptions::default().compression_method(compression_method); + let mtime = self.mtime().ok(); + if let Some(mtime) = mtime { + options = options.last_modified_time(mtime); + } + let record_filename = self.record_file.to_str().unwrap().replace('\\', "/"); debug!("Adding {}", record_filename); self.zip.start_file(&record_filename, options)?; diff --git a/tests/common/other.rs b/tests/common/other.rs index 9ec5594af..d3d23cd56 100644 --- a/tests/common/other.rs +++ b/tests/common/other.rs @@ -9,6 +9,7 @@ use std::io::Read; use std::iter::FromIterator; use std::path::{Path, PathBuf}; use tar::Archive; +use time::OffsetDateTime; use zip::ZipArchive; /// Tries to compile a sample crate (pyo3-pure) for musl, @@ -171,11 +172,7 @@ pub fn test_source_distribution( Ok(()) } -pub fn check_wheel_files( - package: impl AsRef, - expected_files: Vec<&str>, - unique_name: &str, -) -> Result<()> { +fn build_wheel_files(package: impl AsRef, unique_name: &str) -> Result> { let manifest_path = package.as_ref().join("Cargo.toml"); let wheel_directory = Path::new("test-crates").join("wheels").join(unique_name); @@ -202,6 +199,33 @@ pub fn check_wheel_files( let (wheel_path, _) = &wheels[0]; let wheel = ZipArchive::new(File::open(wheel_path)?)?; + Ok(wheel) +} + +pub fn check_wheel_mtimes( + package: impl AsRef, + expected_mtime: Vec, + unique_name: &str, +) -> Result<()> { + let mut wheel = build_wheel_files(package, unique_name)?; + let mut mtimes = BTreeSet::::new(); + + for idx in 0..wheel.len() { + let mtime = wheel.by_index(idx)?.last_modified().to_time()?; + mtimes.insert(mtime); + } + + assert_eq!(mtimes, expected_mtime.into_iter().collect::>()); + + Ok(()) +} + +pub fn check_wheel_files( + package: impl AsRef, + expected_files: Vec<&str>, + unique_name: &str, +) -> Result<()> { + let wheel = build_wheel_files(package, unique_name)?; let drop_platform_specific_files = |file: &&str| -> bool { !matches!(Path::new(file).extension(), Some(ext) if ext == "pyc" || ext == "pyd" || ext == "so") }; diff --git a/tests/run.rs b/tests/run.rs index 892ef2c10..7364c92d2 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -5,7 +5,9 @@ use common::{ }; use indoc::indoc; use maturin::Target; +use std::env; use std::path::{Path, PathBuf}; +use time::macros::datetime; mod common; @@ -576,3 +578,13 @@ fn workspace_inheritance_sdist() { fn abi3_python_interpreter_args() { handle_result(other::abi3_python_interpreter_args()); } + +#[test] +fn pyo3_source_date_epoch() { + env::set_var("SOURCE_DATE_EPOCH", "0"); + handle_result(other::check_wheel_mtimes( + "test-crates/pyo3-mixed-include-exclude", + vec![datetime!(1980-01-01 0:00 UTC)], + "pyo3_source_date_epoch", + )) +}