Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cp: Implement --sparse flag #3766

Merged
merged 3 commits into from
Aug 4, 2022
Merged

cp: Implement --sparse flag #3766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
33eb68d
cp: Refactor `reflink`/`sparse` handling to enable `--sparse` flag
pimzero Jul 31, 2022
b8fc067
cp: Implement `--sparse` flag
pimzero Jul 31, 2022
c1eb226
Merge branch 'main' into 3362-sparse-clean
sylvestre Aug 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 131 additions & 32 deletions src/uu/cp/src/cp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
// For the full copyright and license information, please view the LICENSE file
// that was distributed with this source code.

// spell-checker:ignore (ToDO) ficlone linkgs lstat nlink nlinks pathbuf reflink strs xattrs symlinked
// spell-checker:ignore (ToDO) ficlone ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked

#[macro_use]
extern crate quick_error;
Expand Down Expand Up @@ -165,6 +165,14 @@ pub enum ReflinkMode {
Never,
}

/// Possible arguments for `--sparse`.
#[derive(Copy, Clone, Eq, PartialEq)]
pub enum SparseMode {
Always,
Auto,
Never,
}

/// Specifies the expected file type of copy target
pub enum TargetType {
Directory,
Expand All @@ -174,7 +182,6 @@ pub enum TargetType {
pub enum CopyMode {
Link,
SymLink,
Sparse,
Copy,
Update,
AttrOnly,
Expand Down Expand Up @@ -206,6 +213,7 @@ pub struct Options {
one_file_system: bool,
overwrite: OverwriteMode,
parents: bool,
sparse_mode: SparseMode,
strip_trailing_slashes: bool,
reflink_mode: ReflinkMode,
preserve_attributes: Vec<Attribute>,
Expand Down Expand Up @@ -439,17 +447,18 @@ pub fn uu_app<'a>() -> Command<'a> {
.short('x')
.long(options::ONE_FILE_SYSTEM)
.help("stay on this file system"))
.arg(Arg::new(options::SPARSE)
.long(options::SPARSE)
.takes_value(true)
.value_name("WHEN")
.possible_values(["never", "auto", "always"])
.help("NotImplemented: control creation of sparse files. See below"))

// TODO: implement the following args
.arg(Arg::new(options::COPY_CONTENTS)
.long(options::COPY_CONTENTS)
.overrides_with(options::ATTRIBUTES_ONLY)
.help("NotImplemented: copy contents of special files when recursive"))
.arg(Arg::new(options::SPARSE)
.long(options::SPARSE)
.takes_value(true)
.value_name("WHEN")
.help("NotImplemented: control creation of sparse files. See below"))
.arg(Arg::new(options::CONTEXT)
.long(options::CONTEXT)
.takes_value(true)
Expand Down Expand Up @@ -545,8 +554,6 @@ impl CopyMode {
Self::Link
} else if matches.contains_id(options::SYMBOLIC_LINK) {
Self::SymLink
} else if matches.contains_id(options::SPARSE) {
Self::Sparse
} else if matches.contains_id(options::UPDATE) {
Self::Update
} else if matches.contains_id(options::ATTRIBUTES_ONLY) {
Expand Down Expand Up @@ -601,7 +608,6 @@ impl Options {
fn from_matches(matches: &ArgMatches) -> CopyResult<Self> {
let not_implemented_opts = vec![
options::COPY_CONTENTS,
options::SPARSE,
#[cfg(not(any(windows, unix)))]
options::ONE_FILE_SYSTEM,
options::CONTEXT,
Expand Down Expand Up @@ -710,6 +716,18 @@ impl Options {
}
}
},
sparse_mode: match matches.value_of(options::SPARSE) {
Some("always") => SparseMode::Always,
Some("auto") => SparseMode::Auto,
Some("never") => SparseMode::Never,
Some(val) => {
return Err(Error::InvalidArgument(format!(
"invalid argument {} for \'sparse\'",
val
)));
}
None => SparseMode::Auto,
},
backup: backup_mode,
backup_suffix,
overwrite,
Expand Down Expand Up @@ -1376,7 +1394,6 @@ fn copy_file(
CopyMode::SymLink => {
symlink_file(&source, &dest, context, symlinked_files)?;
}
CopyMode::Sparse => return Err(Error::NotImplemented(options::SPARSE.to_string())),
CopyMode::Update => {
if dest.exists() {
let src_metadata = fs::symlink_metadata(&source)?;
Expand Down Expand Up @@ -1461,18 +1478,33 @@ fn copy_helper(
copy_fifo(dest, options.overwrite)?;
} else if source_is_symlink {
copy_link(source, dest, symlinked_files)?;
} else if options.reflink_mode != ReflinkMode::Never {
#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))]
return Err("--reflink is only supported on linux and macOS"
.to_string()
.into());

} else {
#[cfg(target_os = "macos")]
copy_on_write_macos(source, dest, options.reflink_mode, context)?;
copy_on_write_macos(
source,
dest,
options.reflink_mode,
options.sparse_mode,
context,
)?;

#[cfg(any(target_os = "linux", target_os = "android"))]
copy_on_write_linux(source, dest, options.reflink_mode, context)?;
} else {
fs::copy(source, dest).context(context)?;
copy_on_write_linux(
source,
dest,
options.reflink_mode,
options.sparse_mode,
context,
)?;

#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))]
copy_no_cow_fallback(
source,
dest,
options.reflink_mode,
options.sparse_mode,
context,
)?;
}

Ok(())
Expand Down Expand Up @@ -1522,25 +1554,50 @@ fn copy_link(
symlink_file(&link, &dest, &context_for(&link, &dest), symlinked_files)
}

/// Copies `source` to `dest` for systems without copy-on-write
#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))]
fn copy_no_cow_fallback(
source: &Path,
dest: &Path,
reflink_mode: ReflinkMode,
sparse_mode: SparseMode,
context: &str,
) -> CopyResult<()> {
if reflink_mode != ReflinkMode::Never {
return Err("--reflink is only supported on linux and macOS"
.to_string()
.into());
}
if sparse_mode != SparseMode::Auto {
return Err("--sparse is only supported on linux".to_string().into());
}

fs::copy(source, dest).context(context)?;

Ok(())
}

/// Copies `source` to `dest` using copy-on-write if possible.
#[cfg(any(target_os = "linux", target_os = "android"))]
fn copy_on_write_linux(
source: &Path,
dest: &Path,
mode: ReflinkMode,
reflink_mode: ReflinkMode,
sparse_mode: SparseMode,
context: &str,
) -> CopyResult<()> {
debug_assert!(mode != ReflinkMode::Never);
use std::os::unix::prelude::MetadataExt;

let src_file = File::open(source).context(context)?;
let mut src_file = File::open(source).context(context)?;
let dst_file = OpenOptions::new()
.write(true)
.truncate(true)
.create(true)
.open(dest)
.context(context)?;
match mode {
ReflinkMode::Always => unsafe {

match (reflink_mode, sparse_mode) {
(ReflinkMode::Always, SparseMode::Auto) => unsafe {
let result = libc::ioctl(dst_file.as_raw_fd(), FICLONE!(), src_file.as_raw_fd());

if result != 0 {
Expand All @@ -1555,15 +1612,54 @@ fn copy_on_write_linux(
Ok(())
}
},
ReflinkMode::Auto => unsafe {
(ReflinkMode::Always, SparseMode::Always) | (ReflinkMode::Always, SparseMode::Never) => {
Err("`--reflink=always` can be used only with --sparse=auto".into())
}
(_, SparseMode::Always) => unsafe {
let size: usize = src_file.metadata()?.size().try_into().unwrap();
if libc::ftruncate(dst_file.as_raw_fd(), size.try_into().unwrap()) < 0 {
return Err(format!(
"failed to ftruncate {:?} to size {}: {}",
dest,
size,
std::io::Error::last_os_error()
)
.into());
}

let blksize = dst_file.metadata()?.blksize();
let mut buf: Vec<u8> = vec![0; blksize.try_into().unwrap()];
let mut current_offset: usize = 0;

while current_offset < size {
use std::io::Read;

let this_read = src_file.read(&mut buf)?;

if buf.iter().any(|&x| x != 0) {
libc::pwrite(
dst_file.as_raw_fd(),
buf.as_ptr() as *const libc::c_void,
this_read,
current_offset.try_into().unwrap(),
);
}
current_offset += this_read;
}
Ok(())
},
(ReflinkMode::Auto, SparseMode::Auto) | (ReflinkMode::Auto, SparseMode::Never) => unsafe {
let result = libc::ioctl(dst_file.as_raw_fd(), FICLONE!(), src_file.as_raw_fd());

if result != 0 {
fs::copy(source, dest).context(context)?;
}
Ok(())
},
ReflinkMode::Never => unreachable!(),
(ReflinkMode::Never, _) => {
fs::copy(source, dest).context(context)?;
Ok(())
}
}
}

Expand All @@ -1572,10 +1668,13 @@ fn copy_on_write_linux(
fn copy_on_write_macos(
source: &Path,
dest: &Path,
mode: ReflinkMode,
reflink_mode: ReflinkMode,
sparse_mode: SparseMode,
context: &str,
) -> CopyResult<()> {
debug_assert!(mode != ReflinkMode::Never);
if sparse_mode != SparseMode::Auto {
return Err("--sparse is only supported on linux".to_string().into());
}

// Extract paths in a form suitable to be passed to a syscall.
// The unwrap() is safe because they come from the command-line and so contain non nul
Expand Down Expand Up @@ -1612,14 +1711,14 @@ fn copy_on_write_macos(
if raw_pfn.is_null() || error != 0 {
// clonefile(2) is either not supported or it errored out (possibly because the FS does not
// support COW).
match mode {
match reflink_mode {
ReflinkMode::Always => {
return Err(
format!("failed to clone {:?} from {:?}: {}", source, dest, error).into(),
)
}
ReflinkMode::Auto => fs::copy(source, dest).context(context)?,
ReflinkMode::Never => unreachable!(),
ReflinkMode::Never => fs::copy(source, dest).context(context)?,
};
}

Expand Down
Loading