Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix zip memory warnings #217

Merged
merged 3 commits into from
Dec 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 40 additions & 18 deletions src/commands.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use crate::{
progress::Progress,
utils::{
self, concatenate_os_str_list, dir_is_empty, nice_directory_display, to_utf, try_infer_extension,
user_wants_to_continue_decompressing,
user_wants_to_continue_compressing, user_wants_to_continue_decompressing,
},
warning, Opts, QuestionPolicy, Subcommand,
};
Expand Down Expand Up @@ -157,7 +157,7 @@ pub fn run(args: Opts, question_policy: QuestionPolicy) -> crate::Result<()> {
formats = new_formats;
}
}
let compress_result = compress_files(files, formats, output_file);
let compress_result = compress_files(files, formats, output_file, &output_path, question_policy);

// If any error occurred, delete incomplete file
if compress_result.is_err() {
Expand Down Expand Up @@ -268,7 +268,7 @@ pub fn run(args: Opts, question_policy: QuestionPolicy) -> crate::Result<()> {
println!();
}
let formats = formats.iter().flat_map(Extension::iter).map(Clone::clone).collect();
list_archive_contents(archive_path, formats, list_options)?;
list_archive_contents(archive_path, formats, list_options, question_policy)?;
}
}
}
Expand All @@ -280,7 +280,13 @@ pub fn run(args: Opts, question_policy: QuestionPolicy) -> crate::Result<()> {
// files are the list of paths to be compressed: ["dir/file1.txt", "dir/file2.txt"]
// formats contains each format necessary for compression, example: [Tar, Gz] (in compression order)
// output_file is the resulting compressed file name, example: "compressed.tar.gz"
fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs::File) -> crate::Result<()> {
fn compress_files(
files: Vec<PathBuf>,
formats: Vec<Extension>,
output_file: fs::File,
output_dir: &Path,
question_policy: QuestionPolicy,
) -> crate::Result<()> {
// The next lines are for displaying the progress bar
// If the input files contain a directory, then the total size will be underestimated
let (total_input_size, precise) = files
Expand Down Expand Up @@ -347,14 +353,17 @@ fn compress_files(files: Vec<PathBuf>, formats: Vec<Extension>, output_file: fs:
writer.flush()?;
}
Zip => {
eprintln!("{yellow}Warning:{reset}", yellow = *colors::YELLOW, reset = *colors::RESET);
eprintln!("\tCompressing .zip entirely in memory.");
eprintln!("\tIf the file is too big, your PC might freeze!");
eprintln!("{orange}[WARNING]{reset}", orange = *colors::ORANGE, reset = *colors::RESET);
eprintln!(
"\tThis is a limitation for formats like '{}'.",
formats.iter().map(|format| format.to_string()).collect::<String>()
"\tThere is a limitation for .zip archives with extra extensions. (e.g. <file>.zip.gz)\
\n\tThe design of .zip makes it impossible to compress via stream, so it must be done entirely in memory.\
\n\tBy compressing .zip with extra compression formats, you can run out of RAM if the file is too large!"
);
eprintln!("\tThe design of .zip makes it impossible to compress via stream.");

// give user the option to continue compressing after warning is shown
if !user_wants_to_continue_compressing(output_dir, question_policy)? {
return Ok(());
}

let mut vec_buffer = io::Cursor::new(vec![]);

Expand Down Expand Up @@ -505,12 +514,17 @@ fn decompress_file(
};
}
Zip => {
eprintln!("Compressing first into .zip.");
eprintln!("Warning: .zip archives with extra extensions have a downside.");
eprintln!("{orange}[WARNING]{reset}", orange = *colors::ORANGE, reset = *colors::RESET);
eprintln!(
"The only way is loading everything into the RAM while compressing, and then write everything down."
"\tThere is a limitation for .zip archives with extra extensions. (e.g. <file>.zip.gz)\
\n\tThe design of .zip makes it impossible to compress via stream, so it must be done entirely in memory.\
\n\tBy compressing .zip with extra compression formats, you can run out of RAM if the file is too large!"
);
eprintln!("this means that by compressing .zip with extra compression formats, you can run out of RAM if the file is too large!");

// give user the option to continue decompressing after warning is shown
if !user_wants_to_continue_decompressing(input_file_path, question_policy)? {
return Ok(());
}

let mut vec = vec![];
io::copy(&mut reader, &mut vec)?;
Expand Down Expand Up @@ -552,6 +566,7 @@ fn list_archive_contents(
archive_path: &Path,
formats: Vec<CompressionFormat>,
list_options: ListOptions,
question_policy: QuestionPolicy,
) -> crate::Result<()> {
let reader = fs::File::open(&archive_path)?;

Expand Down Expand Up @@ -593,10 +608,17 @@ fn list_archive_contents(
let files = match formats[0] {
Tar => crate::archive::tar::list_archive(reader)?,
Zip => {
eprintln!("Listing files from zip archive.");
eprintln!("Warning: .zip archives with extra extensions have a downside.");
eprintln!("The only way is loading everything into the RAM while compressing, and then reading the archive contents.");
eprintln!("this means that by compressing .zip with extra compression formats, you can run out of RAM if the file is too large!");
eprintln!("{orange}[WARNING]{reset}", orange = *colors::ORANGE, reset = *colors::RESET);
eprintln!(
"\tThere is a limitation for .zip archives with extra extensions. (e.g. <file>.zip.gz)\
\n\tThe design of .zip makes it impossible to compress via stream, so it must be done entirely in memory.\
\n\tBy compressing .zip with extra compression formats, you can run out of RAM if the file is too large!"
);

// give user the option to continue decompressing after warning is shown
if !user_wants_to_continue_decompressing(archive_path, question_policy)? {
return Ok(());
}

let mut vec = vec![];
io::copy(&mut reader, &mut vec)?;
Expand Down
3 changes: 2 additions & 1 deletion src/utils/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ mod question;
pub use formatting::{concatenate_os_str_list, nice_directory_display, strip_cur_dir, to_utf, Bytes};
pub use fs::{cd_into_same_dir_as, clear_path, create_dir_if_non_existent, dir_is_empty, try_infer_extension};
pub use question::{
create_or_ask_overwrite, user_wants_to_continue_decompressing, user_wants_to_overwrite, QuestionPolicy,
create_or_ask_overwrite, user_wants_to_continue_compressing, user_wants_to_continue_decompressing,
user_wants_to_overwrite, QuestionPolicy,
};
pub use utf8::{get_invalid_utf8_paths, is_invalid_utf8};

Expand Down
14 changes: 14 additions & 0 deletions src/utils/question.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,20 @@ pub fn create_or_ask_overwrite(path: &Path, question_policy: QuestionPolicy) ->
}
}

/// Check if QuestionPolicy flags were set, otherwise, ask the user if they want to continue compressing.
pub fn user_wants_to_continue_compressing(path: &Path, question_policy: QuestionPolicy) -> crate::Result<bool> {
match question_policy {
QuestionPolicy::AlwaysYes => Ok(true),
QuestionPolicy::AlwaysNo => Ok(false),
QuestionPolicy::Ask => {
let path = to_utf(strip_cur_dir(path));
let path = Some(path.as_str());
let placeholder = Some("FILE");
Confirmation::new("Do you want to continue compressing 'FILE'?", placeholder).ask(path)
}
}
}

/// Check if QuestionPolicy flags were set, otherwise, ask the user if they want to continue decompressing.
pub fn user_wants_to_continue_decompressing(path: &Path, question_policy: QuestionPolicy) -> crate::Result<bool> {
match question_policy {
Expand Down