Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove tokio-tar. #7631

Merged
merged 4 commits into from
Aug 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 5 additions & 39 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions build/ci_utils/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ quote = { workspace = true }
rand = "0.8.4"
regex = { workspace = true }
reqwest = { version = "0.11.5", default-features = false, features = [
"stream"
"stream",
] }
semver = { workspace = true }
serde = { version = "1.0.130", features = ["derive"] }
Expand All @@ -71,7 +71,6 @@ symlink = "0.1.0"
syn = { workspace = true }
sysinfo = "0.26.2"
tar = "0.4.37"
tokio-tar = "0.3.1"
tempfile = "3.2.0"
tokio = { workspace = true }
tokio-stream = { workspace = true }
Expand Down
78 changes: 56 additions & 22 deletions build/ci_utils/src/archive/tar.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
use crate::prelude::*;

use async_compression::tokio::bufread::GzipDecoder;
use tokio::io::AsyncRead;
use tokio::io::BufReader;
use tokio_tar::Archive as Tar;
use flate2::read::GzDecoder;
use std::fs::File;



Expand All @@ -18,57 +16,93 @@ pub struct Archive {
/// The path that the `file` originated from. This is stored for error reporting.
path: Box<Path>,
#[derivative(Debug = "ignore")]
file: Tar<Box<dyn AsyncRead + Unpin + Send>>,
file: tar::Archive<GzDecoder<File>>,
}

impl Archive {
/// Open a gzip-compressed tar archive.
#[context("Failed to open archive: {}", path.as_ref().display())]
pub async fn open_tar_gz(path: impl AsRef<Path>) -> Result<Self> {
let file = crate::fs::tokio::open(&path).await?;
let file = BufReader::new(file);
let file = GzipDecoder::new(file);
let file: Box<dyn AsyncRead + Unpin + Send> = Box::new(file);
let file = Tar::new(file);
let file = file
.try_into_std()
.map_err(|_| anyhow!("Failed to convert tokio::fs::File to std::fs::File"))?;
let tar_stream = flate2::read::GzDecoder::new(file);
let archive = tar::Archive::new(tar_stream);
let path = path.as_ref().to_owned().into_boxed_path();
Ok(Self { path, file })
Ok(Self { path, file: archive })
}

/// The given function will be called with the path of each file within the archive. For each
/// input path, if it returns a path the file will be extracted to the returned path.
///
/// IMPORTANT: If the function uses its input path to generate an output path, care must be
/// taken that the output path is not in an unexpected location, especially if coming from an
/// untrusted archive.
#[context("Failed to extract files from archive: {}", self.path.display())]
pub async fn extract_files(
/// Synchronous version of [`extract_files`].
#[context("Failed to extract files from archive {}", self.path.display())]
pub fn extract_files_sync(
mut self,
mut filter: impl FnMut(&Path) -> Option<PathBuf>,
) -> Result {
let mut entries = self.file.entries()?;
while let Some(entry) = entries.next().await {
let entries = self.file.entries()?;
for entry in entries {
let mut entry = entry?;
let path_in_archive = entry.path()?;
if let Some(output_path) = filter(&path_in_archive) {
trace!("Extracting {}", output_path.display());
entry.unpack(&output_path).await?;
let entry_type = entry.header().entry_type();
let make_message = |prefix, path: Cow<Path>| {
format!(
"{} {:?} entry: {} => {}",
prefix,
entry_type,
path.display(),
output_path.display()
)
};

trace!("{}", make_message("Extracting", path_in_archive));
entry.unpack(&output_path).with_context(|| {
make_message("Failed to extract", entry.path().unwrap_or_default())
})?;
}
}
Ok(())
}

/// The given function will be called with the path of each file within the archive. For each
/// input path, if it returns a path the file will be extracted to the returned path.
///
/// IMPORTANT: If the function uses its input path to generate an output path, care must be
/// taken that the output path is not in an unexpected location, especially if coming from an
/// untrusted archive.
pub async fn extract_files(self, filter: impl FnMut(&Path) -> Option<PathBuf>) -> Result {
let job = move || self.extract_files_sync(filter);
tokio::task::block_in_place(job)
}

/// Extract all files from the specified subtree in the archive, placing them in the specified
/// output directory.
pub async fn extract_subtree(
self,
prefix: impl AsRef<Path>,
output: impl AsRef<Path>,
) -> Result {
let path = self.path.clone();
debug!(
"Extracting subtree '{}' from archive {} to {}",
prefix.as_ref().display(),
self.path.display(),
output.as_ref().display()
);
self.extract_files(|path_in_archive| {
path_in_archive
.strip_prefix(&prefix)
.ok()
.map(|relative_path| output.as_ref().join(relative_path))
})
.await
.with_context(|| {
format!(
"Failed to extract subtree '{}' from archive {} to {}",
prefix.as_ref().display(),
path.display(),
output.as_ref().display()
)
})
}
}
6 changes: 6 additions & 0 deletions build/ci_utils/src/fs/wrappers/tokio.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,9 @@ pub async fn read<P: AsRef<Path>>(path: P) -> Result<Vec<u8>> {
pub async fn read_to_string(path: impl AsRef<Path>) -> Result<String> {
tokio::fs::read_to_string(&path).await.anyhow_err()
}

/// See [`tokio::fs::set_permissions`].
#[context("Failed to set permissions {:?} for file: {}", permissions, path.as_ref().display())]
pub async fn set_permissions(path: impl AsRef<Path>, permissions: std::fs::Permissions) -> Result {
tokio::fs::set_permissions(&path, permissions.clone()).await.anyhow_err()
}
1 change: 1 addition & 0 deletions lib/rust/ensogl/component/text/src/font.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ pub use ttf::Weight;
pub use ttf::Width;



// =================
// === Constants ===
// =================
Expand Down