From a741b3357133c1b0c481ac78b980e8e87f46d945 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Fri, 20 Oct 2023 16:51:19 -0700 Subject: [PATCH] Cargo registry server updates to download crates --- Cargo.lock | 1 + cargo-registry/Cargo.toml | 1 + cargo-registry/src/crate_handler.rs | 61 ++++++++++++-------------- cargo-registry/src/main.rs | 12 +++-- cargo-registry/src/response_builder.rs | 7 +++ 5 files changed, 46 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6de911a6b63477..55f7dc2a20f153 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5489,6 +5489,7 @@ version = "1.18.0" dependencies = [ "clap 2.33.3", "flate2", + "hex", "hyper", "log", "rustc_version 0.4.0", diff --git a/cargo-registry/Cargo.toml b/cargo-registry/Cargo.toml index afc5bf363b0bab..4e13f477ee5e2d 100644 --- a/cargo-registry/Cargo.toml +++ b/cargo-registry/Cargo.toml @@ -12,6 +12,7 @@ edition = { workspace = true } [dependencies] clap = { workspace = true } flate2 = { workspace = true } +hex = { workspace = true } hyper = { workspace = true, features = ["full"] } log = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/cargo-registry/src/crate_handler.rs b/cargo-registry/src/crate_handler.rs index c55ab4ff07395f..e95f51d752552b 100644 --- a/cargo-registry/src/crate_handler.rs +++ b/cargo-registry/src/crate_handler.rs @@ -25,10 +25,9 @@ use { mem::size_of, ops::Deref, path::{Path, PathBuf}, - str::FromStr, sync::Arc, }, - tar::{Archive, Builder}, + tar::{Archive, Builder, HeaderMode}, tempfile::{tempdir, TempDir}, }; @@ -148,9 +147,9 @@ impl Program { } pub(crate) fn crate_name_to_program_id(crate_name: &str) -> Option { - crate_name - .split_once('-') - .and_then(|(_prefix, id_str)| Pubkey::from_str(id_str).ok()) + hex::decode(crate_name) + .ok() + .and_then(|bytes| Pubkey::try_from(bytes).ok()) } } @@ -169,34 +168,21 @@ pub(crate) struct CratePackage(pub(crate) Bytes); impl From for Result { fn from(value: UnpackedCrate) -> Self { let mut archive = Builder::new(Vec::new()); - archive.append_dir_all(".", value.tempdir.path())?; + archive.mode(HeaderMode::Deterministic); + + let base_path = UnpackedCrate::make_path(&value.tempdir, &value.meta, "out"); + archive.append_dir_all( + format!("{}-{}/out", value.meta.name, value.meta.vers), + base_path, + )?; let data = archive.into_inner()?; + let reader = Cursor::new(data); - let mut encoder = GzEncoder::new(reader, Compression::fast()); + let mut encoder = GzEncoder::new(reader, Compression::default()); let mut zipped_data = Vec::new(); encoder.read_to_end(&mut zipped_data)?; - let meta_str = serde_json::to_string(&value.meta)?; - - let sizeof_length = size_of::(); - let mut packed = Vec::with_capacity( - sizeof_length - .saturating_add(meta_str.len()) - .saturating_add(sizeof_length) - .saturating_add(zipped_data.len()), - ); - - packed[..sizeof_length].copy_from_slice(&u32::to_le_bytes(meta_str.len() as u32)); - let offset = sizeof_length; - let end = offset.saturating_add(meta_str.len()); - packed[offset..end].copy_from_slice(meta_str.as_bytes()); - let offset = end; - let end = offset.saturating_add(sizeof_length); - packed[offset..end].copy_from_slice(&u32::to_le_bytes(zipped_data.len() as u32)); - let offset = end; - packed[offset..].copy_from_slice(&zipped_data); - - Ok(CratePackage(Bytes::from(packed))) + Ok(CratePackage(Bytes::from(zipped_data))) } } @@ -227,6 +213,10 @@ impl From for Result { let lib_name = UnpackedCrate::program_library_name(&tempdir, &meta)?; + let base_path = UnpackedCrate::make_path(&tempdir, &meta, "out"); + fs::create_dir_all(base_path) + .map_err(|_| "Failed to create the base directory for output")?; + let program_path = UnpackedCrate::make_path(&tempdir, &meta, format!("out/{}.so", lib_name)) .into_os_string() @@ -273,14 +263,15 @@ impl UnpackedCrate { pub(crate) fn fetch_index(id: Pubkey, client: Arc) -> Result { let (_program, unpacked_crate) = Self::fetch_program(id, client)?; - let mut entry: IndexEntry = unpacked_crate.meta.clone().into(); - entry.cksum = unpacked_crate.cksum.clone(); + let packed_crate: Result = UnpackedCrate::into(unpacked_crate); + let packed_crate = packed_crate?; + + entry.cksum = format!("{:x}", Sha256::digest(&packed_crate.0)); Ok(entry) } - #[allow(dead_code)] pub(crate) fn fetch(id: Pubkey, client: Arc) -> Result { let (_program, unpacked_crate) = Self::fetch_program(id, client)?; UnpackedCrate::into(unpacked_crate) @@ -299,8 +290,8 @@ impl UnpackedCrate { fn new_empty(id: Pubkey) -> Result { let meta = PackageMetaData { - name: id.to_string(), - vers: "0.1".to_string(), + name: hex::encode(id.to_bytes()), + vers: "0.1.0".to_string(), deps: vec![], features: BTreeMap::new(), authors: vec![], @@ -321,6 +312,10 @@ impl UnpackedCrate { let tempdir = tempdir()?; + let base_path = UnpackedCrate::make_path(&tempdir, &meta, "out"); + fs::create_dir_all(base_path) + .map_err(|_| "Failed to create the base directory for output")?; + let program_path = Self::make_path(&tempdir, &meta, format!("out/{}.so", id)) .into_os_string() .into_string() diff --git a/cargo-registry/src/main.rs b/cargo-registry/src/main.rs index 073b8e42cb609c..288e7fc9e69388 100644 --- a/cargo-registry/src/main.rs +++ b/cargo-registry/src/main.rs @@ -91,12 +91,18 @@ impl CargoRegistryService { return response_builder::error_incorrect_length(); } - let _package = Program::crate_name_to_program_id(crate_name) + let package = Program::crate_name_to_program_id(crate_name) .and_then(|id| UnpackedCrate::fetch(id, client).ok()); // Return the package to the caller in the response - - response_builder::error_not_implemented() + if let Some(package) = package { + response_builder::success_response_bytes(package.0) + } else { + response_builder::error_response( + hyper::StatusCode::BAD_REQUEST, + "Failed to find the package", + ) + } } fn handle_yank_request( diff --git a/cargo-registry/src/response_builder.rs b/cargo-registry/src/response_builder.rs index a8da2d9d6fdbba..2b1ae2906b7707 100644 --- a/cargo-registry/src/response_builder.rs +++ b/cargo-registry/src/response_builder.rs @@ -22,6 +22,13 @@ pub(crate) fn success_response_str(value: &str) -> hyper::Response .unwrap() } +pub(crate) fn success_response_bytes(bytes: hyper::body::Bytes) -> hyper::Response { + hyper::Response::builder() + .status(hyper::StatusCode::OK) + .body(hyper::Body::from(bytes)) + .unwrap() +} + pub(crate) fn success_response() -> hyper::Response { success_response_str("") }