From da732201eb520b35eb690edbbb5f316f629c7885 Mon Sep 17 00:00:00 2001 From: Shahen Hovhannisyan Date: Sat, 24 Feb 2024 22:57:01 +0400 Subject: [PATCH] Linker in one folder --- .gitignore | 3 +- Cargo.lock | 8 ++ Cargo.toml | 3 + package.json | 1 + src/actors/install.rs | 16 +++- src/cache/registry.rs | 2 +- src/fs/copy.rs | 40 +++++++++ src/fs/mod.rs | 3 + src/lib.rs | 1 + src/lock/file.rs | 5 +- src/lock/mod.rs | 1 + src/lock/shape.rs | 8 ++ src/pipeline/artifacts/linker_artifacts.rs | 23 ++--- src/pipeline/artifacts/mod.rs | 4 +- src/pipeline/artifacts/resolve_artifacts.rs | 34 ++++++-- src/pipeline/downloader.rs | 6 +- src/pipeline/linker.rs | 95 ++++++++++++++++----- src/pipeline/mod.rs | 1 + src/pipeline/resolver.rs | 19 +++-- 19 files changed, 207 insertions(+), 66 deletions(-) create mode 100644 src/fs/copy.rs create mode 100644 src/fs/mod.rs create mode 100644 src/lock/shape.rs diff --git a/.gitignore b/.gitignore index dd9ba93..9610b35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target -craft.lock \ No newline at end of file +craft.lock +node_modules \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index faf59d3..cce4f0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,7 +257,9 @@ dependencies = [ "colored", "console", "flate2", + "fs_extra", "indicatif", + "lazy_static", "regex", "reqwest", "serde", @@ -365,6 +367,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.30" diff --git a/Cargo.toml b/Cargo.toml index 1f7a72d..9cec259 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,3 +28,6 @@ regex = "1.10.2" indicatif = "0.17.7" console = { version = "0.15", default-features = false, features = ["ansi-parsing"] } + +lazy_static = "1.4.0" +fs_extra = "1.3.0" diff --git a/package.json b/package.json index d06797f..0799a4b 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,5 @@ { + "type": "module", "dependencies": { "express": "^4.18.2", "mongoose": "^5.9.7", diff --git a/src/actors/install.rs b/src/actors/install.rs index c309a0c..9f5bd8a 100644 --- a/src/actors/install.rs +++ b/src/actors/install.rs @@ -42,6 +42,8 @@ impl Actor for InstallActor { let ui_thread = self.start_progress(rx); + // ─── Start Resolving ───────────────────────── + CraftLogger::verbose_n(3, "Resolving dependencies"); let resolve_artifacts = ResolverPipe::new(self.packages.clone(), tx.clone()) .run() @@ -51,6 +53,8 @@ impl Actor for InstallActor { format!("Resolved: {:?}", resolve_artifacts.get_artifacts().len()), ); + // ─── Start Downloading ────────────────────── + CraftLogger::verbose_n(3, "Downloading dependencies"); let download_artifacts = DownloaderPipe::new(&resolve_artifacts, tx.clone()) .run() @@ -60,9 +64,10 @@ impl Actor for InstallActor { 3, format!("Downloaded {:?}", download_artifacts.get_artifacts().len()), ); - CraftLogger::verbose_n(3, "Extracting dependencies"); - #[allow(unused_variables)] + // ─── Start Extracting ─────────────────────── + + CraftLogger::verbose_n(3, "Extracting dependencies"); let extracted_artifacts = ExtractorPipe::new(&download_artifacts, tx.clone()) .run() .await?; @@ -71,6 +76,9 @@ impl Actor for InstallActor { 3, format!("Extracted {:?}", extracted_artifacts.get_artifacts().len()), ); + + // ─── Start Linking ────────────────────────── + CraftLogger::verbose_n(3, "Linking dependencies"); LinkerPipe::new( tx.clone(), @@ -80,12 +88,16 @@ impl Actor for InstallActor { .run() .await?; + // ─── Sync Lock File ──────────────────────── + LockFile::sync( resolve_artifacts.get_artifacts(), env::current_dir().unwrap(), ) .await; + // ─── Cleanup ──────────────────────────────── + ExtractorPipe::cleanup().await; drop(tx); diff --git a/src/cache/registry.rs b/src/cache/registry.rs index f06552b..cb44224 100644 --- a/src/cache/registry.rs +++ b/src/cache/registry.rs @@ -16,7 +16,7 @@ pub struct RegistryCache { // ─────────────────────────────────────────────────────────────────────────────── impl RegistryCache { - pub async fn save(&self) -> Result<(), CacheError> { + pub async fn persist(&self) -> Result<(), CacheError> { let cache_file = File::create(self.directory.join(REGISTRY_CACHE_FILE)).unwrap(); serde_json::to_writer(cache_file, &self.cache).unwrap(); diff --git a/src/fs/copy.rs b/src/fs/copy.rs new file mode 100644 index 0000000..1ea72fc --- /dev/null +++ b/src/fs/copy.rs @@ -0,0 +1,40 @@ +use std::fs; +use std::path::Path; +use fs_extra::dir::CopyOptions; + +pub fn copy_dir(from: &Path, to: &Path) -> Result<(), Box> { + if to.exists() { + fs::remove_dir_all(&to).unwrap(); + } + + fs::create_dir_all(&to).unwrap(); + + let options = CopyOptions::new().overwrite(true); + + copy_recursive(&from, &to, &options) +} + +fn copy_recursive(from: &std::path::Path, to: &std::path::Path, options: &CopyOptions) -> Result<(), Box> { + let from_meta = fs::metadata(from)?; + + if from_meta.is_dir() { + let dir_entries = fs::read_dir(from)?; + + for entry in dir_entries { + let entry = entry?; + let entry_path = entry.path(); + let entry_name = entry.file_name().into_string().unwrap(); // Convert OsString to String + + let dest_path = to.join(&entry_name); + + if entry_path.is_dir() { + fs::create_dir_all(&dest_path)?; + copy_recursive(&entry_path, &dest_path, options)?; + } else { + fs_extra::copy_items(&vec![entry_path], &to, options)?; + } + } + } + + Ok(()) +} diff --git a/src/fs/mod.rs b/src/fs/mod.rs new file mode 100644 index 0000000..43cebcb --- /dev/null +++ b/src/fs/mod.rs @@ -0,0 +1,3 @@ +mod copy; + +pub use copy::copy_dir; \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index a64fe12..9f700f4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ mod perf; mod registry; mod tar; mod ui; +mod fs; mod pipeline; diff --git a/src/lock/file.rs b/src/lock/file.rs index 61969e8..f4c121f 100644 --- a/src/lock/file.rs +++ b/src/lock/file.rs @@ -1,8 +1,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; - -use crate::{contracts::LOCK_FILE_NAME, package::NpmPackage}; +use crate::{contracts::LOCK_FILE_NAME, package::NpmPackage, pipeline::ResolvedItem}; #[derive(Debug, Deserialize, Serialize, Default)] pub struct LockFile { @@ -25,7 +24,7 @@ impl LockFile { lock_file } - pub async fn sync(package: Vec, path: PathBuf) { + pub async fn sync(package: Vec, path: PathBuf) { let file_path = path.join(LOCK_FILE_NAME); if file_path.exists() { diff --git a/src/lock/mod.rs b/src/lock/mod.rs index 3382dbe..82ce8be 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -1,3 +1,4 @@ mod file; +mod shape; pub use file::LockFile; diff --git a/src/lock/shape.rs b/src/lock/shape.rs new file mode 100644 index 0000000..b063cab --- /dev/null +++ b/src/lock/shape.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +use crate::package::NpmPackage; + +#[derive(Debug, Serialize, Deserialize)] +pub struct LockShape { + pub package: NpmPackage, +} \ No newline at end of file diff --git a/src/pipeline/artifacts/linker_artifacts.rs b/src/pipeline/artifacts/linker_artifacts.rs index 6664394..52d8552 100644 --- a/src/pipeline/artifacts/linker_artifacts.rs +++ b/src/pipeline/artifacts/linker_artifacts.rs @@ -1,27 +1,18 @@ use std::path::PathBuf; -// ───────────────────────────────────────────────────────────────────────────── -#[derive(Debug, Default)] -pub struct LinkerArtifacts { - pub artifacts: Vec, -} +// ───────────────────────────────────────────────────────────────────────────── -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct LinkArtifactItem { - pub path: PathBuf, + pub to: PathBuf, + pub from: PathBuf, } // ───────────────────────────────────────────────────────────────────────────── impl LinkArtifactItem { - pub fn new(path: PathBuf) -> Self { - Self { path } + pub fn new(from: PathBuf, to: PathBuf) -> Self { + Self { from, to } } -} - -impl LinkerArtifacts { - pub fn add(&mut self, path: PathBuf) { - self.artifacts.push(LinkArtifactItem::new(path)); - } -} +} \ No newline at end of file diff --git a/src/pipeline/artifacts/mod.rs b/src/pipeline/artifacts/mod.rs index a6feee3..6d4242a 100644 --- a/src/pipeline/artifacts/mod.rs +++ b/src/pipeline/artifacts/mod.rs @@ -5,5 +5,5 @@ mod resolve_artifacts; pub use download_artifacts::{DownloadArtifacts, StoredArtifact}; pub use extract_artifacts::{ExtractArtifacts, ExtractArtifactsMap}; -pub use linker_artifacts::LinkerArtifacts; -pub use resolve_artifacts::ResolveArtifacts; +pub use resolve_artifacts::{ResolveArtifacts, ResolvedItem}; +pub use linker_artifacts::LinkArtifactItem; diff --git a/src/pipeline/artifacts/resolve_artifacts.rs b/src/pipeline/artifacts/resolve_artifacts.rs index a07ac51..e6536f3 100644 --- a/src/pipeline/artifacts/resolve_artifacts.rs +++ b/src/pipeline/artifacts/resolve_artifacts.rs @@ -6,7 +6,25 @@ use crate::{contracts::PipeArtifact, package::NpmPackage}; #[derive(Debug, Clone)] pub struct ResolveArtifacts { - packages: HashMap, + packages: HashMap, +} + +#[derive(Debug, Clone)] +pub struct ResolvedItem { + pub package: NpmPackage, + pub parent: Option, +} + +// -------------------------------------------------------------------------------- + +impl ResolvedItem { + pub fn new(package: NpmPackage, parent: Option) -> Self { + Self { package, parent } + } + + pub fn with_no_parent(package: NpmPackage) -> Self { + Self::new(package, None) + } } // -------------------------------------------------------------------------------- @@ -18,19 +36,19 @@ impl ResolveArtifacts { } } - pub fn get(&self, key: &str) -> Option<&NpmPackage> { + pub fn get(&self, key: &str) -> Option<&ResolvedItem> { self.packages.get(key) } - pub fn insert(&mut self, key: String, value: NpmPackage) { + pub fn insert(&mut self, key: String, value: ResolvedItem) { self.packages.insert(key, value); } } // -------------------------------------------------------------------------------- -impl PipeArtifact> for ResolveArtifacts { - fn get_artifacts(&self) -> Vec { +impl PipeArtifact> for ResolveArtifacts { + fn get_artifacts(&self) -> Vec { self.packages.values().cloned().collect() } } @@ -58,9 +76,9 @@ mod tests { "#, ) .unwrap(); - resolve_artifacts.insert("package".to_string(), package); + resolve_artifacts.insert("package".to_string(), ResolvedItem::with_no_parent(package)); - assert_eq!(resolve_artifacts.get("package").unwrap().version, "1.0.0"); + assert_eq!(resolve_artifacts.get("package").unwrap().package.version, "1.0.0"); } #[test] @@ -80,7 +98,7 @@ mod tests { "#, ) .unwrap(); - resolve_artifacts.insert("package".to_string(), package); + resolve_artifacts.insert("package".to_string(), ResolvedItem::with_no_parent(package)); assert_eq!(resolve_artifacts.get_artifacts().len(), 1); } diff --git a/src/pipeline/downloader.rs b/src/pipeline/downloader.rs index 670b9da..aa2d679 100644 --- a/src/pipeline/downloader.rs +++ b/src/pipeline/downloader.rs @@ -15,7 +15,7 @@ use crate::{ package::NpmPackage, }; -use super::artifacts::DownloadArtifacts; +use super::artifacts::{DownloadArtifacts, ResolvedItem}; // ─── DownloaderPipe ───────────────────────────────────────────────────────────── @@ -30,9 +30,9 @@ pub struct DownloaderPipe> { // ─── Implementation ─────────────────────────────────────────────────────────── impl DownloaderPipe { - pub fn new(artifacts: &dyn PipeArtifact>, tx: Sender) -> Self { + pub fn new(artifacts: &dyn PipeArtifact>, tx: Sender) -> Self { Self { - packages: artifacts.get_artifacts(), + packages: artifacts.get_artifacts().iter().map(|item| item.package.clone()).collect(), cache: Arc::new(Mutex::new(PackagesCache::default())), artifacts: Arc::new(Mutex::new(DownloadArtifacts::new())), tx, diff --git a/src/pipeline/linker.rs b/src/pipeline/linker.rs index 5f785c5..435f8e9 100644 --- a/src/pipeline/linker.rs +++ b/src/pipeline/linker.rs @@ -1,68 +1,117 @@ -use std::sync::mpsc::Sender; +use std::{env, path::PathBuf, sync::mpsc::Sender}; use async_trait::async_trait; +use lazy_static::lazy_static; use crate::{ contracts::{Logger, Phase, Pipe, ProgressAction}, errors::ExecutionError, + fs::copy_dir, logger::CraftLogger, - package::NpmPackage, }; -use super::artifacts::{ExtractArtifactsMap, LinkerArtifacts}; +use super::artifacts::{ExtractArtifactsMap, LinkArtifactItem, ResolvedItem}; + +// ───────────────────────────────────────────────────────────────────────────── #[derive(Debug)] pub struct LinkerPipe { tx: Sender, - resolved: Vec, + resolved: Vec, extracted: ExtractArtifactsMap, - artifacts: LinkerArtifacts, } +// ───────────────────────────────────────────────────────────────────────────── + +lazy_static! { + pub static ref NODE_MODULES: PathBuf = env::current_dir().unwrap().join("node_modules"); +} + +// ───────────────────────────────────────────────────────────────────────────── + impl LinkerPipe { pub fn new( tx: Sender, - resolved: Vec, + resolved: Vec, extracted: ExtractArtifactsMap, ) -> Self { Self { tx, - artifacts: LinkerArtifacts::default(), resolved, extracted, } } - fn build_linker_artifacts(&mut self) { - // We need to iterate over the resolved packages - // And build the linker artifacts using extracted artifacts + fn build_linker_artifacts(&mut self) -> Vec { + let mut linker_artifacts = vec![]; - for package in &self.resolved { - if let Some(extracted) = self.extracted.get(&package.name) { - self.artifacts.add(extracted.unzip_at.clone()); - } else { - CraftLogger::warn(format!( - "Package not found in extracted artifacts: {}@{}", - package.name, package.version - )); - todo!("Handle missing extracted artifacts"); + for resolved in &self.resolved { + let pkg = &resolved.package; + let parent = &resolved.parent; + + if resolved.package.name == "body-parser" { + CraftLogger::info(format!("Parent: {:?}", parent)); } + + let from = self.extracted.get(&pkg.to_string()).unwrap().clone(); + let to = NODE_MODULES.join(&pkg.name); + let to = if let Some(parent) = parent { + let path_vec = parent.split("/").collect::>(); + let mut path = PathBuf::new(); + + for p in path_vec { + path.push(p); + } + NODE_MODULES + .join(&path) + .join("node_modules") + .join(&pkg.name) + } else { + NODE_MODULES.join(&pkg.name) + }; + + linker_artifacts.push(LinkArtifactItem::new(from.unzip_at, to)); } + + linker_artifacts } - fn link(&mut self) { - println!("Linking artifacts: {:?}", self.artifacts.artifacts); + async fn link(&mut self, artifacts: Vec) { + for artifact in artifacts { + if let Err(e) = std::fs::create_dir_all(&artifact.to) { + CraftLogger::error(format!( + "Failed to create directory: {}", + artifact.to.display() + )); + CraftLogger::error(format!("Error: {}", e)); + } + + if let Err(e) = copy_dir(&artifact.from.join("package"), &artifact.to) { + CraftLogger::error(format!( + "Failed to copy from: {} to: {}: Error: {}", + artifact.from.display(), + artifact.to.display(), + e + )); + CraftLogger::error(format!("Error: {}", e)); + } + } } } +// ───────────────────────────────────────────────────────────────────────────── + #[async_trait] impl Pipe<()> for LinkerPipe { async fn run(&mut self) -> Result<(), ExecutionError> { let _ = self.tx.send(ProgressAction::new(Phase::Linking)); - self.build_linker_artifacts(); - self.link(); + let artifacts = self.build_linker_artifacts(); + + self.link(artifacts).await; Ok(()) } } + +// ───────────────────────────────────────────────────────────────────────────── diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs index 0d66e1e..31e19b0 100644 --- a/src/pipeline/mod.rs +++ b/src/pipeline/mod.rs @@ -12,3 +12,4 @@ pub use extractor::ExtractorPipe; pub use linker::LinkerPipe; pub use cache_clean::CacheCleanPipe; +pub use artifacts::ResolvedItem; diff --git a/src/pipeline/resolver.rs b/src/pipeline/resolver.rs index 3dec562..ccf9e56 100644 --- a/src/pipeline/resolver.rs +++ b/src/pipeline/resolver.rs @@ -11,7 +11,7 @@ use crate::package::{NpmPackage, Package}; use crate::registry::GitRegistry; use crate::registry::NpmRegistry; -use super::artifacts::ResolveArtifacts; +use super::artifacts::{ResolveArtifacts, ResolvedItem}; // ─── ResolverPipe ──────────────────────────────────────────────────────────── @@ -44,7 +44,7 @@ impl ResolverPipe { } #[async_recursion] - async fn resolve_pkg(&mut self, package: &Package) -> Result<(), NetworkError> { + async fn resolve_pkg(&mut self, package: &Package, parent: Option) -> Result<(), NetworkError> { CraftLogger::verbose(format!("Resolving package: {}", package.to_string())); let artifact_key = package.to_string(); @@ -61,7 +61,7 @@ impl ResolverPipe { if let Some(pkg) = cached_pkg { CraftLogger::verbose(format!("Package found in cache: {}", package.to_string())); - self.artifacts.insert(artifact_key.clone(), pkg.clone()); + self.artifacts.insert(artifact_key.clone(), ResolvedItem::new(pkg.clone(), parent)); return Ok(()); } @@ -70,7 +70,7 @@ impl ResolverPipe { let pkg_cache_key = remote_package.to_string(); self.artifacts - .insert(pkg_cache_key.clone(), remote_package.clone()); + .insert(pkg_cache_key.clone(), ResolvedItem::new(remote_package.clone(), parent.clone())); self.cache.set(&pkg_cache_key, remote_package.clone()).await; @@ -79,10 +79,15 @@ impl ResolverPipe { let package = Package::new(pkg.as_str()); - self.resolve_pkg(&package).await?; + let parent = if let Some(ref p) = parent { + Some(format!("{}/{}", p, remote_package.name)) + } else { + Some(remote_package.name.clone()) + }; + self.resolve_pkg(&package, parent).await?; } - match self.cache.save().await { + match self.cache.persist().await { Ok(_) => (), Err(e) => { println!("Failed to save registry: {}", e); @@ -98,7 +103,7 @@ impl ResolverPipe { for pkg in self.packages.clone() { let package = Package::new(pkg.as_str()); - self.resolve_pkg(&package).await?; + self.resolve_pkg(&package, None).await?; } Ok(())