From 4d9230af220c414b4738f2b1c328f09863828589 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Thu, 9 Nov 2023 11:46:22 -0500 Subject: [PATCH] feat(core): make createNodes async --- packages/nx/src/native/index.d.ts | 4 +- .../src/native/tests/workspace_files.spec.ts | 118 +++++++++++++++++- packages/nx/src/native/types/file_data.rs | 4 +- .../nx/src/native/workspace/config_files.rs | 5 +- packages/nx/src/native/workspace/context.rs | 21 +++- .../src/native/workspace/workspace_files.rs | 110 ++++++++-------- 6 files changed, 200 insertions(+), 62 deletions(-) diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index 6db4dd8c9ecf9f..edce7e11cff6bf 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -134,9 +134,9 @@ export class Watcher { export class WorkspaceContext { workspaceRoot: string constructor(workspaceRoot: string) - getWorkspaceFiles(globs: Array, parseConfigurations: (arg0: Array) => Record): NxWorkspaceFiles + getWorkspaceFiles(globs: Array, parseConfigurations: (arg0: Array) => Promise>): object | null glob(globs: Array): Array - getProjectConfigurations(globs: Array, parseConfigurations: (arg0: Array) => Record): Record + getProjectConfigurations(globs: Array, parseConfigurations: (arg0: Array) => Promise>): object incrementalUpdate(updatedFiles: Array, deletedFiles: Array): Record allFileData(): Array } diff --git a/packages/nx/src/native/tests/workspace_files.spec.ts b/packages/nx/src/native/tests/workspace_files.spec.ts index 84e1d58ca92b85..5782cec7fc6e27 100644 --- a/packages/nx/src/native/tests/workspace_files.spec.ts +++ b/packages/nx/src/native/tests/workspace_files.spec.ts @@ -6,7 +6,7 @@ import { readJsonFile } from '../../utils/fileutils'; describe('workspace files', () => { function createParseConfigurationsFunction(tempDir: string) { - return (filenames: string[]) => { + return async (filenames: string[]) => { const res = {}; for (const filename of filenames) { const json = readJsonFile(join(tempDir, filename)); @@ -51,9 +51,20 @@ describe('workspace files', () => { let globs = ['project.json', '**/project.json', 'libs/*/package.json']; const context = new WorkspaceContext(fs.tempDir); +<<<<<<< HEAD let { projectFileMap, globalFiles } = context.getWorkspaceFiles( globs, createParseConfigurationsFunction(fs.tempDir) +======= + let { projectFileMap, projectConfigurations, globalFiles } = + (await context.getWorkspaceFiles( + globs, + createParseConfigurationsFunction(fs.tempDir) + )) as any; + + let sortedConfigs = Object.values(projectConfigurations).sort((a, b) => + a['name'].localeCompare(b['name']) +>>>>>>> 6d080c8bf (feat(core): make createNodes async) ); expect(projectFileMap).toMatchInlineSnapshot(` @@ -128,6 +139,7 @@ describe('workspace files', () => { `); }); +<<<<<<< HEAD it('should assign files to the root project if it exists', async () => { const fs = new TempFs('workspace-files'); const nxJson: NxJsonConfiguration = {}; @@ -179,6 +191,110 @@ describe('workspace files', () => { ] `); }); +======= + // it('should assign files to the root project if it exists', async () => { + // const fs = new TempFs('workspace-files'); + // const nxJson: NxJsonConfiguration = {}; + // await fs.createFiles({ + // './nx.json': JSON.stringify(nxJson), + // './package.json': JSON.stringify({ + // name: 'repo-name', + // version: '0.0.0', + // dependencies: {}, + // }), + // './project.json': JSON.stringify({ + // name: 'repo-name', + // }), + // './src/index.js': '', + // './jest.config.js': '', + // }); + // + // const context = new WorkspaceContext(fs.tempDir); + // + // const globs = ['project.json', '**/project.json', '**/package.json']; + // const { globalFiles, projectFileMap } = context.getWorkspaceFiles( + // globs, + // createParseConfigurationsFunction(fs.tempDir) + // ); + // + // expect(globalFiles).toEqual([]); + // expect(projectFileMap['repo-name']).toMatchInlineSnapshot(` + // [ + // { + // "file": "jest.config.js", + // "hash": "3244421341483603138", + // }, + // { + // "file": "nx.json", + // "hash": "1389868326933519382", + // }, + // { + // "file": "package.json", + // "hash": "14409636362330144230", + // }, + // { + // "file": "project.json", + // "hash": "4357927788053707201", + // }, + // { + // "file": "src/index.js", + // "hash": "3244421341483603138", + // }, + // ] + // `); + // }); + // + // it('should dedupe configuration files', async () => { + // const fs = new TempFs('workspace-files'); + // const nxJson: NxJsonConfiguration = {}; + // await fs.createFiles({ + // './nx.json': JSON.stringify(nxJson), + // './package.json': JSON.stringify({ + // name: 'repo-name', + // version: '0.0.0', + // dependencies: {}, + // }), + // './project.json': JSON.stringify({ + // name: 'repo-name', + // }), + // './libs/project1/project.json': JSON.stringify({ + // name: 'project1', + // }), + // './libs/project1/package.json': JSON.stringify({ + // name: 'project1', + // }), + // './libs/project1/index.js': '', + // }); + // + // const context = new WorkspaceContext(fs.tempDir); + // let globs = ['project.json', '**/project.json', '**/package.json']; + // + // let nodes = context.getProjectConfigurations(globs, (filenames) => { + // const res = {}; + // for (const filename of filenames) { + // const json = readJsonFile(join(fs.tempDir, filename)); + // res[json.name] = { + // ...json, + // root: dirname(filename), + // }; + // } + // return { + // externalNodes: {}, + // projectNodes: res, + // }; + // }); + // expect(nodes.projectNodes).toEqual({ + // project1: { + // name: 'project1', + // root: 'libs/project1', + // }, + // 'repo-name': expect.objectContaining({ + // name: 'repo-name', + // root: '.', + // }), + // }); + // }); +>>>>>>> 6d080c8bf (feat(core): make createNodes async) // describe('errors', () => { // it('it should infer names of configuration files without a name', async () => { diff --git a/packages/nx/src/native/types/file_data.rs b/packages/nx/src/native/types/file_data.rs index 3b3b4f7d1e202f..2bfdf4fb057781 100644 --- a/packages/nx/src/native/types/file_data.rs +++ b/packages/nx/src/native/types/file_data.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; #[napi(object)] -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct FileData { pub file: String, pub hash: String, @@ -17,7 +17,7 @@ impl PartialEq for FileData { impl PartialOrd for FileData { fn partial_cmp(&self, other: &Self) -> Option { - self.file.partial_cmp(&other.file) + Some(self.cmp(other)) } } diff --git a/packages/nx/src/native/workspace/config_files.rs b/packages/nx/src/native/workspace/config_files.rs index 814c005b8cf71a..4a0cb2e8591c4d 100644 --- a/packages/nx/src/native/workspace/config_files.rs +++ b/packages/nx/src/native/workspace/config_files.rs @@ -1,6 +1,7 @@ use crate::native::glob::build_glob_set; use crate::native::utils::path::Normalize; use std::collections::HashMap; +use napi::bindgen_prelude::{Object, Promise}; use crate::native::workspace::errors::{InternalWorkspaceErrors, WorkspaceErrors}; use rayon::prelude::*; @@ -29,9 +30,9 @@ pub(super) fn get_project_configurations( globs: Vec, files: Option<&[(PathBuf, String)]>, parse_configurations: ConfigurationParser, -) -> napi::Result> +) -> napi::Result>> where - ConfigurationParser: Fn(Vec) -> napi::Result>, + ConfigurationParser: Fn(Vec) -> napi::Result>>, { let config_paths = glob_files(globs, files).map_err(anyhow::Error::from)?; diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index 0dc7bfee6e28e8..937b5da0fccdca 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use crate::native::types::FileData; use crate::native::utils::path::Normalize; +use napi::bindgen_prelude::*; use parking_lot::lock_api::MutexGuard; use parking_lot::{Condvar, Mutex, RawMutex}; use rayon::prelude::*; @@ -151,17 +152,20 @@ impl WorkspaceContext { #[napi] pub fn get_workspace_files( &self, + env: Env, globs: Vec, parse_configurations: ConfigurationParser, - ) -> napi::Result + ) -> anyhow::Result> where - ConfigurationParser: Fn(Vec) -> napi::Result>, + ConfigurationParser: Fn(Vec) -> napi::Result>>, { workspace_files::get_files( + env, globs, parse_configurations, self.files_worker.get_files().as_deref(), ) + .map_err(anyhow::Error::from) } #[napi] @@ -172,17 +176,22 @@ impl WorkspaceContext { #[napi] pub fn get_project_configurations( &self, + env: Env, globs: Vec, parse_configurations: ConfigurationParser, - ) -> napi::Result> + ) -> napi::Result where - ConfigurationParser: Fn(Vec) -> napi::Result>, + ConfigurationParser: Fn(Vec) -> napi::Result>>, { - config_files::get_project_configurations( + let promise = config_files::get_project_configurations( globs, self.files_worker.get_files().as_deref(), parse_configurations, - ) + )?; + env.spawn_future(async move { + let result = promise.await?; + Ok(result) + }) } #[napi] diff --git a/packages/nx/src/native/workspace/workspace_files.rs b/packages/nx/src/native/workspace/workspace_files.rs index 64c61b38bcbbf3..1caeaad02824a7 100644 --- a/packages/nx/src/native/workspace/workspace_files.rs +++ b/packages/nx/src/native/workspace/workspace_files.rs @@ -1,7 +1,10 @@ +use napi::bindgen_prelude::{Object, Promise}; use std::collections::HashMap; use std::path::{Path, PathBuf}; +use napi::{Env, JsObject}; use rayon::prelude::*; +use serde_json::Value; use tracing::trace; use crate::native::types::FileData; @@ -18,74 +21,83 @@ pub struct NxWorkspaceFiles { } pub(super) fn get_files( + env: Env, globs: Vec, parse_configurations: ConfigurationParser, file_data: Option<&[(PathBuf, String)]>, -) -> napi::Result +) -> napi::Result> where - ConfigurationParser: Fn(Vec) -> napi::Result>, + ConfigurationParser: Fn(Vec) -> napi::Result>>, { let Some(file_data) = file_data else { return Ok(Default::default()); }; trace!("{globs:?}"); - let root_map = transform_root_map( - config_files::get_project_configurations(globs, Some(file_data), parse_configurations) - .map_err(|e| InternalWorkspaceErrors::ParseError(e.to_string()))?, - ); + let file_data = file_data.to_vec(); + let promise = + config_files::get_project_configurations(globs, Some(&file_data), parse_configurations)?; - trace!(?root_map); + let result = env.spawn_future(async move { + let parsed_graph_nodes = promise.await?; - let file_locations = file_data - .into_par_iter() - .map(|(file_path, hash)| { - let mut parent = file_path.parent().unwrap_or_else(|| Path::new(".")); + let root_map = transform_root_map( + parsed_graph_nodes + ); - while root_map.get(parent).is_none() && parent != Path::new(".") { - parent = parent.parent().unwrap_or_else(|| Path::new(".")); - } + trace!(?root_map); - let file_data = FileData { - file: file_path.to_normalized_string(), - hash: hash.clone(), - }; + let file_locations = file_data + .into_par_iter() + .map(|(file_path, hash)| { + let mut parent = file_path.parent().unwrap_or_else(|| Path::new(".")); - match root_map.get(parent) { - Some(project_name) => (FileLocation::Project(project_name.into()), file_data), - None => (FileLocation::Global, file_data), - } - }) - .collect::>(); + while root_map.get(parent).is_none() && parent != Path::new(".") { + parent = parent.parent().unwrap_or_else(|| Path::new(".")); + } - let mut project_file_map: HashMap> = HashMap::with_capacity( - file_locations - .iter() - .filter(|&f| f.0 != FileLocation::Global) - .count(), - ); - let mut global_files: Vec = Vec::with_capacity( - file_locations - .iter() - .filter(|&f| f.0 == FileLocation::Global) - .count(), - ); - for (file_location, file_data) in file_locations { - match file_location { - FileLocation::Global => global_files.push(file_data), - FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) { - None => { - project_file_map.insert(project_name.clone(), vec![file_data]); + let file_data = FileData { + file: file_path.to_normalized_string(), + hash: hash.clone(), + }; + + match root_map.get(parent) { + Some(project_name) => (FileLocation::Project(project_name.into()), file_data), + None => (FileLocation::Global, file_data), } - Some(project_files) => project_files.push(file_data), - }, + }) + .collect::>(); + + let mut project_file_map: HashMap> = HashMap::with_capacity( + file_locations + .iter() + .filter(|&f| f.0 != FileLocation::Global) + .count(), + ); + let mut global_files: Vec = Vec::with_capacity( + file_locations + .iter() + .filter(|&f| f.0 == FileLocation::Global) + .count(), + ); + for (file_location, file_data) in file_locations { + match file_location { + FileLocation::Global => global_files.push(file_data), + FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) { + None => { + project_file_map.insert(project_name.clone(), vec![file_data]); + } + Some(project_files) => project_files.push(file_data), + }, + } } - } - Ok(NxWorkspaceFiles { - project_file_map, - global_files, - }) + Ok(NxWorkspaceFiles { + project_file_map, + global_files, + }) + })?; + Ok(Some(result)) } fn transform_root_map(root_map: HashMap) -> hashbrown::HashMap {