From 2fcf807bb6f0769d6b092798a220517947d9eb47 Mon Sep 17 00:00:00 2001 From: Jason Jean Date: Fri, 7 Jul 2023 16:06:44 -0400 Subject: [PATCH] fix(core): parse project configs only in js (#18009) (cherry picked from commit c04053b4e931176d8be6cd364841f3e0990710d2) --- Cargo.lock | 43 --- packages/nx/Cargo.toml | 3 - packages/nx/src/config/workspaces.ts | 30 +- .../generators/utils/project-configuration.ts | 2 +- packages/nx/src/native/index.d.ts | 7 +- packages/nx/src/native/index.js | 4 +- .../src/native/tests/workspace_files.spec.ts | 305 +++++++++--------- packages/nx/src/native/utils/path.rs | 4 + .../src/native/workspace/get_config_files.rs | 65 ++-- .../workspace/get_nx_workspace_files.rs | 122 ++----- packages/nx/src/native/workspace/types.rs | 7 - .../utils/retrieve-workspace-files.ts | 83 +++-- 12 files changed, 298 insertions(+), 377 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03fbb212db6cc..b50c6e97e05e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -844,15 +844,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" -[[package]] -name = "jsonc-parser" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b56a20e76235284255a09fcd1f45cf55d3c524ea657ebd3854735925c57743d" -dependencies = [ - "serde_json", -] - [[package]] name = "kqueue" version = "1.0.7" @@ -1171,13 +1162,10 @@ dependencies = [ "ignore", "ignore-files", "itertools", - "jsonc-parser", "napi", "napi-build", "napi-derive", "rayon", - "serde", - "serde_json", "thiserror", "tokio", "tracing", @@ -1447,12 +1435,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "ryu" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" - [[package]] name = "same-file" version = "1.0.6" @@ -1479,31 +1461,6 @@ name = "serde" version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", -] - -[[package]] -name = "serde_json" -version = "1.0.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" -dependencies = [ - "itoa", - "ryu", - "serde", -] [[package]] name = "sha1_smol" diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index 37d4445d2e912..2443f8823382c 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -13,12 +13,9 @@ hashbrown = { version = "0.14.0", features = ["rayon"] } ignore = '0.4' ignore-files = "1.3.0" itertools = "0.10.5" -jsonc-parser = { version = "0.21.1", features = ["serde"] } napi = { version = '2.12.6', default-features = false, features = ['anyhow', 'napi4', 'tokio_rt'] } napi-derive = '2.9.3' rayon = "1.7.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" thiserror = "1.0.40" tokio = { version = "1.28.2", features = ["fs"] } tracing = "0.1.37" diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts index 40f4ab823a2d6..acb529bd28e3d 100644 --- a/packages/nx/src/config/workspaces.ts +++ b/packages/nx/src/config/workspaces.ts @@ -95,7 +95,7 @@ export class Workspaces { return this.cachedProjectsConfig; } const nxJson = this.readNxJson(); - const projectsConfigurations = buildProjectsConfigurationsFromProjectPaths( + let projectsConfigurations = buildProjectsConfigurationsFromProjectPaths( nxJson, globForProjectFiles( this.root, @@ -116,15 +116,18 @@ export class Workspaces { opts?._includeProjectsFromAngularJson ) ) { - projectsConfigurations.projects = mergeAngularJsonAndProjects( - projectsConfigurations.projects, + projectsConfigurations = mergeAngularJsonAndProjects( + projectsConfigurations, this.root ); } - this.cachedProjectsConfig = this.mergeTargetDefaultsIntoProjectDescriptions( - projectsConfigurations, - nxJson - ); + this.cachedProjectsConfig = { + version: 2, + projects: this.mergeTargetDefaultsIntoProjectDescriptions( + projectsConfigurations, + nxJson + ), + }; return this.cachedProjectsConfig; } @@ -140,10 +143,10 @@ export class Workspaces { } private mergeTargetDefaultsIntoProjectDescriptions( - config: ProjectsConfigurations, + projects: Record, nxJson: NxJsonConfiguration ) { - for (const proj of Object.values(config.projects)) { + for (const proj of Object.values(projects)) { if (proj.targets) { for (const targetName of Object.keys(proj.targets)) { const projectTargetDefinition = proj.targets[targetName]; @@ -163,7 +166,7 @@ export class Workspaces { } } } - return config; + return projects; } isNxExecutor(nodeModule: string, executor: string) { @@ -808,7 +811,7 @@ export function buildProjectsConfigurationsFromProjectPaths( projectFiles: string[], // making this parameter allows devkit to pick up newly created projects readJson: (string) => T = (string) => readJsonFile(string) // making this an arg allows us to reuse in devkit -): ProjectsConfigurations { +): Record { const projects: Record = {}; for (const file of projectFiles) { @@ -868,10 +871,7 @@ export function buildProjectsConfigurationsFromProjectPaths( } } - return { - version: 2, - projects: projects, - }; + return projects; } export function mergeTargetConfigurations( diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 841fb0d860192..6a3e5b7b4113e 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -198,7 +198,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { nxJson, projectFiles, (file) => readJson(tree, file) - ).projects; + ); } /** diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index a8f0c14fdc5c4..d5445ab9341d4 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -37,14 +37,13 @@ export const enum WorkspaceErrors { Generic = 'Generic' } /** Get workspace config files based on provided globs */ -export function getConfigFiles(workspaceRoot: string, globs: Array): Array +export function getProjectConfigurations(workspaceRoot: string, globs: Array, parseConfigurations: (arg0: Array) => Record): Record export interface NxWorkspaceFiles { projectFileMap: Record> globalFiles: Array - configFiles: Array + projectConfigurations: Record } -/** Throws exceptions */ -export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array): NxWorkspaceFiles +export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array, parseConfigurations: (arg0: Array) => Record): NxWorkspaceFiles export class Watcher { origin: string /** diff --git a/packages/nx/src/native/index.js b/packages/nx/src/native/index.js index 88e15863f1ddd..a1c22704faa12 100644 --- a/packages/nx/src/native/index.js +++ b/packages/nx/src/native/index.js @@ -246,7 +246,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, EventType, Watcher, WorkspaceErrors, getConfigFiles, getWorkspaceFilesNative } = nativeBinding +const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, EventType, Watcher, WorkspaceErrors, getProjectConfigurations, getWorkspaceFilesNative } = nativeBinding module.exports.expandOutputs = expandOutputs module.exports.remove = remove @@ -258,5 +258,5 @@ module.exports.hashFilesMatchingGlobs = hashFilesMatchingGlobs module.exports.EventType = EventType module.exports.Watcher = Watcher module.exports.WorkspaceErrors = WorkspaceErrors -module.exports.getConfigFiles = getConfigFiles +module.exports.getProjectConfigurations = getProjectConfigurations module.exports.getWorkspaceFilesNative = getWorkspaceFilesNative diff --git a/packages/nx/src/native/tests/workspace_files.spec.ts b/packages/nx/src/native/tests/workspace_files.spec.ts index 50f8cf3c1565e..9e067a911bb5c 100644 --- a/packages/nx/src/native/tests/workspace_files.spec.ts +++ b/packages/nx/src/native/tests/workspace_files.spec.ts @@ -1,12 +1,24 @@ -import { - getConfigFiles, - getWorkspaceFilesNative, - WorkspaceErrors, -} from '../index'; +import { getProjectConfigurations, getWorkspaceFilesNative } from '../index'; import { TempFs } from '../../utils/testing/temp-fs'; import { NxJsonConfiguration } from '../../config/nx-json'; +import { dirname, join } from 'path'; +import { readJsonFile } from '../../utils/fileutils'; describe('workspace files', () => { + function createParseConfigurationsFunction(tempDir: string) { + return (filenames: string[]) => { + const res = {}; + for (const filename of filenames) { + const json = readJsonFile(join(tempDir, filename)); + res[json.name] = { + ...json, + root: dirname(filename), + }; + } + return res; + }; + } + it('should gather workspace file information', async () => { const fs = new TempFs('workspace-files'); const nxJson: NxJsonConfiguration = {}; @@ -41,12 +53,16 @@ describe('workspace files', () => { }); let globs = ['project.json', '**/project.json', 'libs/*/package.json']; - let { projectFileMap, configFiles, globalFiles } = getWorkspaceFilesNative( - fs.tempDir, - globs - ); + let { projectFileMap, projectConfigurations, globalFiles } = + getWorkspaceFilesNative( + fs.tempDir, + globs, + createParseConfigurationsFunction(fs.tempDir) + ); - let sortedConfigs = configFiles.sort(); + let sortedConfigs = Object.values(projectConfigurations).sort((a, b) => + a['name'].localeCompare(b['name']) + ); expect(projectFileMap).toMatchInlineSnapshot(` { @@ -104,11 +120,26 @@ describe('workspace files', () => { `); expect(sortedConfigs).toMatchInlineSnapshot(` [ - "libs/nested/project/project.json", - "libs/package-project/package.json", - "libs/project1/project.json", - "libs/project2/project.json", - "libs/project3/project.json", + { + "name": "nested-project", + "root": "libs/nested/project", + }, + { + "name": "package-project", + "root": "libs/package-project", + }, + { + "name": "project1", + "root": "libs/project1", + }, + { + "name": "project2", + "root": "libs/project2", + }, + { + "name": "project3", + "root": "libs/project3", + }, ] `); expect(globalFiles).toMatchInlineSnapshot(` @@ -148,7 +179,8 @@ describe('workspace files', () => { const globs = ['project.json', '**/project.json', '**/package.json']; const { globalFiles, projectFileMap } = getWorkspaceFilesNative( fs.tempDir, - globs + globs, + createParseConfigurationsFunction(fs.tempDir) ); expect(globalFiles).toEqual([]); @@ -201,140 +233,117 @@ describe('workspace files', () => { }); let globs = ['project.json', '**/project.json', '**/package.json']; - let { configFiles } = getWorkspaceFilesNative(fs.tempDir, globs); - - configFiles = configFiles.sort(); - - expect(configFiles).toMatchInlineSnapshot(` - [ - "libs/project1/project.json", - "project.json", - ] - `); - let configFiles2 = getConfigFiles(fs.tempDir, globs).sort(); - expect(configFiles2).toMatchInlineSnapshot(` - [ - "libs/project1/project.json", - "project.json", - ] + let projectConfigurations = getProjectConfigurations( + fs.tempDir, + globs, + (filenames) => { + const res = {}; + for (const filename of filenames) { + const json = readJsonFile(join(fs.tempDir, filename)); + res[json.name] = { + ...json, + root: dirname(filename), + }; + } + return res; + } + ); + expect(projectConfigurations).toMatchInlineSnapshot(` + { + "project1": { + "name": "project1", + "root": "libs/project1", + }, + "repo-name": { + "name": "repo-name", + "root": ".", + }, + } `); }); - describe('errors', () => { - it('it should infer names of configuration files without a name', async () => { - const fs = new TempFs('workspace-files'); - const nxJson: NxJsonConfiguration = {}; - await fs.createFiles({ - './nx.json': JSON.stringify(nxJson), - './package.json': JSON.stringify({ - name: 'repo-name', - version: '0.0.0', - dependencies: {}, - }), - './libs/project1/project.json': JSON.stringify({ - name: 'project1', - }), - './libs/project1/index.js': '', - './libs/project2/project.json': JSON.stringify({}), - }); - - let globs = ['project.json', '**/project.json', 'libs/*/package.json']; - expect(getWorkspaceFilesNative(fs.tempDir, globs).projectFileMap) - .toMatchInlineSnapshot(` - { - "project1": [ - { - "file": "libs/project1/index.js", - "hash": "3244421341483603138", - }, - { - "file": "libs/project1/project.json", - "hash": "13466615737813422520", - }, - ], - "project2": [ - { - "file": "libs/project2/project.json", - "hash": "1389868326933519382", - }, - ], - } - `); - }); - - it('handles comments', async () => { - const fs = new TempFs('workspace-files'); - const nxJson: NxJsonConfiguration = {}; - await fs.createFiles({ - './nx.json': JSON.stringify(nxJson), - './package.json': JSON.stringify({ - name: 'repo-name', - version: '0.0.0', - dependencies: {}, - }), - './libs/project1/project.json': `{ - "name": "temp" - // this should not fail - }`, - './libs/project1/index.js': '', - }); - - let globs = ['project.json', '**/project.json', 'libs/*/package.json']; - expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow(); - }); - - it('handles extra comma', async () => { - const fs = new TempFs('workspace-files'); - const nxJson: NxJsonConfiguration = {}; - await fs.createFiles({ - './nx.json': JSON.stringify(nxJson), - './package.json': JSON.stringify({ - name: 'repo-name', - version: '0.0.0', - dependencies: {}, - }), - './libs/project1/project.json': `{ - "name": "temp", - }`, - './libs/project1/index.js': '', - }); - - let globs = ['**/project.json']; - expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow(); - }); - - it('throws parsing errors: missing brackets', async () => { - const fs = new TempFs('workspace-files'); - const nxJson: NxJsonConfiguration = {}; - await fs.createFiles({ - './nx.json': JSON.stringify(nxJson), - './package.json': JSON.stringify({ - name: 'repo-name', - version: '0.0.0', - dependencies: {}, - }), - './libs/project1/project.json': `{ - "name": "temp", "property": "child": 2 } - }`, - './libs/project1/index.js': '', - }); - - let globs = ['**/project.json']; - - const error = getError(() => getWorkspaceFilesNative(fs.tempDir, globs)); - expect(error.message).toMatchInlineSnapshot( - `"libs/project1/project.json"` - ); - expect(error).toHaveProperty('code', WorkspaceErrors.ParseError); - }); - }); + // describe('errors', () => { + // it('it should infer names of configuration files without a name', async () => { + // const fs = new TempFs('workspace-files'); + // const nxJson: NxJsonConfiguration = {}; + // await fs.createFiles({ + // './nx.json': JSON.stringify(nxJson), + // './package.json': JSON.stringify({ + // name: 'repo-name', + // version: '0.0.0', + // dependencies: {}, + // }), + // './libs/project1/project.json': JSON.stringify({ + // name: 'project1', + // }), + // './libs/project1/index.js': '', + // './libs/project2/project.json': JSON.stringify({}), + // }); + // + // let globs = ['project.json', '**/project.json', 'libs/*/package.json']; + // expect(getWorkspaceFilesNative(fs.tempDir, globs).projectFileMap) + // .toMatchInlineSnapshot(` + // { + // "libs/project1": [ + // { + // "file": "libs/project1/index.js", + // "hash": "3244421341483603138", + // }, + // { + // "file": "libs/project1/project.json", + // "hash": "13466615737813422520", + // }, + // ], + // "libs/project2": [ + // { + // "file": "libs/project2/project.json", + // "hash": "1389868326933519382", + // }, + // ], + // } + // `); + // }); + // + // it('handles comments', async () => { + // const fs = new TempFs('workspace-files'); + // const nxJson: NxJsonConfiguration = {}; + // await fs.createFiles({ + // './nx.json': JSON.stringify(nxJson), + // './package.json': JSON.stringify({ + // name: 'repo-name', + // version: '0.0.0', + // dependencies: {}, + // }), + // './libs/project1/project.json': `{ + // "name": "temp" + // // this should not fail + // }`, + // './libs/project1/index.js': '', + // }); + // + // let globs = ['project.json', '**/project.json', 'libs/*/package.json']; + // expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow(); + // }); + // + // it('handles extra comma', async () => { + // const fs = new TempFs('workspace-files'); + // const nxJson: NxJsonConfiguration = {}; + // await fs.createFiles({ + // './nx.json': JSON.stringify(nxJson), + // './package.json': JSON.stringify({ + // name: 'repo-name', + // version: '0.0.0', + // dependencies: {}, + // }), + // './libs/project1/project.json': `{ + // "name": "temp", + // }`, + // './libs/project1/index.js': '', + // }); + // + // let globs = ['**/project.json']; + // expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow(); + // }); + // }); }); - -const getError = (fn: () => unknown): Error => { - try { - fn(); - } catch (error: unknown) { - return error as Error; - } -}; diff --git a/packages/nx/src/native/utils/path.rs b/packages/nx/src/native/utils/path.rs index eb9edc0e4eea2..15fc00727f596 100644 --- a/packages/nx/src/native/utils/path.rs +++ b/packages/nx/src/native/utils/path.rs @@ -20,6 +20,10 @@ fn normalize_path

(path: P) -> String where P: AsRef, { + if path.as_ref() == Path::new("") { + return ".".into(); + } + // convert back-slashes in Windows paths, since the js expects only forward-slash path separators if cfg!(windows) { path.as_ref().display().to_string().replace('\\', "/") diff --git a/packages/nx/src/native/workspace/get_config_files.rs b/packages/nx/src/native/workspace/get_config_files.rs index 0050611acf8f6..518cc3b60827f 100644 --- a/packages/nx/src/native/workspace/get_config_files.rs +++ b/packages/nx/src/native/workspace/get_config_files.rs @@ -2,29 +2,42 @@ use crate::native::utils::glob::build_glob_set; use crate::native::utils::path::Normalize; use crate::native::walker::nx_walker; use globset::GlobSet; + +use napi::JsObject; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::path::{Path, PathBuf}; #[napi] /// Get workspace config files based on provided globs -pub fn get_config_files(workspace_root: String, globs: Vec) -> anyhow::Result> { +pub fn get_project_configurations( + workspace_root: String, + globs: Vec, + + parse_configurations: ConfigurationParser, +) -> napi::Result> +where + ConfigurationParser: Fn(Vec) -> napi::Result>, +{ let globs = build_glob_set(globs)?; - Ok(nx_walker(workspace_root, move |rec| { - let mut config_paths: HashMap)> = HashMap::new(); - for (path, content) in rec { - insert_config_file_into_map((path, content), &mut config_paths, &globs); + let config_paths: Vec = nx_walker(workspace_root, move |rec| { + let mut config_paths: HashMap = HashMap::new(); + for (path, _) in rec { + insert_config_file_into_map(path, &mut config_paths, &globs); } + config_paths - .into_iter() - .map(|(_, (val, _))| val.to_normalized_string()) + .into_values() + .map(|p| p.to_normalized_string()) .collect() - })) + }); + + parse_configurations(config_paths) } pub fn insert_config_file_into_map( - (path, content): (PathBuf, Vec), - config_paths: &mut HashMap)>, + path: PathBuf, + config_paths: &mut HashMap, globs: &GlobSet, ) { if globs.is_match(&path) { @@ -34,25 +47,24 @@ pub fn insert_config_file_into_map( .file_name() .expect("Config paths always have file names"); if file_name == "project.json" { - config_paths.insert(parent, (path, content)); + config_paths.insert(parent, path); } else if file_name == "package.json" { match config_paths.entry(parent) { Entry::Occupied(mut o) => { if o.get() - .0 .file_name() .expect("Config paths always have file names") != "project.json" { - o.insert((path, content)); + o.insert(path); } } Entry::Vacant(v) => { - v.insert((path, content)); + v.insert(path); } } } else { - config_paths.entry(parent).or_insert((path, content)); + config_paths.entry(parent).or_insert(path); } } } @@ -65,34 +77,23 @@ mod test { #[test] fn should_insert_config_files_properly() { - let mut config_paths: HashMap)> = HashMap::new(); + let mut config_paths: HashMap = HashMap::new(); let globs = build_glob_set(vec!["**/*".into()]).unwrap(); + insert_config_file_into_map(PathBuf::from("project.json"), &mut config_paths, &globs); + insert_config_file_into_map(PathBuf::from("package.json"), &mut config_paths, &globs); insert_config_file_into_map( - (PathBuf::from("project.json"), vec![]), - &mut config_paths, - &globs, - ); - insert_config_file_into_map( - (PathBuf::from("package.json"), vec![]), - &mut config_paths, - &globs, - ); - insert_config_file_into_map( - (PathBuf::from("lib1/project.json"), vec![]), + PathBuf::from("lib1/project.json"), &mut config_paths, &globs, ); insert_config_file_into_map( - (PathBuf::from("lib2/package.json"), vec![]), + PathBuf::from("lib2/package.json"), &mut config_paths, &globs, ); - let config_files: Vec = config_paths - .into_iter() - .map(|(_, (path, _))| path) - .collect(); + let config_files: Vec = config_paths.into_values().collect(); assert!(config_files.contains(&PathBuf::from("project.json"))); assert!(config_files.contains(&PathBuf::from("lib1/project.json"))); diff --git a/packages/nx/src/native/workspace/get_nx_workspace_files.rs b/packages/nx/src/native/workspace/get_nx_workspace_files.rs index b8c6e68b48890..68973833035ad 100644 --- a/packages/nx/src/native/workspace/get_nx_workspace_files.rs +++ b/packages/nx/src/native/workspace/get_nx_workspace_files.rs @@ -1,5 +1,7 @@ -use jsonc_parser::ParseOptions; -use std::collections::HashMap; +use itertools::Itertools; +use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode}; +use napi::{JsFunction, JsObject, JsUnknown, Status}; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use rayon::prelude::*; @@ -13,21 +15,24 @@ use crate::native::utils::path::Normalize; use crate::native::walker::nx_walker; use crate::native::workspace::errors::{InternalWorkspaceErrors, WorkspaceErrors}; use crate::native::workspace::get_config_files::insert_config_file_into_map; -use crate::native::workspace::types::{FileLocation, ProjectConfiguration}; +use crate::native::workspace::types::FileLocation; #[napi(object)] pub struct NxWorkspaceFiles { pub project_file_map: HashMap>, pub global_files: Vec, - pub config_files: Vec, + pub project_configurations: HashMap, } #[napi] -/// Throws exceptions -pub fn get_workspace_files_native( +pub fn get_workspace_files_native( workspace_root: String, globs: Vec, -) -> napi::Result { + parse_configurations: ConfigurationParser, +) -> napi::Result +where + ConfigurationParser: Fn(Vec) -> napi::Result>, +{ enable_logger(); trace!("{workspace_root}, {globs:?}"); @@ -35,7 +40,12 @@ pub fn get_workspace_files_native( let (projects, mut file_data) = get_file_data(&workspace_root, globs) .map_err(|err| napi::Error::new(WorkspaceErrors::Generic, err.to_string()))?; - let root_map = create_root_map(&projects)?; + let projects_vec: Vec = projects.iter().map(|p| p.to_normalized_string()).collect(); + + let project_configurations = parse_configurations(projects_vec) + .map_err(|e| napi::Error::new(WorkspaceErrors::ParseError, e.to_string()))?; + + let root_map = create_root_map(&project_configurations); trace!(?root_map); @@ -46,14 +56,14 @@ pub fn get_workspace_files_native( .into_par_iter() .map(|file_data| { let file_path = Path::new(&file_data.file); - let mut parent = file_path.parent().unwrap_or_else(|| Path::new("")); + let mut parent = file_path.parent().unwrap_or_else(|| Path::new(".")); - while root_map.get(parent).is_none() && parent != Path::new("") { - parent = parent.parent().unwrap_or_else(|| Path::new("")); + while root_map.get(parent).is_none() && parent != Path::new(".") { + parent = parent.parent().unwrap_or_else(|| Path::new(".")); } match root_map.get(parent) { - Some(project_name) => (FileLocation::Project(project_name.clone()), file_data), + Some(project_name) => (FileLocation::Project(project_name.into()), file_data), None => (FileLocation::Global, file_data), } }) @@ -76,7 +86,7 @@ pub fn get_workspace_files_native( FileLocation::Global => global_files.push(file_data), FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) { None => { - project_file_map.insert(project_name, vec![file_data]); + project_file_map.insert(project_name.clone(), vec![file_data]); } Some(project_files) => project_files.push(file_data), }, @@ -86,98 +96,34 @@ pub fn get_workspace_files_native( Ok(NxWorkspaceFiles { project_file_map, global_files, - config_files: projects - .keys() - .map(|path| path.to_normalized_string()) - .collect(), + project_configurations, }) } fn create_root_map( - projects: &HashMap>, -) -> Result, InternalWorkspaceErrors> { - projects - .par_iter() - .map(|(path, content)| { - let file_name = path - .file_name() - .expect("path should always have a filename"); - return if file_name == "project.json" || file_name == "package.json" { - // use serde_json to do the initial parse, if that fails fall back to jsonc_parser. - // If all those fail, expose the error from jsonc_parser - let project_configuration: ProjectConfiguration = - read_project_configuration(content, path)?; - - let Some(parent_path) = path.parent() else { - return Err(InternalWorkspaceErrors::Generic { - msg: format!("{path:?} has no parent"), - }) - }; - - let name: String = if let Some(name) = project_configuration.name { - Ok(name) - } else { - parent_path - .file_name() - .unwrap_or_default() - .to_os_string() - .into_string() - .map_err(|os_string| InternalWorkspaceErrors::Generic { - msg: format!("Cannot turn {os_string:?} into String"), - }) - }?; - Ok((parent_path, name)) - } else if let Some(parent_path) = path.parent() { - Ok(( - parent_path, - parent_path - .file_name() - .unwrap_or_default() - .to_os_string() - .into_string() - .map_err(|os_string| InternalWorkspaceErrors::Generic { - msg: format!("Cannot turn {os_string:?} into String"), - })?, - )) - } else { - Err(InternalWorkspaceErrors::Generic { - msg: format!("{path:?} has no parent"), - }) - }; + project_configurations: &HashMap, +) -> hashbrown::HashMap { + project_configurations + .iter() + .map(|(project_name, project_configuration)| { + let root: String = project_configuration.get("root").unwrap().unwrap(); + (PathBuf::from(root), project_name.clone()) }) .collect() } -fn read_project_configuration( - content: &[u8], - path: &Path, -) -> Result { - serde_json::from_slice(content).or_else(|_| { - let content_str = std::str::from_utf8(content).expect("content should be valid utf8"); - let parser_value = - jsonc_parser::parse_to_serde_value(content_str, &ParseOptions::default()).map_err( - |_| InternalWorkspaceErrors::ParseError { - file: PathBuf::from(path), - }, - )?; - serde_json::from_value(parser_value.into()).map_err(|_| InternalWorkspaceErrors::Generic { - msg: format!("Failed to parse {path:?}"), - }) - }) -} - -type WorkspaceData = (HashMap>, Vec); +type WorkspaceData = (HashSet, Vec); fn get_file_data(workspace_root: &str, globs: Vec) -> anyhow::Result { let globs = build_glob_set(globs)?; let (projects, file_data) = nx_walker(workspace_root, move |rec| { - let mut projects: HashMap)> = HashMap::new(); + let mut projects: HashMap = HashMap::new(); let mut file_hashes: Vec = vec![]; for (path, content) in rec { file_hashes.push(FileData { file: path.to_normalized_string(), hash: xxh3::xxh3_64(&content).to_string(), }); - insert_config_file_into_map((path, content), &mut projects, &globs) + insert_config_file_into_map(path, &mut projects, &globs) } (projects, file_hashes) }); diff --git a/packages/nx/src/native/workspace/types.rs b/packages/nx/src/native/workspace/types.rs index 83318cfac7df2..dc460dabcec66 100644 --- a/packages/nx/src/native/workspace/types.rs +++ b/packages/nx/src/native/workspace/types.rs @@ -1,10 +1,3 @@ -use serde::Deserialize; - -#[derive(Debug, Deserialize)] -pub(crate) struct ProjectConfiguration { - pub name: Option, -} - #[derive(Debug, Eq, PartialEq)] pub enum FileLocation { Global, diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 4a114ba70eef5..57b549e453037 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -9,14 +9,17 @@ import { import { getNxRequirePaths } from '../../utils/installation-directory'; import { readJsonFile } from '../../utils/fileutils'; import { join } from 'path'; -import { ProjectsConfigurations } from '../../config/workspace-json-project-json'; +import { + ProjectConfiguration, + ProjectsConfigurations, +} from '../../config/workspace-json-project-json'; import { mergeAngularJsonAndProjects, shouldMergeAngularProjects, } from '../../adapter/angular-json'; import { NxJsonConfiguration } from '../../config/nx-json'; import { FileData, ProjectFileMap } from '../../config/project-graph'; -import { NxWorkspaceFiles, WorkspaceErrors } from '../../native'; +import type { NxWorkspaceFiles } from '../../native'; /** * Walks the workspace directory to create the `projectFileMap`, `ProjectConfigurations` and `allWorkspaceFiles` @@ -40,19 +43,21 @@ export async function retrieveWorkspaceFiles( ); performance.mark('get-workspace-files:start'); - let workspaceFiles: NxWorkspaceFiles; - try { - workspaceFiles = getWorkspaceFilesNative(workspaceRoot, globs); - } catch (e) { - // If the error is a parse error from Rust, then use the JS readJsonFile function to write a pretty error message - if (e.code === WorkspaceErrors.ParseError) { - readJsonFile(join(workspaceRoot, e.message)); - // readJsonFile should always fail, but if it doesn't, then throw the original error - throw e; - } else { - throw e; - } - } + + const { projectConfigurations, projectFileMap, globalFiles } = + getWorkspaceFilesNative( + workspaceRoot, + globs, + (configs: string[]): Record => { + const projectConfigurations = createProjectConfigurations( + workspaceRoot, + nxJson, + configs + ); + + return projectConfigurations.projects; + } + ) as NxWorkspaceFiles; performance.mark('get-workspace-files:end'); performance.measure( 'get-workspace-files', @@ -61,16 +66,12 @@ export async function retrieveWorkspaceFiles( ); return { - allWorkspaceFiles: buildAllWorkspaceFiles( - workspaceFiles.projectFileMap, - workspaceFiles.globalFiles - ), - projectFileMap: workspaceFiles.projectFileMap, - projectConfigurations: createProjectConfigurations( - workspaceRoot, - nxJson, - workspaceFiles.configFiles - ), + allWorkspaceFiles: buildAllWorkspaceFiles(projectFileMap, globalFiles), + projectFileMap, + projectConfigurations: { + version: 2, + projects: projectConfigurations, + } as ProjectsConfigurations, }; } @@ -84,10 +85,21 @@ export async function retrieveProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration ) { - const { getConfigFiles } = require('../../native'); + const { getProjectConfigurations } = require('../../native'); const globs = await configurationGlobs(workspaceRoot, nxJson); - const configPaths = getConfigFiles(workspaceRoot, globs); - return createProjectConfigurations(workspaceRoot, nxJson, configPaths); + return getProjectConfigurations( + workspaceRoot, + globs, + (configs: string[]): Record => { + const projectConfigurations = createProjectConfigurations( + workspaceRoot, + nxJson, + configs + ); + + return projectConfigurations.projects; + } + ); } function buildAllWorkspaceFiles( @@ -123,8 +135,8 @@ function createProjectConfigurations( ); if (shouldMergeAngularProjects(workspaceRoot, false)) { - projectConfigurations.projects = mergeAngularJsonAndProjects( - projectConfigurations.projects, + projectConfigurations = mergeAngularJsonAndProjects( + projectConfigurations, workspaceRoot ); } @@ -135,14 +147,17 @@ function createProjectConfigurations( 'build-project-configs:end' ); - return projectConfigurations; + return { + version: 2, + projects: projectConfigurations, + }; } function mergeTargetDefaultsIntoProjectDescriptions( - config: ProjectsConfigurations, + projects: Record, nxJson: NxJsonConfiguration ) { - for (const proj of Object.values(config.projects)) { + for (const proj of Object.values(projects)) { if (proj.targets) { for (const targetName of Object.keys(proj.targets)) { const projectTargetDefinition = proj.targets[targetName]; @@ -162,7 +177,7 @@ function mergeTargetDefaultsIntoProjectDescriptions( } } } - return config; + return projects; } async function configurationGlobs(