Skip to content

Commit

Permalink
feat(js): remove ts implementation of ts processing (#18752)
Browse files Browse the repository at this point in the history
  • Loading branch information
@FrozenPandaz
FrozenPandaz authored Aug 23, 2023
1 parent abc147c commit 165250e
Show file tree
Hide file tree
Showing 7 changed files with 228 additions and 837 deletions.
185 changes: 35 additions & 150 deletions packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import {
DependencyType,
ProjectFileMap,
ProjectGraphProcessorContext,
} from '../../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
import { join } from 'path';
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
import * as os from 'os';
import { ExplicitDependency } from './explicit-project-dependencies';
import {
buildExplicitTypeScriptDependencies,
ExplicitDependency,
} from './explicit-project-dependencies';
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';

export function buildExplicitDependencies(
jsPluginConfig: {
Expand All @@ -17,31 +17,40 @@ export function buildExplicitDependencies(
ctx: ProjectGraphProcessorContext,
builder: ProjectGraphBuilder
) {
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
if (totalNumOfFilesToProcess === 0) return;
// using workers has an overhead, so we only do it when the number of
// files we need to process is >= 100 and there are more than 2 CPUs
// to be able to use at least 2 workers (1 worker per CPU and
// 1 CPU for the main thread)
if (totalNumberOfFilesToProcess(ctx) === 0) return;

let dependencies: ExplicitDependency[] = [];

if (
process.env.NX_NATIVE_TS_DEPS !== 'false' ||
jsPluginConfig.analyzeSourceFiles === false ||
totalNumOfFilesToProcess < 100 ||
getNumberOfWorkers() <= 2
jsPluginConfig.analyzeSourceFiles === undefined ||
jsPluginConfig.analyzeSourceFiles === true
) {
return buildExplicitDependenciesWithoutWorkers(
jsPluginConfig,
ctx,
builder
);
} else {
return buildExplicitDependenciesUsingWorkers(
jsPluginConfig,
ctx,
totalNumOfFilesToProcess,
builder
let tsExists = false;
try {
require.resolve('typescript');
tsExists = true;
} catch {}
if (tsExists) {
dependencies = dependencies.concat(
buildExplicitTypeScriptDependencies(builder.graph, ctx.filesToProcess)
);
}
}
if (
jsPluginConfig.analyzePackageJson === undefined ||
jsPluginConfig.analyzePackageJson === true
) {
dependencies = dependencies.concat(
buildExplicitPackageJsonDependencies(
ctx.nxJsonConfiguration,
ctx.projectsConfigurations,
builder.graph,
ctx.filesToProcess
)
);
}

dependencies.forEach((r) => addDependency(builder, r));
}

function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
Expand All @@ -51,50 +60,6 @@ function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
);
return totalNumOfFilesToProcess;
}

function splitFilesIntoBins(
ctx: ProjectGraphProcessorContext,
totalNumOfFilesToProcess: number,
numberOfWorkers: number
) {
// we want to have numberOfWorkers * 5 bins
const filesPerBin =
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
const bins: ProjectFileMap[] = [];
let currentProjectFileMap = {};
let currentNumberOfFiles = 0;
for (const source of Object.keys(ctx.filesToProcess)) {
for (const f of Object.values(ctx.filesToProcess[source])) {
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
currentProjectFileMap[source].push(f);
currentNumberOfFiles++;

if (currentNumberOfFiles >= filesPerBin) {
bins.push(currentProjectFileMap);
currentProjectFileMap = {};
currentNumberOfFiles = 0;
}
}
}
bins.push(currentProjectFileMap);
return bins;
}

function createWorkerPool(numberOfWorkers: number) {
const res = [];
for (let i = 0; i < numberOfWorkers; ++i) {
res.push(
new (require('worker_threads').Worker)(
join(__dirname, './project-graph-worker.js'),
{
env: process.env,
}
)
);
}
return res;
}

function addDependency(
builder: ProjectGraphBuilder,
dependency: ExplicitDependency
Expand All @@ -113,83 +78,3 @@ function addDependency(
);
}
}

function buildExplicitDependenciesWithoutWorkers(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
builder: ProjectGraphBuilder
) {
buildExplicitTypescriptAndPackageJsonDependencies(
jsPluginConfig,
ctx.nxJsonConfiguration,
ctx.projectsConfigurations,
builder.graph,
ctx.filesToProcess
).forEach((r) => addDependency(builder, r));
}

function buildExplicitDependenciesUsingWorkers(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
totalNumOfFilesToProcess: number,
builder: ProjectGraphBuilder
) {
const numberOfWorkers = Math.min(
totalNumOfFilesToProcess,
getNumberOfWorkers()
);
const bins = splitFilesIntoBins(
ctx,
totalNumOfFilesToProcess,
numberOfWorkers
);
const workers = createWorkerPool(numberOfWorkers);
let numberOfExpectedResponses = bins.length;

return new Promise((res, reject) => {
for (let w of workers) {
w.on('message', (explicitDependencies) => {
explicitDependencies.forEach((r) => addDependency(builder, r));
if (bins.length > 0) {
w.postMessage({ filesToProcess: bins.shift() });
}
// we processed all the bins
if (--numberOfExpectedResponses === 0) {
for (let w of workers) {
w.terminate();
}
res(null);
}
});
w.on('error', reject);
w.on('exit', (code) => {
if (code !== 0) {
reject(
new Error(
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
)
);
}
});
w.postMessage({
nxJsonConfiguration: ctx.nxJsonConfiguration,
projectsConfigurations: ctx.projectsConfigurations,
projectGraph: builder.graph,
jsPluginConfig,
});
w.postMessage({ filesToProcess: bins.shift() });
}
});
}

function getNumberOfWorkers(): number {
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
}
51 changes: 0 additions & 51 deletions ...oject-graph/build-dependencies/build-explicit-typescript-and-package-json-dependencies.ts
View file

This file was deleted.

92 changes: 1 addition & 91 deletions packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { TypeScriptImportLocator } from './typescript-import-locator';
import { TargetProjectLocator } from './target-project-locator';
import {
DependencyType,
Expand All @@ -20,44 +19,7 @@ export function buildExplicitTypeScriptDependencies(
graph: ProjectGraph,
filesToProcess: ProjectFileMap
): ExplicitDependency[] {
let results: ExplicitDependency[];
if (process.env.NX_NATIVE_TS_DEPS !== 'false') {
results = buildExplicitTypeScriptDependenciesWithSwc(filesToProcess, graph);
} else {
results = buildExplicitTypeScriptDependenciesWithTs(filesToProcess, graph);
}
if (
process.env.NX_NATIVE_TS_DEPS &&
process.env.NX_NATIVE_TS_DEPS === 'debug'
) {
const tsResults = buildExplicitTypeScriptDependenciesWithTs(
filesToProcess,
graph
);

const set = new Set<string>();

for (const dep of results) {
set.add(
`+ ${dep.sourceProjectName} -> ${dep.targetProjectName} (${dep.sourceProjectFile})`
);
}
for (const dep of tsResults) {
set.delete(
`+ ${dep.sourceProjectName} -> ${dep.targetProjectName} (${dep.sourceProjectFile})`
);
set.add(
`- ${dep.sourceProjectName} -> ${dep.targetProjectName} (${dep.sourceProjectFile})`
);
}
for (const dep of results) {
set.delete(
`- ${dep.sourceProjectName} -> ${dep.targetProjectName} (${dep.sourceProjectFile})`
);
}
set.forEach((s) => console.log(s));
}
return results;
return buildExplicitTypeScriptDependenciesWithSwc(filesToProcess, graph);
}

function isRoot(graph: ProjectGraph, projectName: string): boolean {
Expand Down Expand Up @@ -161,55 +123,3 @@ function buildExplicitTypeScriptDependenciesWithSwc(

return res;
}

function buildExplicitTypeScriptDependenciesWithTs(
filesToProcess: ProjectFileMap,
graph: ProjectGraph
): ExplicitDependency[] {
const importLocator = new TypeScriptImportLocator();
const targetProjectLocator = new TargetProjectLocator(
graph.nodes as any,
graph.externalNodes
);
const res: ExplicitDependency[] = [];
Object.keys(filesToProcess).forEach((source) => {
Object.values(filesToProcess[source]).forEach((f) => {
importLocator.fromFile(
f.file,
(
importExpr: string,
filePath: string,
type: DependencyType.static | DependencyType.dynamic
) => {
const target = targetProjectLocator.findProjectWithImport(
importExpr,
f.file
);
let targetProjectName;
if (target) {
if (!isRoot(graph, source) && isRoot(graph, target)) {
// TODO: These edges technically should be allowed but we need to figure out how to separate config files out from root
return;
}

targetProjectName = target;
} else {
// treat all unknowns as npm packages, they can be eiher
// - mistyped local import, which has to be fixed manually
// - node internals, which should still be tracked as a dependency
// - npm packages, which are not yet installed but should be tracked
targetProjectName = `npm:${importExpr}`;
}

res.push({
sourceProjectName: source,
targetProjectName,
sourceProjectFile: f.file,
type,
});
}
);
});
});
return res;
}
Loading

1 comment on commit 165250e

@vercel
Copy link

@vercel vercel bot commented on 165250e Aug 23, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

nx-dev – ./

nx-dev-nrwl.vercel.app
nx.dev
nx-five.vercel.app
nx-dev-git-master-nrwl.vercel.app

Please sign in to comment.