Skip to content

Commit

Permalink
feat(core): add initial draft of v2 project inference
Browse files Browse the repository at this point in the history
  • Loading branch information
@AgentEnder
AgentEnder committed Jul 7, 2023
1 parent d4d48fb commit ceef985
Show file tree
Hide file tree
Showing 12 changed files with 247 additions and 132 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@
"@tailwindcss/line-clamp": "^0.4.2",
"@tailwindcss/typography": "^0.5.7",
"@types/license-checker": "^25.0.3",
"@types/minimatch": "^5.1.2",
"@yarnpkg/lockfile": "^1.1.0",
"@yarnpkg/parsers": "^3.0.0-rc.18",
"@zkochan/js-yaml": "0.0.6",
Expand Down Expand Up @@ -349,4 +350,3 @@
]
}
}

1 change: 1 addition & 0 deletions packages/nx/src/config/project-graph.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ export interface ProjectGraphProcessorContext {

/**
* A function that produces an updated ProjectGraph
* @deprecated(v18) Use `buildProjectDependencies` and `buildProjectNodes` instead.
*/
export type ProjectGraphProcessor = (
currentGraph: ProjectGraph,
Expand Down
23 changes: 17 additions & 6 deletions packages/nx/src/config/workspaces.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import { TargetConfiguration } from './workspace-json-project-json';

jest.mock('fs', () => require('memfs').fs);

const libConfig = (name) => ({
root: `libs/${name}`,
sourceRoot: `libs/${name}/src`,
const libConfig = (root, name?: string) => ({
name: name ?? toProjectName(`${root}/some-file`),
root: `libs/${root}`,
sourceRoot: `libs/${root}/src`,
});

const packageLibConfig = (root) => ({
const packageLibConfig = (root, name?: string) => ({
name: name ?? toProjectName(`${root}/some-file`),
root,
sourceRoot: root,
projectType: 'library',
Expand Down Expand Up @@ -66,13 +68,17 @@ describe('Workspaces', () => {

const workspaces = new Workspaces('/root');
const resolved = workspaces.readProjectsConfigurations();
console.log(resolved);
expect(resolved.projects.lib1).toEqual(standaloneConfig);
});

it('should build project configurations from glob', () => {
const lib1Config = libConfig('lib1');
const lib2Config = packageLibConfig('libs/lib2');
const domainPackageConfig = packageLibConfig('libs/domain/lib3');
const domainPackageConfig = packageLibConfig(
'libs/domain/lib3',
'domain-lib3'
);
const domainLibConfig = libConfig('domain/lib4');

vol.fromJSON(
Expand All @@ -94,10 +100,14 @@ describe('Workspaces', () => {

const workspaces = new Workspaces('/root');
const { projects } = workspaces.readProjectsConfigurations();
// projects got deduped so the workspace one remained
console.log(projects);

// projects got merged for lib1
expect(projects['lib1']).toEqual({
name: 'lib1',
root: 'libs/lib1',
sourceRoot: 'libs/lib1/src',
projectType: 'library',
});
expect(projects.lib2).toEqual(lib2Config);
expect(projects['domain-lib3']).toEqual(domainPackageConfig);
Expand Down Expand Up @@ -137,6 +147,7 @@ describe('Workspaces', () => {
const workspaces = new Workspaces('/root2');
const resolved = workspaces.readProjectsConfigurations();
expect(resolved.projects['my-package']).toEqual({
name: 'my-package',
root: 'packages/my-package',
sourceRoot: 'packages/my-package',
projectType: 'library',
Expand Down
216 changes: 123 additions & 93 deletions packages/nx/src/config/workspaces.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { sync as globSync } from 'fast-glob';
import { existsSync, readFileSync } from 'fs';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import * as path from 'path';
import { basename, dirname, extname, join } from 'path';
import { performance } from 'perf_hooks';
Expand All @@ -9,9 +9,11 @@ import { logger, NX_PREFIX } from '../utils/logger';
import {
loadNxPlugins,
loadNxPluginsSync,
NxPluginV2,
readPluginPackageJson,
registerPluginTSTranspiler,
} from '../utils/nx-plugin';
import minimatch = require('minimatch');

import type { NxJsonConfiguration, TargetDefaults } from './nx-json';
import {
Expand Down Expand Up @@ -108,6 +110,7 @@ export class Workspaces {
),
nxJson
),
this.root,
(path) => readJsonFile(join(this.root, path))
);
if (
Expand Down Expand Up @@ -538,19 +541,15 @@ export function getGlobPatternsFromPlugins(
paths: string[],
root = workspaceRoot
): string[] {
const plugins = loadNxPluginsSync(nxJson?.plugins, paths, root);
try {
const plugins = loadNxPluginsSync(nxJson?.plugins, paths, root);

const patterns = [];
for (const plugin of plugins) {
if (!plugin.projectFilePatterns) {
continue;
}
for (const filePattern of plugin.projectFilePatterns) {
patterns.push('**/' + filePattern);
}
return plugins.flatMap((plugin) =>
plugin.buildProjectNodes ? Object.keys(plugin.buildProjectNodes) : []
);
} catch (e) {
console.log(e);
}

return patterns;
}

export async function getGlobPatternsFromPluginsAsync(
Expand All @@ -560,17 +559,9 @@ export async function getGlobPatternsFromPluginsAsync(
): Promise<string[]> {
const plugins = await loadNxPlugins(nxJson?.plugins, paths, root);

const patterns = [];
for (const plugin of plugins) {
if (!plugin.projectFilePatterns) {
continue;
}
for (const filePattern of plugin.projectFilePatterns) {
patterns.push('**/' + filePattern);
}
}

return patterns;
return plugins.flatMap((plugin) =>
plugin.buildProjectNodes ? Object.keys(plugin.buildProjectNodes) : []
);
}

/**
Expand Down Expand Up @@ -681,7 +672,7 @@ export function globForProjectFiles(

projectGlobPatterns.push(...pluginsGlobPatterns);

const combinedProjectGlobPattern = '{' + projectGlobPatterns.join(',') + '}';
const combinedProjectGlobPattern = combineGlobPatterns(projectGlobPatterns);

performance.mark('start-glob-for-projects');
/**
Expand Down Expand Up @@ -716,7 +707,7 @@ export function globForProjectFiles(

const globResults = globSync(combinedProjectGlobPattern, opts);

projectGlobCache = deduplicateProjectFiles(globResults);
projectGlobCache = globResults;

// TODO @vsavkin remove after Nx 16
if (
Expand All @@ -739,23 +730,12 @@ export function globForProjectFiles(
return projectGlobCache;
}

/**
* @description Loops through files and reduces them to 1 file per project.
* @param files Array of files that may represent projects
*/
export function deduplicateProjectFiles(files: string[]): string[] {
const filtered = new Map();
files.forEach((file) => {
const projectFolder = dirname(file);
const projectFile = basename(file);
if (filtered.has(projectFolder) && projectFile !== 'project.json') return;
filtered.set(projectFolder, projectFile);
});

return Array.from(filtered.entries()).map(([folder, file]) =>
join(folder, file)
);
}
const combineGlobPatterns = (patterns: string[]) =>
patterns.length > 1
? '{' + patterns.join(',') + '}'
: patterns.length === 1
? patterns[0]
: '';

function buildProjectConfigurationFromPackageJson(
path: string,
Expand Down Expand Up @@ -784,6 +764,7 @@ function buildProjectConfigurationFromPackageJson(
directory.startsWith(nxJson.workspaceLayout.appsDir)
? 'application'
: 'library';

return {
root: directory,
sourceRoot: directory,
Expand All @@ -803,66 +784,115 @@ export function inferProjectFromNonStandardFile(
};
}

function mergeProjectConfigurationIntoWorkspace(
project: ProjectConfiguration,
existingProjects: Record<string, ProjectConfiguration>
): void {
const matchingProject =
existingProjects[project.name]?.root === project.root
? existingProjects[project.name]
: Object.values(existingProjects).find((c) => c.root === project.root);

if (!matchingProject) {
existingProjects[project.name] = project;
return;
}

// This handles top level properties that are overwritten. `srcRoot`, `projectType`, or fields that Nx doesn't know about.
const updatedProjectConfiguration = { ...matchingProject, ...project };

// The next blocks handle properties that should be themselves merged (e.g. targets, tags, and implicit dependencies)

if (project.tags && matchingProject.tags) {
updatedProjectConfiguration.tags = matchingProject.tags.concat(
project.tags
);
}

if (project.implicitDependencies && matchingProject.tags) {
updatedProjectConfiguration.implicitDependencies =
matchingProject.implicitDependencies.concat(project.implicitDependencies);
}

if (project.generators && matchingProject.generators) {
updatedProjectConfiguration.generators = {
...matchingProject.generators,
...project.generators,
};
}

if (project.targets && matchingProject.targets) {
updatedProjectConfiguration.targets = {
...matchingProject.targets,
...project.targets,
};
}

if (updatedProjectConfiguration.name !== matchingProject.name) {
delete existingProjects[matchingProject.name];
}
existingProjects[updatedProjectConfiguration.name] =
updatedProjectConfiguration;
}

export function buildProjectsConfigurationsFromProjectPaths(
nxJson: NxJsonConfiguration,
projectFiles: string[], // making this parameter allows devkit to pick up newly created projects
root: string = workspaceRoot,
readJson: <T extends Object>(string) => T = <T extends Object>(string) =>
readJsonFile<T>(string) // making this an arg allows us to reuse in devkit
): ProjectsConfigurations {
const projects: Record<string, ProjectConfiguration> = {};
// We go in reverse here s.t. plugins listed first in the plugins array have highest priority - they overwrite
// whatever configuration was added by plugins later in the array.
const plugins = loadNxPluginsSync(nxJson.plugins).reverse();

for (const file of projectFiles) {
const directory = dirname(file).split('\\').join('/');
const fileName = basename(file);

if (fileName === 'project.json') {
// Nx specific project configuration (`project.json` files) in the same
// directory as a package.json should overwrite the inferred package.json
// project configuration.
const configuration = readJson<ProjectConfiguration>(file);

configuration.root = directory;
// We push the nx core node builder onto the end, s.t. it overwrites any user specified behavior
const globPatternsFromPackageManagerWorkspaces =
getGlobPatternsFromPackageManagerWorkspaces(root);
plugins.push({
name: 'nx-core-build-nodes',
buildProjectNodes: {
// Load projects from pnpm / npm workspaces
...(globPatternsFromPackageManagerWorkspaces.length
? ({
[combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: (
pkgJsonPath
) => {
const json = readJson<PackageJson>(pkgJsonPath);
return {
[json.name]: buildProjectConfigurationFromPackageJson(
pkgJsonPath,
json,
nxJson
),
};
},
} as NxPluginV2['buildProjectNodes'])
: {}),
// Load projects from project.json files. These will be read second, since
// they are listed last in the plugin, so they will overwrite things from the package.json
// based projects.
'{project.json,**/project.json}': (file) => {
const json = readJson<ProjectConfiguration>(file);
json.name ??= toProjectName(file);
return {
[json.name]: json,
};
},
},
});

let name = configuration.name;
if (!configuration.name) {
name = toProjectName(file);
}
if (!projects[name]) {
projects[name] = configuration;
} else {
logger.warn(
`Skipping project found at ${directory} since project ${name} already exists at ${projects[name].root}! Specify a unique name for the project to allow Nx to differentiate between the two projects.`
);
}
} else {
// We can infer projects from package.json files,
// if a package.json file is in a directory w/o a `project.json` file.
// this results in targets being inferred by Nx from package scripts,
// and the root / sourceRoot both being the directory.
if (fileName === 'package.json') {
const projectPackageJson = readJson<PackageJson>(file);
const { name, ...config } = buildProjectConfigurationFromPackageJson(
file,
projectPackageJson,
nxJson
);
if (!projects[name]) {
projects[name] = config;
} else {
logger.warn(
`Skipping project found at ${directory} since project ${name} already exists at ${projects[name].root}! Specify a unique name for the project to allow Nx to differentiate between the two projects.`
);
}
} else {
// This project was created from an nx plugin.
// The only thing we know about the file is its location
const { name, ...config } = inferProjectFromNonStandardFile(file);
if (!projects[name]) {
projects[name] = config;
} else {
logger.warn(
`Skipping project inferred from ${file} since project ${name} already exists.`
);
// We iterate over plugins first - this ensures that plugins specified first take precedence.
for (const plugin of plugins) {
// Within a plugin patterns specified later overwrite info from earlier matches.
for (const pattern in plugin.buildProjectNodes ?? {}) {
for (const file of projectFiles) {
if (minimatch(file, pattern)) {
const nodes = plugin.buildProjectNodes[pattern](file, projects);
for (const node in nodes) {
mergeProjectConfigurationIntoWorkspace(nodes[node], projects);
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion packages/nx/src/generators/tree.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ describe('tree', () => {

beforeEach(() => {
console.error = jest.fn();
console.log = jest.fn();
// console.log = jest.fn();

dir = dirSync().name;
ensureDirSync(path.join(dir, 'parent/child'));
Expand Down
Loading

0 comments on commit ceef985

Please sign in to comment.