Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[rush-lib] Fix concurrency of rush-lib build script #4411

Merged
merged 4 commits into from
Oct 27, 2023
Merged

[rush-lib] Fix concurrency of rush-lib build script #4411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e4b045f
Fix concurrency in copyEmptyModules rush-lib script
D4N14L Oct 27, 2023
13d1adc
Rush change
D4N14L Oct 27, 2023
bff26ed
Fix script
D4N14L Oct 27, 2023
22dee5e
Reuse empty module buffer
D4N14L Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/changes/@microsoft/rush/user-danade-FixConcurrency_2023-10-27-21-34.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
68 changes: 37 additions & 31 deletions libraries/rush-lib/scripts/copyEmptyModules.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict';

const { FileSystem, Async } = require('@rushstack/node-core-library');
const { FileSystem, Async, AsyncQueue } = require('@rushstack/node-core-library');

const JS_FILE_EXTENSION = '.js';
const DTS_FILE_EXTENSION = '.d.ts';
Expand Down Expand Up @@ -46,39 +46,45 @@ module.exports = {
const jsInFolderPath = `${buildFolderPath}/lib-esnext`;
const dtsInFolderPath = `${buildFolderPath}/lib-commonjs`;
const outFolderPath = `${buildFolderPath}/lib`;
async function searchAsync(relativeFolderPath) {
const folderItems = await FileSystem.readFolderItemsAsync(
relativeFolderPath ? `${jsInFolderPath}/${relativeFolderPath}` : jsInFolderPath
);
await Async.forEachAsync(folderItems, async (folderItem) => {
const itemName = folderItem.name;
const relativeItemPath = relativeFolderPath ? `${relativeFolderPath}/${itemName}` : itemName;
const emptyModuleBuffer = Buffer.from('module.exports = {};', 'utf8');
const folderPathQueue = new AsyncQueue([undefined]);

if (folderItem.isDirectory()) {
await searchAsync(relativeItemPath);
} else if (folderItem.isFile() && itemName.endsWith(JS_FILE_EXTENSION)) {
const jsInPath = `${jsInFolderPath}/${relativeItemPath}`;
const jsFileText = await FileSystem.readFileAsync(jsInPath);
const strippedJsFileText = stripCommentsFromJsFile(jsFileText);
if (strippedJsFileText === 'export {};') {
const outJsPath = `${outFolderPath}/${relativeItemPath}`;
terminal.writeVerboseLine(`Writing stub to ${outJsPath}`);
await FileSystem.writeFileAsync(outJsPath, 'module.exports = {};', {
ensureFolderExists: true
});
await Async.forEachAsync(
folderPathQueue,
async ([relativeFolderPath, callback]) => {
const folderPath = relativeFolderPath ? `${jsInFolderPath}/${relativeFolderPath}` : jsInFolderPath;
const folderItems = await FileSystem.readFolderItemsAsync(folderPath);
for (const folderItem of folderItems) {
const itemName = folderItem.name;
const relativeItemPath = relativeFolderPath ? `${relativeFolderPath}/${itemName}` : itemName;

const relativeDtsPath = relativeItemPath.slice(0, -JS_FILE_EXTENSION.length) + DTS_FILE_EXTENSION;
const inDtsPath = `${dtsInFolderPath}/${relativeDtsPath}`;
const outDtsPath = `${outFolderPath}/${relativeDtsPath}`;
terminal.writeVerboseLine(`Copying ${inDtsPath} to ${outDtsPath}`);
// We know this is a file, don't need the redundant checks in FileSystem.copyFileAsync
const buffer = await FileSystem.readFileToBufferAsync(inDtsPath);
await FileSystem.writeFileAsync(outDtsPath, buffer, { ensureFolderExists: true });
if (folderItem.isDirectory()) {
folderPathQueue.push(relativeItemPath);
} else if (folderItem.isFile() && itemName.endsWith(JS_FILE_EXTENSION)) {
const jsInPath = `${jsInFolderPath}/${relativeItemPath}`;
const jsFileText = await FileSystem.readFileAsync(jsInPath);
const strippedJsFileText = stripCommentsFromJsFile(jsFileText);
if (strippedJsFileText === 'export {};') {
const outJsPath = `${outFolderPath}/${relativeItemPath}`;
terminal.writeVerboseLine(`Writing stub to ${outJsPath}`);
await FileSystem.writeFileAsync(outJsPath, emptyModuleBuffer, {
ensureFolderExists: true
});

const relativeDtsPath =
relativeItemPath.slice(0, -JS_FILE_EXTENSION.length) + DTS_FILE_EXTENSION;
const inDtsPath = `${dtsInFolderPath}/${relativeDtsPath}`;
const outDtsPath = `${outFolderPath}/${relativeDtsPath}`;
terminal.writeVerboseLine(`Copying ${inDtsPath} to ${outDtsPath}`);
// We know this is a file, don't need the redundant checks in FileSystem.copyFileAsync
const buffer = await FileSystem.readFileToBufferAsync(inDtsPath);
await FileSystem.writeFileAsync(outDtsPath, buffer, { ensureFolderExists: true });
}
}
}
});
}

await searchAsync(undefined);
callback();
},
{ concurrency: 10 }
);
}
};