Skip to content

Commit

Permalink
workaround electron-rebuild bug
Browse files Browse the repository at this point in the history 
See electron/rebuild#888

The way we invoke `electron-rebuild` should allow Theia application
developers to run `theia rebuild:*` from the root of their monorepos,
but because of a bug in `electron-rebuild` it will only build
dependencies listed in the `package.json` file located in `cwd`.

Implement a workaround that modifies or creates a `package.json` file to
list the dependencies to rebuild.

Add exit hooks to cleanup in case users do `Ctrl+C` while theia rebuild
is running so that we don't pollute user workspaces.
  • Loading branch information
@paul-marechal
paul-marechal committed Nov 17, 2021
1 parent 3bea909 commit 068c907
Showing 1 changed file with 195 additions and 50 deletions.
245 changes: 195 additions & 50 deletions dev-packages/application-manager/src/rebuild.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,17 @@
import cp = require('child_process');
import fs = require('fs-extra');
import path = require('path');
import os = require('os');

export type RebuildTarget = 'electron' | 'browser';

const EXIT_SIGNALS: NodeJS.Signals[] = ['SIGINT', 'SIGTERM'];

interface ExitToken {
getLastSignal(): NodeJS.Signals | undefined
onSignal(callback: (signal: NodeJS.Signals) => void): void
}

export const DEFAULT_MODULES = [
'@theia/node-pty',
'nsfw',
Expand Down Expand Up @@ -54,13 +62,21 @@ export function rebuild(target: RebuildTarget, options: RebuildOptions = {}): vo
} = options;
const cache = path.resolve(cacheRoot, '.browser_modules');
const cacheExists = folderExists(cache);
if (target === 'electron' && !cacheExists) {
rebuildElectronModules(cache, modules);
} else if (target === 'browser' && cacheExists) {
revertBrowserModules(cache, modules);
} else {
console.log(`native node modules are already rebuilt for ${target}`);
}
guardExit(async token => {
if (target === 'electron' && !cacheExists) {
process.exitCode = await rebuildElectronModules(cache, modules, token);
} else if (target === 'browser' && cacheExists) {
process.exitCode = await revertBrowserModules(cache, modules);
} else {
console.log(`native node modules are already rebuilt for ${target}`);
}
}).catch(errorOrSignal => {
if (typeof errorOrSignal === 'string' && errorOrSignal in os.constants.signals) {
process.kill(process.pid, errorOrSignal);
} else {
throw errorOrSignal;
}
});
}

function folderExists(folder: string): boolean {
Expand All @@ -75,7 +91,7 @@ function folderExists(folder: string): boolean {
}

/**
* Schema for `<browserModuleCache>/modules.json`
* Schema for `<browserModuleCache>/modules.json`.
*/
interface ModulesJson {
[moduleName: string]: ModuleBackup
Expand All @@ -84,14 +100,14 @@ interface ModuleBackup {
originalLocation: string
}

async function rebuildElectronModules(browserModuleCache: string, modules: string[]): Promise<void> {
async function rebuildElectronModules(browserModuleCache: string, modules: string[], token: ExitToken): Promise<number> {
const modulesJsonPath = path.join(browserModuleCache, 'modules.json');
const modulesJson: ModulesJson = await fs.access(modulesJsonPath).then(
exists => fs.readJSON(modulesJsonPath),
missing => ({})
() => fs.readJson(modulesJsonPath),
() => ({})
);
let success = true;
// backup already-built browser modules
// Backup already built browser modules.
await Promise.all(modules.map(async module => {
let modulePath;
try {
Expand All @@ -100,13 +116,13 @@ async function rebuildElectronModules(browserModuleCache: string, modules: strin
});
} catch (_) {
console.debug(`Module not found: ${module}`);
return; // Skip
return; // Skip current module.
}
const src = path.dirname(modulePath);
const dest = path.join(browserModuleCache, module);
try {
await fs.remove(dest);
await fs.copy(src, dest);
await fs.copy(src, dest, { overwrite: true });
modulesJson[module] = {
originalLocation: src,
};
Expand All @@ -118,65 +134,194 @@ async function rebuildElectronModules(browserModuleCache: string, modules: strin
}));
if (Object.keys(modulesJson).length === 0) {
console.debug('No module to rebuild.');
process.exit(0);
return 0;
}
// update manifest tracking the backups original locations
await fs.writeJSON(modulesJsonPath, modulesJson, { spaces: 2 });
// if we failed to process a module then exit now
// Update manifest tracking the backups original locations.
await fs.writeJson(modulesJsonPath, modulesJson, { spaces: 2 });
// If we failed to process a module then exit now.
if (!success) {
process.exit(1);
return 1;
}
// rebuild for electron
const todo = modules
.map(m => {
// electron-rebuild ignores the module namespace...
const slash = m.indexOf('/');
return m.startsWith('@') && slash !== -1
? m.substring(slash + 1)
: m;
})
.join(',');
await new Promise<void>((resolve, reject) => {
const electronRebuild = cp.spawn(`npx --no-install electron-rebuild -f -w="${todo}" -o="${todo}"`, {
stdio: 'inherit',
shell: true,
});
electronRebuild.on('error', reject);
electronRebuild.on('close', (code, signal) => {
if (code || signal) {
reject(`electron-rebuild exited with "${code || signal}"`);
} else {
resolve();
}
});
const todo = modules.map(m => {
// electron-rebuild ignores the module namespace...
const slash = m.indexOf('/');
return m.startsWith('@') && slash !== -1
? m.substring(slash + 1)
: m;
});
try {
if (process.env.THEIA_REBUILD_NO_WORKAROUND) {
return await runElectronRebuild(todo, token);
} else {
return await electronRebuildExtraModulesWorkaround(process.cwd(), todo, () => runElectronRebuild(todo, token), token);
}
} catch (error) {
return revertBrowserModules(browserModuleCache, modules);
}
}

async function revertBrowserModules(browserModuleCache: string, modules: string[]): Promise<void> {
async function revertBrowserModules(browserModuleCache: string, modules: string[]): Promise<number> {
let exitCode = 0;
const modulesJsonPath = path.join(browserModuleCache, 'modules.json');
const modulesJson: ModulesJson = await fs.readJSON(modulesJsonPath);
const modulesJson: ModulesJson = await fs.readJson(modulesJsonPath);
await Promise.all(Object.entries(modulesJson).map(async ([moduleName, entry]) => {
if (!modules.includes(moduleName)) {
return; // skip modules that weren't requested
return; // Skip modules that weren't requested.
}
const src = path.join(browserModuleCache, moduleName);
if (!await fs.pathExists(src)) {
delete modulesJson[moduleName];
console.error(`Missing backup for ${moduleName}!`);
exitCode = 1;
return;
}
const dest = entry.originalLocation;
try {
await fs.remove(dest);
await fs.copy(src, dest);
await fs.copy(src, dest, { overwrite: false });
await fs.remove(src);
delete modulesJson[moduleName];
console.debug(`Reverted "${moduleName}"`);
} catch (error) {
console.error(`Error while reverting "${moduleName}": ${error}`);
process.exitCode = 1;
exitCode = 1;
}
}));
if (Object.keys(modulesJson).length === 0) {
// we restored everything so we can delete the cache
// We restored everything so we can delete the cache.
await fs.remove(browserModuleCache);
} else {
// some things were not restored so we update the manifest
await fs.writeJSON(modulesJsonPath, modulesJson, { spaces: 2 });
// Some things were not restored so we update the manifest.
await fs.writeJson(modulesJsonPath, modulesJson, { spaces: 2 });
}
return exitCode;
}

async function runElectronRebuild(modules: string[], token: ExitToken): Promise<number> {
const todo = modules.join(',');
return new Promise((resolve, reject) => {
const electronRebuild = cp.spawn(`npx --no-install electron-rebuild -f -w="${todo}" -o="${todo}"`, {
stdio: 'inherit',
shell: true,
});
token.onSignal(signal => electronRebuild.kill(signal));
electronRebuild.on('error', reject);
electronRebuild.on('close', (code, signal) => {
if (signal) {
reject(new Error(`electron-rebuild exited with "${signal}"`));
} else {
resolve(code);
}
});
});
}

/**
* `electron-rebuild` is supposed to accept a list of modules to build, even when not part of the dependencies.
* But there is a bug that causes `electron-rebuild` to not correctly process this list of modules.
*
* This workaround will temporarily modify the current package.json file.
*
* PR with fix: https://github.com/electron/electron-rebuild/pull/888
*
* TODO: Remove this workaround.
*/
async function electronRebuildExtraModulesWorkaround<T>(cwd: string, extraModules: string[], run: (token: ExitToken) => Promise<T>, token: ExitToken): Promise<T> {
const packageJsonPath = path.resolve(cwd, 'package.json');
if (await fs.pathExists(packageJsonPath)) {
// package.json exists: We back it up before modifying it then revert it.
const packageJsonCopyPath = `${packageJsonPath}.copy`;
const packageJson = await fs.readJson(packageJsonPath);
await fs.copy(packageJsonPath, packageJsonCopyPath);
await throwIfSignal(token, async () => {
await fs.unlink(packageJsonCopyPath);
});
if (typeof packageJson.dependencies !== 'object') {
packageJson.dependencies = {};
}
for (const extraModule of extraModules) {
if (!packageJson.dependencies[extraModule]) {
packageJson.dependencies[extraModule] = '*';
}
}
try {
await fs.writeJson(packageJsonPath, packageJson, { spaces: 2 });
await throwIfSignal(token);
return await run(token);
} finally {
await fs.move(packageJsonCopyPath, packageJsonPath, { overwrite: true });
}
} else {
// package.json does not exist: We create one then remove it.
const packageJson = {
name: 'theia-rebuild-workaround',
version: '0.0.0',
dependencies: {} as Record<string, string>,
};
for (const extraModule of extraModules) {
packageJson.dependencies[extraModule] = '*';
}
try {
await fs.writeJson(packageJsonPath, packageJson);
await throwIfSignal(token);
return await run(token);
} finally {
await fs.unlink(packageJsonPath);
}
}
}

/**
* Temporarily install hooks to **try** to prevent the process from exiting while `run` is running.
*
* Note that it is still possible to kill the process and prevent cleanup logic (e.g. SIGTERM, computer forced shutdown, etc).
*/
async function guardExit<T>(run: (token: ExitToken) => Promise<T>): Promise<T> {
const token = new ExitTokenImpl();
const signalListener = (signal: NodeJS.Signals) => token._emitSignal(signal);
for (const signal of EXIT_SIGNALS) {
process.on(signal, signalListener);
}
try {
return await run(token);
} finally {
for (const signal of EXIT_SIGNALS) {
process.off(signal, signalListener);
}
}
}

class ExitTokenImpl implements ExitToken {

protected _listeners = new Set<(signal: NodeJS.Signals) => void>();
protected _lastSignal?: NodeJS.Signals;

onSignal(callback: (signal: NodeJS.Signals) => void): void {
this._listeners.add(callback);
}

getLastSignal(): NodeJS.Signals | undefined {
return this._lastSignal;
}

_emitSignal(signal: NodeJS.Signals): void {
this._lastSignal = signal;
for (const listener of this._listeners) {
listener(signal);
}
}
}

/**
* Throw `signal` if one was received, runs `cleanup` before doing so.
*/
async function throwIfSignal(token: ExitToken, cleanup?: () => Promise<void>): Promise<void> {
const signal = token.getLastSignal();
if (signal) {
try {
await cleanup?.();
} finally {
throw signal;
}
}
}

0 comments on commit 068c907

Please sign in to comment.