diff --git a/packages/ast-utils/src/exports.ts b/packages/ast-utils/src/exports.ts index b7851805..5e710633 100644 --- a/packages/ast-utils/src/exports.ts +++ b/packages/ast-utils/src/exports.ts @@ -1,5 +1,6 @@ import { isString } from './isPrimitive' -import type { ASTNode, Collection, JSCodeshift } from 'jscodeshift' +import { isTopLevel } from './isTopLevel' +import type { ASTNode, AssignmentExpression, Collection, Identifier, JSCodeshift, MemberExpression, VariableDeclarator } from 'jscodeshift' type Exported = string type Local = string @@ -21,7 +22,7 @@ export class ExportManager { this.exportsFrom.set(local, source) } - collect(j: JSCodeshift, root: Collection) { + collectEsModuleExport(j: JSCodeshift, root: Collection) { root .find(j.ExportDefaultDeclaration) .forEach((path) => { @@ -75,6 +76,253 @@ export class ExportManager { }) } + collectCommonJsExport(j: JSCodeshift, root: Collection) { + /** + * Default export + * + * Note: `exports = { ... }` is not valid + * So we won't handle it + * + * @example + * module.exports = 1 + * -> + * export default 1 + * + * @example + * module.exports = { foo: 1 } + * -> + * export default { foo: 1 } + */ + root + .find(j.ExpressionStatement, { + expression: { + type: 'AssignmentExpression', + operator: '=', + left: { + type: 'MemberExpression', + object: { + type: 'Identifier', + name: 'module', + }, + property: { + type: 'Identifier', + name: 'exports', + }, + }, + }, + }) + .forEach((path) => { + if (!isTopLevel(j, path)) return + + const expression = path.node.expression as AssignmentExpression + if (j.Identifier.check(expression.right)) { + this.addDefaultExport(expression.right.name) + } + else if (j.ObjectExpression.check(expression.right)) { + const object = expression.right + const properties = object.properties + properties.forEach((property) => { + if (j.ObjectProperty.check(property)) { + const key = property.key + const value = property.value + if (j.Identifier.check(key) && j.Identifier.check(value)) { + this.addNamedExport(key.name, value.name) + } + } + }) + } + }) + + /** + * Individual exports + * + * @example + * module.exports.foo = 1 + * -> + * export const foo = 1 + * + * @example + * module.exports.foo = foo + * -> + * export { foo } + */ + root + .find(j.ExpressionStatement, { + expression: { + type: 'AssignmentExpression', + operator: '=', + left: { + type: 'MemberExpression', + object: { + type: 'MemberExpression', + object: { + type: 'Identifier', + name: 'module', + }, + property: { + type: 'Identifier', + name: 'exports', + }, + }, + property: { + type: 'Identifier', + }, + }, + }, + }) + .forEach((path) => { + if (!isTopLevel(j, path)) return + + const expression = path.node.expression as AssignmentExpression + const left = expression.left as MemberExpression + const name = (left.property as Identifier).name + this.addNamedExport(name, name) + }) + + /** + * Individual exports + * + * @example + * exports.foo = 2 + * -> + * export const foo = 2 + */ + root + .find(j.ExpressionStatement, { + expression: { + type: 'AssignmentExpression', + operator: '=', + left: { + type: 'MemberExpression', + object: { + type: 'Identifier', + name: 'exports', + }, + property: { + type: 'Identifier', + }, + }, + }, + }) + .forEach((path) => { + if (!isTopLevel(j, path)) return + + const expression = path.node.expression as AssignmentExpression + const left = expression.left as MemberExpression + const name = (left.property as Identifier).name + this.addNamedExport(name, name) + }) + + /** + * Special case: + * + * @example + * var foo = exports.foo = 1 + */ + root + .find(j.VariableDeclaration, { + declarations: [ + { + type: 'VariableDeclarator', + id: { + type: 'Identifier', + }, + init: { + type: 'AssignmentExpression', + operator: '=', + left: { + type: 'MemberExpression', + object: { + type: 'Identifier', + name: 'exports', + }, + property: { + type: 'Identifier', + }, + }, + }, + }, + ], + }) + .forEach((path) => { + if (!isTopLevel(j, path)) return + + const declaration = path.node.declarations[0] as VariableDeclarator + const init = declaration.init as AssignmentExpression + const left = init.left as MemberExpression + const right = init.right + + const name = (left.property as Identifier).name + + if (j.Identifier.check(right)) { + if (name === 'default') { + this.addDefaultExport(right.name) + } + else { + this.addNamedExport(name, right.name) + } + } + }) + + /** + * Special case: + * + * @example + * var bar = module.exports.baz = 2 + */ + root + .find(j.VariableDeclaration, { + declarations: [ + { + type: 'VariableDeclarator', + id: { + type: 'Identifier', + }, + init: { + type: 'AssignmentExpression', + operator: '=', + left: { + type: 'MemberExpression', + object: { + type: 'MemberExpression', + object: { + type: 'Identifier', + name: 'module', + }, + property: { + type: 'Identifier', + name: 'exports', + }, + }, + property: { + type: 'Identifier', + }, + }, + }, + }, + ], + }) + .forEach((path) => { + if (!isTopLevel(j, path)) return + + const declaration = path.node.declarations[0] as VariableDeclarator + const init = declaration.init as AssignmentExpression + const left = init.left as MemberExpression + const right = init.right + + const name = (left.property as Identifier).name + + if (j.Identifier.check(right)) { + if (name === 'default') { + this.addDefaultExport(right.name) + } + else { + this.addNamedExport(name, right.name) + } + } + }) + } + toJSON(): Record { return Object.fromEntries(this.exports) } diff --git a/packages/ast-utils/src/imports.ts b/packages/ast-utils/src/imports.ts index 847173db..8c2913ef 100644 --- a/packages/ast-utils/src/imports.ts +++ b/packages/ast-utils/src/imports.ts @@ -1,5 +1,7 @@ +import { isNumber, isString } from './isPrimitive' +import { isTopLevel } from './isTopLevel' import type { NodePath } from 'ast-types/lib/node-path' -import type { Collection, ImportDeclaration, JSCodeshift, VariableDeclaration } from 'jscodeshift' +import type { CallExpression, Collection, ImportDeclaration, JSCodeshift, Literal, VariableDeclaration, VariableDeclarator } from 'jscodeshift' type Source = string type Imported = string @@ -159,7 +161,7 @@ export class ImportManager { ] } - collectImportsFromRoot(j: JSCodeshift, root: Collection) { + collectEsModuleImport(j: JSCodeshift, root: Collection) { root .find(j.ImportDeclaration) .forEach((path) => { @@ -204,6 +206,69 @@ export class ImportManager { }) } + collectCommonJsImport(j: JSCodeshift, root: Collection) { + /** + * Basic require and require with destructuring + * + * @example + * var foo = require('foo') + * var { bar } = require('bar') + */ + root + .find(j.VariableDeclaration, { + declarations: [ + { + type: 'VariableDeclarator', + init: { + type: 'CallExpression', + callee: { + type: 'Identifier', + name: 'require', + }, + arguments: [{ + type: 'Literal' as const, + value: (value: unknown) => isString(value) || isNumber(value), + }], + }, + }, + ], + }) + .forEach((path) => { + if (!isTopLevel(j, path)) return + + const firstDeclaration = path.node.declarations[0] as VariableDeclarator + const id = firstDeclaration.id + const init = firstDeclaration.init as CallExpression + + const sourceLiteral = init.arguments[0] as Literal + const source = sourceLiteral.value as string + + if (j.Identifier.check(id)) { + const local = id.name + this.addDefaultImport(source, local) + return + } + + /** + * var { bar } = require('bar') + */ + if (j.ObjectPattern.check(id)) { + id.properties.forEach((property) => { + if (j.Property.check(property) + && j.Identifier.check(property.key) + && j.Identifier.check(property.value) + ) { + const imported = property.key.name + const local = property.value.name + this.addNamedImport(source, imported, local) + } + }) + // eslint-disable-next-line no-useless-return + return + } + }) + } + /** * Remove all collected import declarations */ diff --git a/packages/ast-utils/src/index.ts b/packages/ast-utils/src/index.ts index 2a81d2d6..c400fce8 100644 --- a/packages/ast-utils/src/index.ts +++ b/packages/ast-utils/src/index.ts @@ -9,3 +9,4 @@ export { wrapAstTransformation } from './wrapAstTransformation' export * from './imports' export * from './exports' export * from './isPrimitive' +export * from './types' diff --git a/packages/ast-utils/src/types.ts b/packages/ast-utils/src/types.ts new file mode 100644 index 00000000..24ef3f3b --- /dev/null +++ b/packages/ast-utils/src/types.ts @@ -0,0 +1,11 @@ +import type { ImportInfo } from './imports' + +export type ModuleMapping = Record + +export interface ModuleMeta { + [moduleId: string]: { + import: ImportInfo[] + export: Record + tags: Record + } +} diff --git a/packages/playground/package.json b/packages/playground/package.json index e4e280e0..b572a04b 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -44,6 +44,7 @@ "@fortawesome/vue-fontawesome": "^3.0.3", "@headlessui/vue": "^1.7.16", "@heroicons/vue": "^2.0.18", + "@unminify-kit/ast-utils": "workspace:*", "@unminify-kit/unminify": "workspace:*", "@unminify-kit/unpacker": "workspace:*", "@vueuse/core": "^10.4.1", diff --git a/packages/playground/src/codemod.worker.ts b/packages/playground/src/codemod.worker.ts index 7162dd81..984b9f46 100644 --- a/packages/playground/src/codemod.worker.ts +++ b/packages/playground/src/codemod.worker.ts @@ -1,7 +1,7 @@ import { runTransformations, transformationMap } from '@unminify-kit/unminify' import type { ModuleMeta } from './composables/useModuleMeta' import type { TransformedModule } from './types' -import type { ModuleMapping } from '@unminify-kit/unpacker' +import type { ModuleMapping } from '@unminify-kit/ast-utils' onmessage = ( msg: MessageEvent<{ diff --git a/packages/playground/src/composables/useCodemod.ts b/packages/playground/src/composables/useCodemod.ts index 8ed20244..03300d9e 100644 --- a/packages/playground/src/composables/useCodemod.ts +++ b/packages/playground/src/composables/useCodemod.ts @@ -1,7 +1,7 @@ import CodemodWorker from '../codemod.worker?worker' import type { ModuleMeta } from './useModuleMeta' import type { TransformedModule } from '../types' -import type { ModuleMapping } from '@unminify-kit/unpacker' +import type { ModuleMapping } from '@unminify-kit/ast-utils' export function useCodemod() { const transform = ( diff --git a/packages/playground/src/composables/useModuleMapping.ts b/packages/playground/src/composables/useModuleMapping.ts index fefc036c..be024743 100644 --- a/packages/playground/src/composables/useModuleMapping.ts +++ b/packages/playground/src/composables/useModuleMapping.ts @@ -1,6 +1,6 @@ import { KEY_MODULE_MAPPING } from '../const' import { useLocalStorage } from './shared/useLocalStorage' -import type { ModuleMapping } from '@unminify-kit/unpacker' +import type { ModuleMapping } from '@unminify-kit/ast-utils' export function useModuleMapping() { const [moduleMapping, setModuleMapping] = useLocalStorage(KEY_MODULE_MAPPING, {}) diff --git a/packages/playground/src/composables/useModuleMeta.ts b/packages/playground/src/composables/useModuleMeta.ts index 787abf31..46b7f97d 100644 --- a/packages/playground/src/composables/useModuleMeta.ts +++ b/packages/playground/src/composables/useModuleMeta.ts @@ -1,10 +1,6 @@ import { KEY_MODULE_META } from '../const' import { useLocalStorage } from './shared/useLocalStorage' -import type { Module } from '@unminify-kit/unpacker' - -export interface ModuleMeta { - [moduleId: string]: Pick -} +import type { ModuleMeta } from '@unminify-kit/ast-utils' export const useModuleMeta = () => { const [moduleMeta, setModuleMeta] = useLocalStorage(KEY_MODULE_META, {}) diff --git a/packages/playground/src/pages/Uploader.vue b/packages/playground/src/pages/Uploader.vue index 4974b2c3..9e12a7ba 100644 --- a/packages/playground/src/pages/Uploader.vue +++ b/packages/playground/src/pages/Uploader.vue @@ -23,11 +23,13 @@ import type { TransformedModule } from '../types' const [source] = useState('') const [isLoading, setIsLoading] = useState(false) const [processedCount, setProcessedCount] = useState(0) -const { fileIds, setFileIds } = useFileIds() + +const router = useRouter() const { transform } = useCodemod() + +const { fileIds, setFileIds } = useFileIds() const { moduleMeta, setModuleMeta } = useModuleMeta() const { moduleMapping, setModuleMapping } = useModuleMapping() -const router = useRouter() function onUpload(file: File) { const reader = new FileReader() @@ -76,8 +78,8 @@ async function startUnpack(code: string) { }) setFileIds([ - ...unpackedModules.filter(module => module.isEntry).map(module => module.id), - ...unpackedModules.filter(module => !module.isEntry).map(module => module.id), + ...unpackedModules.filter(module => module.isEntry).map(module => module.id).sort((a, b) => +a - +b), + ...unpackedModules.filter(module => !module.isEntry).map(module => module.id).sort((a, b) => +a - +b), ]) setModuleMeta( diff --git a/packages/unminify/src/transformations/babel-helpers/index.ts b/packages/unminify/src/transformations/babel-helpers/index.ts new file mode 100644 index 00000000..a7ed144f --- /dev/null +++ b/packages/unminify/src/transformations/babel-helpers/index.ts @@ -0,0 +1,15 @@ +import wrap from '../../wrapAstTransformation' +import { transformAST as arrayLikeToArray } from './arrayLikeToArray' +import { transformAST as arrayWithoutHoles } from './arrayWithoutHoles' +import { transformAST as objectSpread } from './objectSpread' +import { transformAST as toConsumableArray } from './toConsumableArray' +import type { ASTTransformation } from '../../wrapAstTransformation' + +export const transformAST: ASTTransformation = (context, params) => { + arrayLikeToArray(context, params) + arrayWithoutHoles(context, params) + objectSpread(context, params) + toConsumableArray(context, params) +} + +export default wrap(transformAST) diff --git a/packages/unminify/src/transformations/index.ts b/packages/unminify/src/transformations/index.ts index fce5f986..dec21f5e 100644 --- a/packages/unminify/src/transformations/index.ts +++ b/packages/unminify/src/transformations/index.ts @@ -22,6 +22,7 @@ import unNumericLiteral from './un-numeric-literal' import unOptionalChaining from './un-optional-chaining' import unParameters from './un-parameters' import unReturn from './un-return' +import unRuntimeHelper from './un-runtime-helper' import unSequenceExpression from './un-sequence-expression' import unTemplateLiteral from './un-template-literal' import unTypeConstructor from './un-type-constructor' @@ -42,6 +43,7 @@ export const transformationMap: { lebab, 'un-esm': unEsm, 'un-export-rename': unExportRename, + 'un-runtime-helper': unRuntimeHelper, 'un-use-strict': unUseStrict, 'un-esmodule-flag': unEsModuleFlag, 'un-boolean': unBoolean, diff --git a/packages/unminify/src/transformations/module-mapping.ts b/packages/unminify/src/transformations/module-mapping.ts index f67ec357..745f0d8b 100644 --- a/packages/unminify/src/transformations/module-mapping.ts +++ b/packages/unminify/src/transformations/module-mapping.ts @@ -1,5 +1,6 @@ import wrap from '../wrapAstTransformation' import type { ASTTransformation } from '../wrapAstTransformation' +import type { ModuleMapping } from '@unminify-kit/ast-utils' import type { Literal } from 'jscodeshift' /** @@ -9,7 +10,7 @@ import type { Literal } from 'jscodeshift' * const a = require('index.js') */ interface Params { - moduleMapping: Record + moduleMapping: ModuleMapping } export const transformAST: ASTTransformation = (context, params = { moduleMapping: {} }) => { const { root, j } = context diff --git a/packages/unminify/src/transformations/un-esm.ts b/packages/unminify/src/transformations/un-esm.ts index 4ad7c4df..9b720ae9 100644 --- a/packages/unminify/src/transformations/un-esm.ts +++ b/packages/unminify/src/transformations/un-esm.ts @@ -57,7 +57,7 @@ function transformImport(context: Context, hoist: boolean) { */ const importManager = new ImportManager() - importManager.collectImportsFromRoot(j, root) + importManager.collectEsModuleImport(j, root) /** * Scan through all `require` call for the recording the order of imports diff --git a/packages/unminify/src/transformations/un-indirect-call.ts b/packages/unminify/src/transformations/un-indirect-call.ts index f7863ca0..0c4fe110 100644 --- a/packages/unminify/src/transformations/un-indirect-call.ts +++ b/packages/unminify/src/transformations/un-indirect-call.ts @@ -51,7 +51,7 @@ export const transformAST: ASTTransformation = (context) => { if (!rootScope) return const importManager = new ImportManager() - importManager.collectImportsFromRoot(j, root) + importManager.collectEsModuleImport(j, root) /** * Adding imports one by one will cause scope issues. diff --git a/packages/unminify/src/transformations/un-runtime-helper.ts b/packages/unminify/src/transformations/un-runtime-helper.ts new file mode 100644 index 00000000..ecd0be57 --- /dev/null +++ b/packages/unminify/src/transformations/un-runtime-helper.ts @@ -0,0 +1,51 @@ +import { getTopLevelStatements } from '@unminify-kit/ast-utils' +import { mergeComments } from '../utils/comments' +import wrap from '../wrapAstTransformation' + +import { transformAST as babelHelpers } from './babel-helpers' +import type { ASTTransformation, Context } from '../wrapAstTransformation' +import type { ModuleMapping, ModuleMeta } from '@unminify-kit/ast-utils' +import type { FunctionDeclaration } from 'jscodeshift' + +interface Params { + moduleMapping?: ModuleMapping + moduleMeta?: ModuleMeta +} + +const addAnnotationOnHelper = (context: Context, params: Params) => { + const { moduleMapping, moduleMeta } = params + if (!moduleMapping || !moduleMeta) return + + const { root, j, filename } = context + const moduleId = Object.entries(moduleMapping).find(([_, path]) => path === filename)?.[0] + if (moduleId === undefined) return + + const modMeta = moduleMeta[moduleId] + if (!modMeta) return + + const statements = getTopLevelStatements(root) + const functions = statements.filter((s): s is FunctionDeclaration => j.FunctionDeclaration.check(s)) + + functions.forEach((fn) => { + if (!j.Identifier.check(fn.id)) return + const functionName = fn.id.name + + const tags = modMeta.tags[functionName] + if (!tags || tags.length === 0) return + + const commentContent = tags.map(tag => ` * ${tag}`).join('\n') + const comment = j.commentBlock(`*\n${commentContent}\n `, true, false) + mergeComments(fn, [comment]) + }) +} + +/** + * Replace runtime helper with the actual original code. + */ +export const transformAST: ASTTransformation = (context, params) => { + addAnnotationOnHelper(context, params) + + babelHelpers(context, params) +} + +export default wrap(transformAST) diff --git a/packages/unpacker/src/Module.ts b/packages/unpacker/src/Module.ts index 7ef6c00d..a765a235 100644 --- a/packages/unpacker/src/Module.ts +++ b/packages/unpacker/src/Module.ts @@ -1,6 +1,5 @@ -import { scanModule } from './module-scan' import type { ImportInfo } from '@unminify-kit/ast-utils' -import type { Collection, JSCodeshift } from 'jscodeshift' +import type { Collection } from 'jscodeshift' export class Module { /** The module's id */ @@ -32,11 +31,9 @@ export class Module { return this.ast.toSource() } - constructor(id: string | number, j: JSCodeshift, root: Collection, isEntry = false) { + constructor(id: string | number, root: Collection, isEntry = false) { this.id = id this.ast = root this.isEntry = isEntry - - scanModule(j, this) } } diff --git a/packages/unpacker/src/ModuleMapping.ts b/packages/unpacker/src/ModuleMapping.ts deleted file mode 100644 index d56c496a..00000000 --- a/packages/unpacker/src/ModuleMapping.ts +++ /dev/null @@ -1 +0,0 @@ -export type ModuleMapping = Record diff --git a/packages/unpacker/src/extractors/browserify/index.ts b/packages/unpacker/src/extractors/browserify/index.ts index 1d39101b..f711ac4d 100644 --- a/packages/unpacker/src/extractors/browserify/index.ts +++ b/packages/unpacker/src/extractors/browserify/index.ts @@ -1,6 +1,6 @@ import { isFunctionExpression, isNumber, isString, renameFunctionParameters } from '@unminify-kit/ast-utils' import { Module } from '../../Module' -import type { ModuleMapping } from '../../ModuleMapping' +import type { ModuleMapping } from '@unminify-kit/ast-utils' import type { ArrayExpression, ArrowFunctionExpression, Collection, FunctionExpression, JSCodeshift, Literal, ObjectExpression } from 'jscodeshift' /** @@ -82,7 +82,7 @@ export function getModulesFromBrowserify(j: JSCodeshift, root: Collection): const moduleContent = j({ type: 'Program', body: moduleFactory.body.body }) const isEntry = entryIds.includes(moduleId) - const module = new Module(moduleId, j, moduleContent, isEntry) + const module = new Module(moduleId, moduleContent, isEntry) modules.add(module) moduleMap.properties.forEach((property) => { diff --git a/packages/unpacker/src/extractors/webpack/jsonp.ts b/packages/unpacker/src/extractors/webpack/jsonp.ts index d195848f..28ecf3dd 100644 --- a/packages/unpacker/src/extractors/webpack/jsonp.ts +++ b/packages/unpacker/src/extractors/webpack/jsonp.ts @@ -1,7 +1,7 @@ import { renameFunctionParameters } from '@unminify-kit/ast-utils' import { Module } from '../../Module' import { convertRequireHelpersForWebpack4, convertRequireHelpersForWebpack5 } from './requireHelpers' -import type { ModuleMapping } from '../../ModuleMapping' +import type { ModuleMapping } from '@unminify-kit/ast-utils' import type { ArrayExpression, Collection, FunctionExpression, JSCodeshift, Literal, MemberExpression, ObjectExpression, Property } from 'jscodeshift' /** @@ -107,7 +107,7 @@ export function getModulesForWebpackJsonP(j: JSCodeshift, root: Collection): convertRequireHelpersForWebpack4(j, moduleContent) convertRequireHelpersForWebpack5(j, moduleContent) - const module = new Module(moduleId, j, moduleContent, false) + const module = new Module(moduleId, moduleContent, false) modules.add(module) }) }) diff --git a/packages/unpacker/src/extractors/webpack/webpack4.ts b/packages/unpacker/src/extractors/webpack/webpack4.ts index ba74a6f0..200f405e 100644 --- a/packages/unpacker/src/extractors/webpack/webpack4.ts +++ b/packages/unpacker/src/extractors/webpack/webpack4.ts @@ -1,7 +1,7 @@ import { isNumber, renameFunctionParameters } from '@unminify-kit/ast-utils' import { Module } from '../../Module' import { convertRequireHelpersForWebpack4 } from './requireHelpers' -import type { ModuleMapping } from '../../ModuleMapping' +import type { ModuleMapping } from '@unminify-kit/ast-utils' import type { ArrayExpression, Collection, FunctionExpression, JSCodeshift, Literal } from 'jscodeshift' /** @@ -78,7 +78,7 @@ export function getModulesForWebpack4(j: JSCodeshift, root: Collection): const moduleContent = j({ type: 'Program', body: functionExpression.body.body }) convertRequireHelpersForWebpack4(j, moduleContent) - const module = new Module(moduleId, j, moduleContent, entryIds.includes(moduleId)) + const module = new Module(moduleId, moduleContent, entryIds.includes(moduleId)) modules.add(module) }) diff --git a/packages/unpacker/src/extractors/webpack/webpack5.ts b/packages/unpacker/src/extractors/webpack/webpack5.ts index 9cfc030d..976e7c67 100644 --- a/packages/unpacker/src/extractors/webpack/webpack5.ts +++ b/packages/unpacker/src/extractors/webpack/webpack5.ts @@ -1,7 +1,7 @@ import { getTopLevelStatements, isIIFE, renameFunctionParameters } from '@unminify-kit/ast-utils' import { Module } from '../../Module' import { convertRequireHelpersForWebpack5 } from './requireHelpers' -import type { ModuleMapping } from '../../ModuleMapping' +import type { ModuleMapping } from '@unminify-kit/ast-utils' import type { ArrowFunctionExpression, Collection, FunctionExpression, JSCodeshift, Literal, Property, Statement, VariableDeclaration } from 'jscodeshift' /** @@ -81,7 +81,7 @@ export function getModulesForWebpack5(j: JSCodeshift, root: Collection): const moduleContent = j({ type: 'Program', body: functionExpression.body.body }) convertRequireHelpersForWebpack5(j, moduleContent) - const module = new Module(moduleId, j, moduleContent, false) + const module = new Module(moduleId, moduleContent, false) modules.add(module) }) @@ -91,7 +91,7 @@ export function getModulesForWebpack5(j: JSCodeshift, root: Collection): // @ts-expect-error - skip type check const entryModule = lastStatement.expression.callee.body.body const moduleContent = j({ type: 'Program', body: entryModule }) - const module = new Module('entry.js', j, moduleContent, true) + const module = new Module('entry.js', moduleContent, true) modules.add(module) } else { diff --git a/packages/unpacker/src/index.ts b/packages/unpacker/src/index.ts index 01f3eaec..f7ff9dfa 100644 --- a/packages/unpacker/src/index.ts +++ b/packages/unpacker/src/index.ts @@ -1,4 +1,3 @@ export type { Module } from './Module' -export type { ModuleMapping } from './ModuleMapping' export * from './unpack' diff --git a/packages/unpacker/src/module-scan/babel-runtime.ts b/packages/unpacker/src/module-scan/babel-runtime.ts index aeb7fe30..f0fb6a02 100644 --- a/packages/unpacker/src/module-scan/babel-runtime.ts +++ b/packages/unpacker/src/module-scan/babel-runtime.ts @@ -1,18 +1,24 @@ -import { getTopLevelStatements } from '@unminify-kit/ast-utils' +import { isTopLevel } from '@unminify-kit/ast-utils' import type { Module } from '../Module' -import type { ArrowFunctionExpression, FunctionDeclaration, FunctionExpression, JSCodeshift } from 'jscodeshift' +import type { ArrowFunctionExpression, FunctionDeclaration, FunctionExpression, JSCodeshift, Statement } from 'jscodeshift' const moduleMatchers: Record>> = { + '@babel/runtime/helpers/arrayLikeToArray': [ + /for\s?\(var \w+\s?=\s?0,\s?\w+\s?=\s?(new\s)?Array\(\w+\);\s?\w+\s?<\s?\w+;\s?\w+\+\+\)\s?\w+\[\w+\]\s?=\s?\w+\[\w+\]/, + ], + '@babel/runtime/helpers/arrayWithHoles': [ + /{(\r\n|\r|\n)?(\s+)?if\s?\(Array\.isArray\(\w+\)\)\s?return\s?\w+;?(\r\n|\r|\n)?(\s+)?}/, + ], '@babel/runtime/helpers/classCallCheck': [ 'throw new TypeError("Cannot call a class as a function")', ], '@babel/runtime/helpers/createForOfIteratorHelperLoose': [ - 'throw new TypeError("Invalid attempt to iterate non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");', + 'throw new TypeError("Invalid attempt to iterate non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")', ], '@babel/runtime/helpers/createForOfIteratorHelper': [ [ /if\s?\(!\w+\s?&&\s?\w+\.return\s?!=\s?null\)\s?\w+\.return\(\)/, - 'throw new TypeError("Invalid attempt to iterate non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");', + 'throw new TypeError("Invalid attempt to iterate non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")', ], ], '@babel/runtime/helpers/inherits': [ @@ -22,10 +28,10 @@ const moduleMatchers: Record = { ], } +/** + * Scan all top level functions and mark tags based on the content of the function. + */ export function scanBabelRuntime(j: JSCodeshift, module: Module) { const root = module.ast - const statements = getTopLevelStatements(root) + const statements = root.get().node.body as Statement[] const functions = statements.filter((node): node is FunctionExpression | FunctionDeclaration | ArrowFunctionExpression => { return j.FunctionDeclaration.check(node) || j.ArrowFunctionExpression.check(node) @@ -183,7 +192,7 @@ export function scanBabelRuntime(j: JSCodeshift, module: Module) { const code = j(func).toSource() - const collectedTags = new Set(Object.keys(moduleMatchers) + const collectedTags = [...new Set(Object.keys(moduleMatchers) .filter((moduleName) => { const matchers = moduleMatchers[moduleName] return matchers.some((matcher) => { @@ -207,18 +216,125 @@ export function scanBabelRuntime(j: JSCodeshift, module: Module) { return false }) }), - ) + )] /** * Module's dependencies might be inlined by compiler. * So we need to remove scanned tag that are dependent * of other scanned tags. */ - const _collectedTags = [...collectedTags] - const tagsDependencies = _collectedTags.flatMap(tag => moduleDeps[tag] ?? []) - const tags = _collectedTags.filter(tag => !tagsDependencies.includes(tag)) + const tagsDependencies = collectedTags.flatMap(tag => moduleDeps[tag] ?? []) + const tags = collectedTags.filter(tag => !tagsDependencies.includes(tag)) module.tags[functionName] ??= [] module.tags[functionName].push(...tags) }) } + +/** + * Go through all tagged functions and check for the usage of other tagged functions. + * If a tagged function is used, then we need to add the tag to the function. + * In the end, we will have a complete list of tags for each function, so that + * we have a chance to change the tags to their upper level functions. + */ +export function postScanBabelRuntime(j: JSCodeshift, modules: Module[]) { + modules.forEach((module) => { + const { ast: root, import: imports } = module + const rootScope = root.get().scope + + const taggedImportLocals = new Map() + imports.forEach((imp) => { + if (imp.type === 'bare' || imp.type === 'namespace') return + + const targetModule = modules.find(m => m.id.toString() === imp.source.toString()) + if (!targetModule || Object.keys(targetModule.tags).length === 0) return + + if (imp.type === 'named') { + const targetTags = targetModule.tags[imp.name] + if (!targetTags || targetTags.length === 0) return + taggedImportLocals.set(imp.local, targetTags) + return + } + + if (imp.type === 'default') { + const targetTags = targetModule.tags.default + if (targetTags && targetTags.length !== 0) { + taggedImportLocals.set(imp.name, targetTags) + } + + Object.entries(targetModule.export).forEach(([exportName, exportLocalName]) => { + const targetTags = targetModule.tags[exportLocalName] + // TODO: Currently we didn't pull in dependent's imported tags. + // We might need to build a module graph and start from the leaf. + if (!targetTags || targetTags.length === 0) return + taggedImportLocals.set(`${imp.name}.${exportName}`, targetTags) + }) + } + }, {} as Record) + + const functionPaths = [ + ...root.find(j.FunctionDeclaration).filter(p => isTopLevel(j, p)).paths(), + ...root.find(j.ArrowFunctionExpression).filter(p => isTopLevel(j, p)).paths(), + ] + + functionPaths.forEach((func) => { + const functionName = func.node.id?.name + if (!functionName || typeof functionName !== 'string') return + + taggedImportLocals.forEach((tags, localName) => { + const [importObj, importProp] = localName.split('.') + const isReferenced = localName.includes('.') + ? j(func) + .find(j.MemberExpression, { + object: { name: importObj }, + property: { name: importProp }, + }) + .filter((path) => { + const scope = path.scope?.lookup(importObj) + return scope === rootScope + }) + .size() > 0 + : j(func) + .find(j.Identifier, { name: localName }) + .filter((path) => { + const scope = path.scope?.lookup(localName) + return scope === rootScope + }) + .size() > 0 + + if (isReferenced) { + module.tags[functionName] ??= [] + module.tags[functionName].push(...tags) + } + }) + + if (module.tags[functionName]?.length === 0) return + + /** + * Try to combine tags based on dependencies. + */ + const moduleTag = module.tags[functionName]! + let score = 0 + let matchedTag: string | null = null + Object.entries(moduleDeps).forEach(([tag, deps]) => { + if (!deps) return + const allMatch = deps.every((dep) => { + return moduleTag.includes(dep) + }) + // TODO: we can further improve the scoring algorithm. + if (allMatch && deps.length > score) { + score = deps.length + matchedTag = tag + } + }) + if (matchedTag) { + const deps = moduleDeps[matchedTag]! + deps.forEach((dep) => { + const index = moduleTag.indexOf(dep) + moduleTag.splice(index, 1) + }) + moduleTag.unshift(matchedTag, ...deps.map(dep => `- ${dep}`)) + } + }) + }) +} diff --git a/packages/unpacker/src/module-scan/index.ts b/packages/unpacker/src/module-scan/index.ts index 2003b660..bdb854ad 100644 --- a/packages/unpacker/src/module-scan/index.ts +++ b/packages/unpacker/src/module-scan/index.ts @@ -1,28 +1,28 @@ import { ExportManager, ImportManager } from '@unminify-kit/ast-utils' -import { scanBabelRuntime } from './babel-runtime' +import { postScanBabelRuntime, scanBabelRuntime } from './babel-runtime' import type { Module } from '../Module' import type { JSCodeshift } from 'jscodeshift' -export function scanModule(j: JSCodeshift, module: Module) { - scanImports(j, module) - scanExports(j, module) - scanRuntime(j, module) -} - -function scanImports(j: JSCodeshift, module: Module) { +export function scanImports(j: JSCodeshift, module: Module) { const root = module.ast const importManager = new ImportManager() - importManager.collectImportsFromRoot(j, root) + importManager.collectEsModuleImport(j, root) + importManager.collectCommonJsImport(j, root) module.import = importManager.getModuleImports() } -function scanExports(j: JSCodeshift, module: Module) { +export function scanExports(j: JSCodeshift, module: Module) { const root = module.ast const exportManager = new ExportManager() - exportManager.collect(j, root) + exportManager.collectEsModuleExport(j, root) + exportManager.collectCommonJsExport(j, root) module.export = exportManager.toJSON() } -function scanRuntime(j: JSCodeshift, module: Module) { +export function scanRuntime(j: JSCodeshift, module: Module) { scanBabelRuntime(j, module) } + +export function postScanRuntime(j: JSCodeshift, modules: Module[]) { + postScanBabelRuntime(j, modules) +} diff --git a/packages/unpacker/src/unpack.ts b/packages/unpacker/src/unpack.ts index bf6cabfd..9cf7c07d 100644 --- a/packages/unpacker/src/unpack.ts +++ b/packages/unpacker/src/unpack.ts @@ -5,7 +5,8 @@ import getParser from 'jscodeshift/src/getParser' import { getModulesFromBrowserify } from './extractors/browserify' import { getModulesFromWebpack } from './extractors/webpack' import { Module } from './Module' -import type { ModuleMapping } from './ModuleMapping' +import { postScanRuntime, scanExports, scanImports, scanRuntime } from './module-scan' +import type { ModuleMapping } from '@unminify-kit/ast-utils' /** * Unpacks the given source code from supported bundlers. @@ -21,20 +22,30 @@ export function unpack(sourceCode: string): { const result = getModulesFromWebpack(j, root) || getModulesFromBrowserify(j, root) - - if (!result) { // Fallback to a single module - const module = new Module(0, j, root, true) - return { - modules: [module], - moduleIdMapping: {}, + || { + modules: new Set([new Module(0, root, true)]), + moduleIdMapping: { + 0: 'entry.js', + }, } - } const { modules, moduleIdMapping } = result + modules.forEach((module) => { + scanImports(j, module) + scanExports(j, module) + }) + + modules.forEach((module) => { + scanRuntime(j, module) + }) + + const modulesArray = [...modules] + postScanRuntime(j, modulesArray) + return { - modules: [...modules], + modules: modulesArray, moduleIdMapping, } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 876afc7e..ba09e11e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,6 +111,9 @@ importers: '@heroicons/vue': specifier: ^2.0.18 version: 2.0.18(vue@3.3.4) + '@unminify-kit/ast-utils': + specifier: workspace:* + version: link:../ast-utils '@unminify-kit/unminify': specifier: workspace:* version: link:../unminify