From 87f243a5daebd1b1109e6fce4b17a77303dac6e0 Mon Sep 17 00:00:00 2001 From: Uri Shaked Date: Thu, 21 Jun 2018 23:47:23 +0300 Subject: [PATCH] feat(core): implement instrumentation as a TypeScript Compiler Transformer close #41 --- lerna.json | 2 +- packages/typewiz-core/package.json | 2 +- packages/typewiz-core/src/index.ts | 3 +- packages/typewiz-core/src/instrument.spec.ts | 31 +- packages/typewiz-core/src/instrument.ts | 176 +---------- packages/typewiz-core/src/integration.spec.ts | 2 +- packages/typewiz-core/src/transformer.ts | 286 ++++++++++++++++++ .../src/type-collector-snippet.ts | 3 +- packages/typewiz-node/package.json | 4 +- packages/typewiz-webpack/package.json | 4 +- packages/typewiz/package.json | 4 +- packages/typewiz/src/cli.spec.ts | 2 +- 12 files changed, 329 insertions(+), 190 deletions(-) create mode 100644 packages/typewiz-core/src/transformer.ts diff --git a/lerna.json b/lerna.json index 9b9957f..75c61a6 100644 --- a/lerna.json +++ b/lerna.json @@ -5,5 +5,5 @@ "packages/*" ], "useWorkspaces": true, - "version": "1.0.3" + "version": "1.1.0-0" } diff --git a/packages/typewiz-core/package.json b/packages/typewiz-core/package.json index 090b098..82e076c 100644 --- a/packages/typewiz-core/package.json +++ b/packages/typewiz-core/package.json @@ -1,6 +1,6 @@ { "name": "typewiz-core", - "version": "1.0.2", + "version": "1.1.0-0", "main": "dist/index.js", "typings": "dist/index.d.ts", "repository": "https://github.com/urish/typewiz", diff --git a/packages/typewiz-core/src/index.ts b/packages/typewiz-core/src/index.ts index 96f7ab3..61830e6 100644 --- a/packages/typewiz-core/src/index.ts +++ b/packages/typewiz-core/src/index.ts @@ -1,6 +1,7 @@ export * from './apply-types'; -export * from './configuration-parser'; export * from './compiler-helper'; +export * from './configuration-parser'; export * from './instrument'; +export * from './transformer'; export * from './type-collector'; export * from './type-coverage'; diff --git a/packages/typewiz-core/src/instrument.spec.ts b/packages/typewiz-core/src/instrument.spec.ts index 5f6fe48..aeed58e 100644 --- a/packages/typewiz-core/src/instrument.spec.ts +++ b/packages/typewiz-core/src/instrument.spec.ts @@ -1,20 +1,28 @@ +import * as ts from 'typescript'; import { instrument } from './instrument'; +function astPrettyPrint(sourceText: string) { + const printer: ts.Printer = ts.createPrinter(); + return printer.printFile(ts.createSourceFile('test.ts', sourceText, ts.ScriptTarget.Latest)); +} + describe('instrument', () => { it('should instrument function parameters without types', () => { const input = `function (a) { return 5; }`; - expect(instrument(input, 'test.ts')).toMatch(`function (a) {$_$twiz("a",a,11,"test.ts",{}); return 5; }`); + expect(instrument(input, 'test.ts')).toContain( + astPrettyPrint(`function (a) { $_$twiz("a", a, 11, "test.ts", "{}"); return 5; }`), + ); }); - it('should correctly instrument optional function parameters', () => { - const input = `function (a?) { return 5; }`; - expect(instrument(input, 'test.ts')).toMatch(`function (a?) {$_$twiz("a",a,12,"test.ts",{}); return 5; }`); + it('should instrument function with two parameters', () => { + const input = `function (a, b) { return 5; }`; + expect(instrument(input, 'test.ts')).toContain(astPrettyPrint(`$_$twiz("b", b, 14, "test.ts", "{}");`).trim()); }); it('should instrument class method parameters', () => { const input = `class Foo { bar(a) { return 5; } }`; - expect(instrument(input, 'test.ts')).toMatch( - `class Foo { bar(a) {$_$twiz("a",a,17,"test.ts",{}); return 5; } }`, + expect(instrument(input, 'test.ts')).toContain( + astPrettyPrint(`class Foo { bar(a) { $_$twiz("a", a, 17, "test.ts", "{}"); return 5; } }`), ); }); @@ -28,6 +36,15 @@ describe('instrument', () => { expect(instrument(input, 'test.ts')).toMatch(`function (a = 12) { return a; }`); }); + it('should add typewiz declarations', () => { + const input = `function (a) { return 5; }`; + expect(instrument(input, 'test.ts')).toContain( + astPrettyPrint( + `declare function $_$twiz(name: string, value: any, pos: number, filename: string, opts: any): void`, + ), + ); + }); + describe('instrumentCallExpressions', () => { it('should instrument function calls', () => { const input = `foo(bar)`; @@ -36,7 +53,7 @@ describe('instrument', () => { instrumentCallExpressions: true, skipTwizDeclarations: true, }), - ).toMatch(`foo($_$twiz.track(bar,"test.ts",4))`); + ).toMatch(`foo($_$twiz.track(bar, "test.ts", 4))`); }); it('should not instrument numeric arguments in function calls', () => { diff --git a/packages/typewiz-core/src/instrument.ts b/packages/typewiz-core/src/instrument.ts index 9d444b1..38b6333 100644 --- a/packages/typewiz-core/src/instrument.ts +++ b/packages/typewiz-core/src/instrument.ts @@ -1,7 +1,6 @@ import * as ts from 'typescript'; - import { getProgram, ICompilerOptions } from './compiler-helper'; -import { applyReplacements, Replacement } from './replacement'; +import { transformSourceFile } from './transformer'; export interface IInstrumentOptions extends ICompilerOptions { instrumentCallExpressions?: boolean; @@ -16,166 +15,6 @@ export interface IExtraOptions { thisNeedsComma?: boolean; } -function hasParensAroundArguments(node: ts.FunctionLike) { - if (ts.isArrowFunction(node)) { - return ( - node.parameters.length !== 1 || - node - .getText() - .substr(0, node.equalsGreaterThanToken.getStart() - node.getStart()) - .includes('(') - ); - } else { - return true; - } -} - -function visit( - node: ts.Node, - replacements: Replacement[], - fileName: string, - options: IInstrumentOptions, - program?: ts.Program, - semanticDiagnostics?: ReadonlyArray, -) { - const isArrow = ts.isArrowFunction(node); - if (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node) || ts.isArrowFunction(node)) { - if (node.body) { - const needsThisInstrumentation = - options.instrumentImplicitThis && - program && - semanticDiagnostics && - semanticDiagnostics.find((diagnostic) => { - if ( - diagnostic.code === 2683 && - diagnostic.file && - diagnostic.file.fileName === node.getSourceFile().fileName && - diagnostic.start - ) { - if (node.body && ts.isBlock(node.body)) { - const body = node.body as ts.FunctionBody; - return ( - body.statements.find((statement) => { - return ( - diagnostic.start !== undefined && - statement.pos <= diagnostic.start && - diagnostic.start <= statement.end - ); - }) !== undefined - ); - } else { - const body = node.body as ts.Expression; - return body.pos <= diagnostic.start && diagnostic.start <= body.end; - } - } - return false; - }) !== undefined; - if (needsThisInstrumentation) { - const opts: IExtraOptions = { thisType: true }; - if (node.parameters.length > 0) { - opts.thisNeedsComma = true; - } - const params = [ - JSON.stringify('this'), - 'this', - node.parameters.pos, - JSON.stringify(fileName), - JSON.stringify(opts), - ]; - const instrumentExpr = `$_$twiz(${params.join(',')})`; - - replacements.push(Replacement.insert(node.body.getStart() + 1, `${instrumentExpr};`)); - } - } - - const isShortArrow = ts.isArrowFunction(node) && !ts.isBlock(node.body); - for (const param of node.parameters) { - if (!param.type && !param.initializer && node.body) { - const typeInsertionPos = param.name.getEnd() + (param.questionToken ? 1 : 0); - const opts: IExtraOptions = {}; - if (isArrow) { - opts.arrow = true; - } - if (!hasParensAroundArguments(node)) { - opts.parens = [node.parameters[0].getStart(), node.parameters[0].getEnd()]; - } - const params = [ - JSON.stringify(param.name.getText()), - param.name.getText(), - typeInsertionPos, - JSON.stringify(fileName), - JSON.stringify(opts), - ]; - const instrumentExpr = `$_$twiz(${params.join(',')})`; - if (isShortArrow) { - replacements.push(Replacement.insert(node.body.getStart(), `(${instrumentExpr},`)); - replacements.push(Replacement.insert(node.body.getEnd(), `)`, 10)); - } else { - replacements.push(Replacement.insert(node.body.getStart() + 1, `${instrumentExpr};`)); - } - } - } - } - - if ( - options.instrumentCallExpressions && - ts.isCallExpression(node) && - node.expression.getText() !== 'require.context' - ) { - for (const arg of node.arguments) { - if (!ts.isStringLiteral(arg) && !ts.isNumericLiteral(arg) && !ts.isSpreadElement(arg)) { - replacements.push(Replacement.insert(arg.getStart(), '$_$twiz.track(')); - replacements.push(Replacement.insert(arg.getEnd(), `,${JSON.stringify(fileName)},${arg.getStart()})`)); - } - } - } - - if ( - ts.isPropertyDeclaration(node) && - ts.isIdentifier(node.name) && - !node.type && - !node.initializer && - !node.decorators - ) { - const name = node.name.getText(); - const params = [ - JSON.stringify(node.name.getText()), - 'value', - node.name.getEnd() + (node.questionToken ? 1 : 0), - JSON.stringify(fileName), - JSON.stringify({}), - ]; - const instrumentExpr = `$_$twiz(${params.join(',')});`; - const preamble = ` - get ${name}() { return this._twiz_private_${name}; } - set ${name}(value: any) { ${instrumentExpr} this._twiz_private_${name} = value; } - `; - // we need to remove any readonly modifiers, otherwise typescript will not let us update - // our _twiz_private_... variable inside the setter. - for (const modifier of node.modifiers || []) { - if (modifier.kind === ts.SyntaxKind.ReadonlyKeyword) { - replacements.push(Replacement.delete(modifier.getStart(), modifier.getEnd())); - } - } - if (node.getStart() === node.name.getStart()) { - replacements.push(Replacement.insert(node.getStart(), `${preamble} _twiz_private_`)); - } else { - replacements.push(Replacement.insert(node.name.getStart(), '_twiz_private_')); - replacements.push(Replacement.insert(node.getStart(), `${preamble}`)); - } - } - - node.forEachChild((child) => visit(child, replacements, fileName, options, program, semanticDiagnostics)); -} - -const declaration = ` - declare function $_$twiz(name: string, value: any, pos: number, filename: string, opts: any): void; - declare namespace $_$twiz { - function track(value: T, filename: string, offset: number): T; - function track(value: any, filename: string, offset: number): any; - } -`; - export function instrument(source: string, fileName: string, options?: IInstrumentOptions) { const instrumentOptions: IInstrumentOptions = { instrumentCallExpressions: false, @@ -183,17 +22,12 @@ export function instrument(source: string, fileName: string, options?: IInstrume skipTwizDeclarations: false, ...options, }; + const program: ts.Program | undefined = getProgram(instrumentOptions); const sourceFile = program ? program.getSourceFile(fileName) : ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); - const replacements = [] as Replacement[]; - if (sourceFile) { - const semanticDiagnostics = program ? program.getSemanticDiagnostics(sourceFile) : undefined; - visit(sourceFile, replacements, fileName, instrumentOptions, program, semanticDiagnostics); - } - if (replacements.length && !instrumentOptions.skipTwizDeclarations) { - replacements.push(Replacement.insert(0, declaration)); - } - return applyReplacements(source, replacements); + + const transformed = transformSourceFile(sourceFile, options, program); + return ts.createPrinter().printFile(transformed); } diff --git a/packages/typewiz-core/src/integration.spec.ts b/packages/typewiz-core/src/integration.spec.ts index 422004a..3f71c26 100644 --- a/packages/typewiz-core/src/integration.spec.ts +++ b/packages/typewiz-core/src/integration.spec.ts @@ -51,7 +51,7 @@ function typeWiz(input: string, typeCheck = false, options?: IApplyTypesOptions) if (mockFs.writeFileSync.mock.calls.length) { expect(mockFs.writeFileSync).toHaveBeenCalledTimes(1); - expect(mockFs.writeFileSync).toHaveBeenCalledWith('c:\\test.ts', expect.any(String)); + expect(mockFs.writeFileSync).toHaveBeenCalledWith('c:/test.ts', expect.any(String)); return mockFs.writeFileSync.mock.calls[0][1]; } else { return null; diff --git a/packages/typewiz-core/src/transformer.ts b/packages/typewiz-core/src/transformer.ts new file mode 100644 index 0000000..7852bd8 --- /dev/null +++ b/packages/typewiz-core/src/transformer.ts @@ -0,0 +1,286 @@ +import * as ts from 'typescript'; +import { IExtraOptions, IInstrumentOptions } from './instrument'; + +const declaration = ` + declare function $_$twiz(name: string, value: any, pos: number, filename: string, opts: any): void; + declare namespace $_$twiz { + function track(value: T, filename: string, offset: number): T; + function track(value: any, filename: string, offset: number): any; + } +`; + +function getDeclarationStatements() { + const sourceFile = ts.createSourceFile('twiz-declarations.ts', declaration, ts.ScriptTarget.Latest); + return sourceFile.statements; +} + +function updateFunction(node: ts.FunctionDeclaration, instrumentStatements: ReadonlyArray) { + return ts.updateFunctionDeclaration( + node, + node.decorators, + node.modifiers, + node.asteriskToken, + node.name, + node.typeParameters, + node.parameters, + node.type, + ts.createBlock([...instrumentStatements, ...(node.body ? node.body.statements : [])]), + ); +} + +function updateMethod(node: ts.MethodDeclaration, instrumentStatements: ReadonlyArray) { + return ts.updateMethod( + node, + node.decorators, + node.modifiers, + node.asteriskToken, + node.name, + node.questionToken, + node.typeParameters, + node.parameters, + node.type, + ts.createBlock([...instrumentStatements, ...(node.body ? node.body.statements : [])]), + ); +} + +function updateArrow(node: ts.ArrowFunction, instrumentStatements: ReadonlyArray) { + const oldBody = ts.isBlock(node.body) ? node.body.statements : [ts.createStatement(node.body)]; + return ts.updateArrowFunction( + node, + node.modifiers, + node.typeParameters, + node.parameters, + node.type, + ts.createBlock([...instrumentStatements, ...oldBody]), + ); +} + +function hasParensAroundArguments(node: ts.FunctionLike) { + if (ts.isArrowFunction(node)) { + return ( + node.parameters.length !== 1 || + node + .getText() + .substr(0, node.equalsGreaterThanToken.getStart() - node.getStart()) + .includes('(') + ); + } else { + return true; + } +} + +function isRequireContextExpression(node: ts.Expression) { + return ( + ts.isPropertyAccessExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === 'require' && + ts.isIdentifier(node.name) && + node.name.text === 'context' + ); +} + +function needsThisInstrumentation( + node: ts.FunctionDeclaration | ts.ArrowFunction | ts.MethodDeclaration, + semanticDiagnostics?: ReadonlyArray, +) { + return ( + semanticDiagnostics && + semanticDiagnostics.find((diagnostic) => { + if ( + diagnostic.code === 2683 && + diagnostic.file && + diagnostic.file.fileName === node.getSourceFile().fileName && + diagnostic.start + ) { + if (node.body && ts.isBlock(node.body)) { + const body = node.body as ts.FunctionBody; + return ( + body.statements.find((statement) => { + return ( + diagnostic.start !== undefined && + statement.pos <= diagnostic.start && + diagnostic.start <= statement.end + ); + }) !== undefined + ); + } else { + const body = node.body as ts.Expression; + return body.pos <= diagnostic.start && diagnostic.start <= body.end; + } + } + return false; + }) !== undefined + ); +} + +function createTwizInstrumentStatement(name: string, fileOffset: number, filename: string, opts: IExtraOptions) { + return ts.createStatement( + ts.createCall( + ts.createIdentifier('$_$twiz'), + [], + [ + ts.createLiteral(name), + ts.createIdentifier(name), + ts.createNumericLiteral(fileOffset.toString()), + ts.createLiteral(filename), + ts.createLiteral(JSON.stringify(opts)), + ], + ), + ); +} + +function visitorFactory( + ctx: ts.TransformationContext, + source: ts.SourceFile, + options: IInstrumentOptions, + semanticDiagnostics?: ReadonlyArray, +) { + const visitor: ts.Visitor = (originalNode: ts.Node): ts.Node | ts.Node[] => { + const node = ts.visitEachChild(originalNode, visitor, ctx); + + if (ts.isSourceFile(node)) { + return ts.updateSourceFileNode(node, [...getDeclarationStatements(), ...node.statements]); + } + + const isArrow = ts.isArrowFunction(node); + if (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node) || ts.isArrowFunction(node)) { + const instrumentStatements: ts.Statement[] = []; + if (options.instrumentImplicitThis && needsThisInstrumentation(node, semanticDiagnostics)) { + const opts: IExtraOptions = { thisType: true }; + if (node.parameters.length > 0) { + opts.thisNeedsComma = true; + } + + instrumentStatements.push( + createTwizInstrumentStatement('this', node.parameters.pos, source.fileName, opts), + ); + } + + for (const param of node.parameters) { + if (!param.type && !param.initializer && node.body) { + const typeInsertionPos = param.name.getEnd() + (param.questionToken ? 1 : 0); + const opts: IExtraOptions = {}; + if (isArrow) { + opts.arrow = true; + } + if (!hasParensAroundArguments(node)) { + opts.parens = [node.parameters[0].getStart(), node.parameters[0].getEnd()]; + } + instrumentStatements.push( + createTwizInstrumentStatement(param.name.getText(), typeInsertionPos, source.fileName, opts), + ); + } + } + if (ts.isFunctionDeclaration(node)) { + return updateFunction(node, instrumentStatements); + } + if (ts.isMethodDeclaration(node)) { + return updateMethod(node, instrumentStatements); + } + if (ts.isArrowFunction(node)) { + return updateArrow(node, instrumentStatements); + } + } + + if (options.instrumentCallExpressions && ts.isCallExpression(node) && !isRequireContextExpression(node)) { + const newArguments = []; + for (const arg of node.arguments) { + if (!ts.isStringLiteral(arg) && !ts.isNumericLiteral(arg) && !ts.isSpreadElement(arg)) { + newArguments.push( + ts.createCall( + ts.createPropertyAccess(ts.createIdentifier('$_$twiz'), ts.createIdentifier('track')), + undefined, + [ + arg, + ts.createLiteral(source.fileName), + ts.createNumericLiteral(arg.getStart().toString()), + ], + ), + ); + } else { + newArguments.push(arg); + } + } + + return ts.updateCall(node, node.expression, node.typeArguments, newArguments); + } + + if ( + ts.isPropertyDeclaration(node) && + ts.isIdentifier(node.name) && + !node.type && + !node.initializer && + !node.decorators + ) { + const privatePropName = '_twiz_private_' + node.name.text; + const typeInsertionPos = node.name.getEnd() + (node.questionToken ? 1 : 0); + return [ + // dummy property + ts.updateProperty( + node, + undefined, + [ts.createToken(ts.SyntaxKind.PrivateKeyword)], + privatePropName, + node.questionToken, + node.type, + node.initializer, + ), + + // getter + ts.createGetAccessor( + node.decorators, + undefined, + node.name, + [], + node.type, + ts.createBlock([ts.createReturn(ts.createPropertyAccess(ts.createThis(), privatePropName))]), + ), + + // setter + ts.createSetAccessor( + undefined, + undefined, + node.name, + [ + ts.createParameter( + undefined, + undefined, + undefined, + node.name.text, + node.type, + ts.createKeywordTypeNode(ts.SyntaxKind.AnyKeyword), + ), + ], + ts.createBlock([ + createTwizInstrumentStatement(node.name.text, typeInsertionPos, source.fileName, {}), + // assign value to privatePropName + ts.createStatement( + ts.createAssignment( + ts.createPropertyAccess(ts.createThis(), privatePropName), + ts.createIdentifier(node.name.text), + ), + ), + ]), + ), + ]; + } + + return node; + }; + + return visitor; +} + +export function typewizTransformer(options: IInstrumentOptions, program?: ts.Program) { + return (ctx: ts.TransformationContext): ts.Transformer => { + return (source: ts.SourceFile) => { + const semanticDiagnostics = + options.instrumentImplicitThis && program ? program.getSemanticDiagnostics(source) : undefined; + return ts.visitNode(source, visitorFactory(ctx, source, options, semanticDiagnostics)); + }; + }; +} + +export function transformSourceFile(sourceFile: ts.SourceFile, options: IInstrumentOptions = {}, program?: ts.Program) { + return ts.transform(sourceFile, [typewizTransformer(options, program)]).transformed[0]; +} diff --git a/packages/typewiz-core/src/type-collector-snippet.ts b/packages/typewiz-core/src/type-collector-snippet.ts index f74e24c..810942e 100644 --- a/packages/typewiz-core/src/type-collector-snippet.ts +++ b/packages/typewiz-core/src/type-collector-snippet.ts @@ -120,7 +120,8 @@ function escapeSpecialKey(key: string) { const logs: { [key: string]: Set } = {}; const trackedObjects = new WeakMap(); -export function $_$twiz(name: string, value: any, pos: number, filename: string, opts: ICollectedTypeInfo) { +export function $_$twiz(name: string, value: any, pos: number, filename: string, optsJson: string) { + const opts = JSON.parse(optsJson) as ICollectedTypeInfo; const objectDeclaration = trackedObjects.get(value); const index = JSON.stringify({ filename, pos, opts } as IKey); try { diff --git a/packages/typewiz-node/package.json b/packages/typewiz-node/package.json index 81f3ce8..f61b339 100644 --- a/packages/typewiz-node/package.json +++ b/packages/typewiz-node/package.json @@ -1,6 +1,6 @@ { "name": "typewiz-node", - "version": "1.0.3", + "version": "1.1.0-0", "bin": { "typewiz-node": "dist/typewiz-node.js" }, @@ -22,7 +22,7 @@ }, "dependencies": { "ts-node": "^5.0.1", - "typewiz-core": "^1.0.2", + "typewiz-core": "^1.1.0-0", "util.promisify": "^1.0.0" }, "devDependencies": { diff --git a/packages/typewiz-webpack/package.json b/packages/typewiz-webpack/package.json index bc966b3..cbf5fb2 100644 --- a/packages/typewiz-webpack/package.json +++ b/packages/typewiz-webpack/package.json @@ -1,6 +1,6 @@ { "name": "typewiz-webpack", - "version": "1.0.2", + "version": "1.1.0-0", "main": "dist/index.js", "typings": "dist/index.d.ts", "repository": "https://github.com/urish/typewiz", @@ -19,7 +19,7 @@ }, "dependencies": { "loader-utils": "^1.1.0", - "typewiz-core": "^1.0.2", + "typewiz-core": "^1.1.0-0", "util.promisify": "^1.0.0", "webpack-sources": "^1.1.0" }, diff --git a/packages/typewiz/package.json b/packages/typewiz/package.json index 0cf3fd2..288d9c2 100644 --- a/packages/typewiz/package.json +++ b/packages/typewiz/package.json @@ -1,6 +1,6 @@ { "name": "typewiz", - "version": "1.0.3", + "version": "1.1.0-0", "main": "dist/cli.js", "bin": "dist/cli.js", "typings": "dist/cli.d.ts", @@ -19,7 +19,7 @@ ], "dependencies": { "commander": "^2.15.1", - "typewiz-core": "^1.0.2", + "typewiz-core": "^1.1.0-0", "update-notifier": "^2.4.0" }, "engines": { diff --git a/packages/typewiz/src/cli.spec.ts b/packages/typewiz/src/cli.spec.ts index 2845238..6a0d400 100644 --- a/packages/typewiz/src/cli.spec.ts +++ b/packages/typewiz/src/cli.spec.ts @@ -38,7 +38,7 @@ describe('typewiz CLI', () => { 'should instrument the given source file', async () => { const output = await runCli(['instrument', path.join(__dirname, '../fixtures/instrument-test.ts')]); - expect(output.trim()).toContain('function f(x) {$_$twiz('); + expect(output.trim()).toContain('function f(x) { $_$twiz('); }, 10000, );