From 583eca0dc4132b6e0bbe04e211a28152213d7848 Mon Sep 17 00:00:00 2001 From: Bruno Jouhier Date: Fri, 6 Mar 2015 13:30:07 +0100 Subject: [PATCH] issue #254 - esprima fibers fast transform is ready - passes all unit tests --- lib/fibers-fast/transform-esprima.js | 378 +++++++++++++++++++++++++++ lib/fibers-fast/transform.js | 1 + 2 files changed, 379 insertions(+) create mode 100644 lib/fibers-fast/transform-esprima.js diff --git a/lib/fibers-fast/transform-esprima.js b/lib/fibers-fast/transform-esprima.js new file mode 100644 index 00000000..c9d4b19c --- /dev/null +++ b/lib/fibers-fast/transform-esprima.js @@ -0,0 +1,378 @@ +// Copyright 2011 Marcel Laverdet +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +"use strict"; +if (typeof exports !== 'undefined') { + var esprima = require('esprima'); + var sourceMap = require('../util/source-map'); +} +(function(exports) { + exports.transform = transform; + exports.version = require("../version").version + " (fibers-fast --esprima)"; + // hack to fix #123 + exports.transform.version = exports.version; + +var Walker = require('../fibers/walker-esprima'); + +function aggressiveReplace(source) { + // arr.forEach_(_, function(_, item) {...}) --> arr.forEach(function(item, __, _) {...}) + // The __ parameter captures index param (2 chars so that replace preserves length) + // we append _ to callback args list because transform needs async marker on callback. + function testArg(arg) { + return new RegExp('\\b' + arg + '\\b').test(source) ? null : arg; + } + function unusedIdent() { + var arg; + if (arg = testArg('__')) return arg; + for (var i = 0; i < 26 && !arg; i++) { + if (arg = testArg(String.fromCharCode(0x41) + '_')) return arg; + if (arg = testArg('_' + String.fromCharCode(0x41))) return arg; + if (arg = testArg(String.fromCharCode(0x61) + '_')) return arg; + if (arg = testArg('_' + String.fromCharCode(0x61))) return arg; + } + return null; + } + var arg = unusedIdent(); + if (!arg) { + console.log("cannot apply aggressive optimization!!"); + return source; + } else { + return source.replace(/\.(forEach|map|filter|every|some|reduce)_\(_, function\(_, ([^)]+)\)/g, function(all, name, params) { + return '.' + name + '(function(' + params + ', ' + arg + ', _)'; + }); + } +} + +/** + * Transforms code to be streamliney. Line numbers are not maintained, but could be if I could + * figure out how to do it with uglifyjs. + */ +function transform(source, options) { + source = source.replace(/\r\n/g, "\n"); + options = options || {}; + if (options.aggressive) source = aggressiveReplace(source); + var callback = options.callback || '_'; + var didRewrite = 0; + var position = 0; + var buffer = ''; + var async = false; + + function isAsyncArg(arg) { + return arg.type === 'UnaryExpression' && arg.operator === '~' && arg.argument.type === 'Identifier' && arg.argument.name === callback; + } + function isArrayArg(arg) { + return arg.type === 'ArrayExpression' && arg.elements.length === 1 && arg.elements[0].type === 'Identifier' && arg.elements[0].name === callback; + } + function isLShiftArg(arg) { + return arg.type === 'BinaryExpression' && arg.operator === '<<' && // + arg.left.type === 'Identifier' && arg.left.name === callback; + } + function isRShiftArg(arg) { + return arg.type === 'BinaryExpression' && arg.operator === '>>' && // + arg.left.type === 'Identifier' && arg.left.name === callback; + } + function isFutureArg(arg) { + return arg.type === 'UnaryExpression' && (arg.operator === '!' || arg.operator === 'void') // + && arg.argument.type === 'Identifier' && arg.argument.name === callback; + } + function isAsyncParam(param) { + return param === callback || (param.type === 'Identifier' && param.name === callback); + } + /** + * Finds the index of the callback param in an argument list, -1 if not found. + */ + function getCallback(args, testFn, lineno) { + var idx = -1; + for (var ii = 0; ii < args.length; ++ii) { + if (testFn(args[ii])) { + if (idx === -1) { + idx = ii; + } else { + lineno = lineno || args[ii].loc.start.line; + throw new Error('Callback argument used more than once in function call on line '+ lineno); + } + } + } + return idx; + } + + /** + * Adds to `buffer` everything that hasn't been rendered so far. + */ + function catchup(end) { + if (end < position || end === undefined) { + throw new Error('Invalid catchup, '+ position+ ' to '+ end); + } + buffer += source.substring(position, end); + position = end; + } + + function skipTo(pos) { + buffer += source.substring(position, pos).replace(/\S/g, ''); + position = pos; + } + + function startsWith(str, start, pat) { + return str.substring(start, start + pat.length) === pat; + } + + function endsWith(str, end, pat) { + return str.substring(end - pat.length, end) === pat; + } + + var walk = Walker({ + Function: function(name, args, body) { + // Open this function + if (name === callback) { + throw new Error('Invalid usage of callback on line '+ this.loc.start.line); + } + var idx = getCallback(args, isAsyncParam, this.loc.start.line); + catchup(this.body.range[0] + 1); + var oldAsync = async; + async = idx !== -1 || this.forceAsync; + walk(this.body); + async = oldAsync; + }, + CallExpression: function(expr, args) { + if (expr.type === 'Identifier' && expr.name === callback && args.length === 2) { + catchup(expr.range[0]); + buffer += 'fstreamline__.star'; + skipTo(expr.range[1]); + args.map(walk); + ++didRewrite; + return; + } + if (expr.type === 'MemberExpression' && !expr.computed && args.length === 2 && args[0].type === 'Identifier' && args[0].name === callback // + && args[1].type === 'Identifier' && args[1].name === callback) { + if (!async) throw error(this.loc.start.line, "Function contains async calls but does not have _ parameter"); + catchup(this.range[0]); + buffer += ('fstreamline__.then.call(this,'); + skipTo(expr.object.range[0]); + walk(expr.object); + catchup(expr.object.range[1]); + buffer += (', "' + expr.property.name + '", _)'); + skipTo(this.range[1]); + ++didRewrite; + return; + } + var idxFast = getCallback(args, isAsyncParam); + var idx = getCallback(args, isAsyncArg); + var idxArray = getCallback(args, isArrayArg); + if (idx === -1) idx = idxArray; + if ((idxFast !== -1 || idx !== -1) && !async) throw new Error("Function contains async calls but does not have _ parameter on line " + this.loc.start.line); + function walkExpr() { + skipTo(expr.range[0]); + if (expr.type === 'MemberExpression') { + if (!expr.computed) { + // Method call: foo.bar(_) + walk(expr.object); + catchup(expr.object.range[1]); + buffer += ', '+ JSON.stringify(expr.property.name); + } else { + // Dynamic method call: foo[bar](_) + walk(expr.object); + catchup(expr.object.range[1]); + buffer += ', '; + skipTo(expr.property.range[0]); + walk(expr.property); + catchup(expr.property.range[1]); + } + skipTo(expr.range[1]); + } else { + // Function call + buffer += 'null, '; + walk(expr); + catchup(expr.range[1]); + } + } + if (idx !== -1) { + // Rewrite streamlined calls + // issue #108: process between expr.range[0] and last arg end rather than this.range[0]/end + catchup(this.range[0]); + buffer += 'fstreamline__.invoke('; + walkExpr(); + // Render arguments + buffer += ', ['; + skipTo(args[0].range[0]); + for (var ii = 0; ii < args.length; ++ii) { + catchup(args[ii].range[0]); + if (ii !== idx) { + walk(args[ii]); + catchup(args[ii].range[1]); + } else { + buffer += '_'; + skipTo(args[ii].range[1]); + } + } + var options = idx; + if (idxArray !== -1) options = '{ callbackIndex: ' + idx + ', returnArray: true }'; + buffer += '], '+ options + ')'; + skipTo(this.range[1]); + ++didRewrite; + } else if ((idx = getCallback(args, isRShiftArg)) !== -1) { + catchup(this.range[0]); + skipTo(expr.range[0]); + if (expr.type === 'MemberExpression' && !expr.computed) { + buffer += 'fstreamline__.createBound('; + walk(expr.object); + catchup(expr.object.range[1]); + buffer += ",'" + expr.property.name + "'"; + skipTo(expr.range[1]); + } else { + buffer += 'fstreamline__.create('; + walk(expr); + catchup(expr.range[1]); + } + buffer += ',' + idx + ')('; + for (var ii = 0; ii < args.length; ++ii) { + skipTo(args[ii].range[0]); + if (ii > 0) buffer += ', '; + if (ii !== idx) { + walk(args[ii]); + catchup(args[ii].range[1]); + } else { + var arg = args[ii].right; + skipTo(arg.range[0]); + walk(arg); + catchup(arg.range[1]); + skipTo(args[ii].range[1]); + } + } + buffer += ')'; + skipTo(this.range[1]); + ++didRewrite; + } else if ((idx = getCallback(args, isLShiftArg)) !== -1) { + catchup(this.range[0]); + skipTo(expr.range[0]); + walk(expr); + catchup(expr.range[1]); + buffer += '('; + for (var ii = 0; ii < args.length; ++ii) { + skipTo(args[ii].range[0]); + if (ii > 0) buffer += ', '; + if (ii !== idx) { + walk(args[ii]); + catchup(args[ii].range[1]); + } else { + var arg = args[ii].right; + if (arg.type !== 'FunctionExpression') throw new Error("Expected function after _ << "); + var idx2 = getCallback(arg.params, isAsyncParam); + if (idx2 === -1) throw new Error("Expected async function after _ << "); + buffer += 'fstreamline__.create('; + skipTo(arg.range[0]); + walk(arg); + catchup(arg.range[1]); + buffer += ',' + idx2 + ', true)' + skipTo(args[ii].range[1]); + } + } + buffer += ')'; + skipTo(this.range[1]); + ++didRewrite; + } else if ((idx = getCallback(args, isFutureArg)) !== -1) { + var isPromise = args[idx].operator === "void"; + catchup(this.range[0]); + buffer += 'fstreamline__.' + (isPromise ? 'promise' : 'spin') + '('; + walkExpr(); + buffer += ', ['; + for (var ii = 0; ii < args.length; ++ii) { + skipTo(args[ii].range[0]); + if (ii > 0) buffer += ', '; + if (ii !== idx) { + walk(args[ii]); + catchup(args[ii].range[1]); + } else { + catchup(args[ii].range[0]); + buffer += isPromise ? 'null' : 'false'; + skipTo(args[ii].range[1]); + } + } + catchup(args[args.length - 1].range[1]); + buffer += '], '+ idx + ')'; + skipTo(this.range[1]); + ++didRewrite; + } else { + var paren = 0; + if (source[this.range[0]] === '(' && source[this.range[0] + 1] === '(' && + source[this.range[1] - 1] === ')' && source[this.range[1] - 2] === ')') { + paren = 1; + } + if (startsWith(source, this.range[0] + paren, '(function() {')) { + // handle coffeescript wrappers: set the forceAsync flag + // so that we don't get an error about _ being used inside non async function + if (endsWith(source, this.range[1] - paren, '})()')) { + expr.forceAsync = async; + } + if (endsWith(source, this.range[1] - paren, '}).call(this)') // + || endsWith(source, this.range[1] - paren, '}).call(_this)') // + || endsWith(source, this.range[1] - paren, '}).apply(this, arguments)')) { + expr.object.forceAsync = async; + } + } + walk(expr); + args.map(walk); + } + }, + Property: function() { + // Dont't walk the property key, because that's an identifier and it will be clobbered, per + // the below code + walk(this.value); + }, + MemberExpression: function() { + // See comment above for Property + walk(this.object); + if (this.computed) walk(this.property) + }, + UnaryExpression: function(operator, argument) { + if (operator === '!' && argument.type === 'Identifier' && argument.name === callback) { + catchup(this.range[0]); + buffer += 'null'; + skipTo(this.range[1]); + } else { + walk(argument); + } + }, + }); + + // Walk parsed source, rendering along the way + var originalSource = source; + var optStr = "(function(){"; + if (options.promise) { + var arg = typeof options.promise === "string" ? "'" + options.promise + "'" : "true"; + optStr += "fstreamline__.globals.setPromise(" + arg + ");"; + } + optStr += "})();"; + source = 'var fstreamline__ = require("' + (options.internal ? '..' : 'streamline/lib') + '/fibers-fast/runtime");' + optStr + ' (function(_) { ' + // + source + // + '\n})(_ >> function(err) {\n' + // + ' if (err) throw err;\n' + // + '});\n'; + var parsed = esprima.parse(source, { + loc: true, + range: true, + }); + walk(parsed); + buffer += source.substring(position); + + if (didRewrite > 0) { + return buffer; + } else { + return originalSource; + } +} +})(typeof exports !== 'undefined' ? exports : (window.Streamline = window.Streamline || {})); diff --git a/lib/fibers-fast/transform.js b/lib/fibers-fast/transform.js index 8bc416a5..0e37382f 100644 --- a/lib/fibers-fast/transform.js +++ b/lib/fibers-fast/transform.js @@ -17,6 +17,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS // IN THE SOFTWARE. "use strict"; +return module.exports = require('./transform-esprima'); if (typeof exports !== 'undefined') { var Narcissus = require('../../deps/narcissus'); }