Skip to content

Commit

Permalink
Performance optimizations (#1822)
Browse files Browse the repository at this point in the history 
Big thank you to @paulrutter for contributing his findings to the community
  • Loading branch information
@paulrutter
paulrutter authored Nov 9, 2023
1 parent 74d475f commit 11e4fe7
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 96 deletions.
87 changes: 45 additions & 42 deletions src/15utility.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
/*jshint unused:false*/
/*
Utilities for Alasql.js
Utilities for Alasql.js
@todo Review the list of utilities
@todo Find more effective utilities
@todo Review the list of utilities
@todo Find more effective utilities
*/

/**
Expand Down Expand Up @@ -59,7 +59,7 @@ function returnTrue() {
@function
@return {undefined} Always undefined
*/
function returnUndefined() {}
function returnUndefined() { }

/**
Escape string
Expand Down Expand Up @@ -354,7 +354,7 @@ var loadFile = (utils.loadFile = function (path, asy, success, error) {
} else if (utils.isCordova) {
/* If Cordova */
utils.global.requestFileSystem(LocalFileSystem.PERSISTENT, 0, function (fileSystem) {
fileSystem.root.getFile(path, {create: false}, function (fileEntry) {
fileSystem.root.getFile(path, { create: false }, function (fileEntry) {
fileEntry.file(function (file) {
var fileReader = new FileReader();
fileReader.onloadend = function (e) {
Expand Down Expand Up @@ -565,7 +565,7 @@ var removeFile = (utils.removeFile = function (path, cb) {
utils.global.requestFileSystem(LocalFileSystem.PERSISTENT, 0, function (fileSystem) {
fileSystem.root.getFile(
path,
{create: false},
{ create: false },
function (fileEntry) {
fileEntry.remove(cb);
cb && cb(); // jshint ignore:line
Expand Down Expand Up @@ -633,7 +633,7 @@ var fileExists = (utils.fileExists = function (path, cb) {
utils.global.requestFileSystem(LocalFileSystem.PERSISTENT, 0, function (fileSystem) {
fileSystem.root.getFile(
path,
{create: false},
{ create: false },
function (fileEntry) {
cb(true);
},
Expand Down Expand Up @@ -699,7 +699,7 @@ var saveFile = (utils.saveFile = function (path, data, cb, opts) {
} else if (utils.isCordova) {
utils.global.requestFileSystem(LocalFileSystem.PERSISTENT, 0, function (fileSystem) {
// alasql.utils.removeFile(path,function(){
fileSystem.root.getFile(path, {create: true}, function (fileEntry) {
fileSystem.root.getFile(path, { create: true }, function (fileEntry) {
fileEntry.createWriter(function (fileWriter) {
fileWriter.onwriteend = function () {
if (cb) {
Expand Down Expand Up @@ -774,7 +774,7 @@ var saveFile = (utils.saveFile = function (path, data, cb, opts) {
disableAutoBom: false,
};
alasql.utils.extend(opt, opts);
var blob = new Blob([data], {type: 'text/plain;charset=utf-8'});
var blob = new Blob([data], { type: 'text/plain;charset=utf-8' });
saveAs(blob, path, opt.disableAutoBom);
if (cb) {
res = cb(res);
Expand Down Expand Up @@ -1175,43 +1175,46 @@ var domEmptyChildren = (utils.domEmptyChildren = function (container) {
@parameter {string} escape Escape character (optional)
@return {boolean} If value LIKE pattern ESCAPE escape
*/

var patternCache = {};
var like = (utils.like = function (pattern, value, escape) {
// Verify escape character
if (!escape) escape = '';

var i = 0;
var s = '^';

while (i < pattern.length) {
var c = pattern[i],
c1 = '';
if (i < pattern.length - 1) c1 = pattern[i + 1];

if (c === escape) {
s += '\\' + c1;
i++;
} else if (c === '[' && c1 === '^') {
s += '[^';
if (!patternCache[pattern]) {
// Verify escape character
if (!escape) escape = '';

var i = 0;
var s = '^';

while (i < pattern.length) {
var c = pattern[i],
c1 = '';
if (i < pattern.length - 1) c1 = pattern[i + 1];

if (c === escape) {
s += '\\' + c1;
i++;
} else if (c === '[' && c1 === '^') {
s += '[^';
i++;
} else if (c === '[' || c === ']') {
s += c;
} else if (c === '%') {
s += '[\\s\\S]*';
} else if (c === '_') {
s += '.';
} else if ('/.*+?|(){}'.indexOf(c) > -1) {
s += '\\' + c;
} else {
s += c;
}
i++;
} else if (c === '[' || c === ']') {
s += c;
} else if (c === '%') {
s += '[\\s\\S]*';
} else if (c === '_') {
s += '.';
} else if ('/.*+?|(){}'.indexOf(c) > -1) {
s += '\\' + c;
} else {
s += c;
}
i++;
}

s += '$';
// if(value == undefined) return false;
//console.log(s,value,(value||'').search(RegExp(s))>-1);
return ('' + (value ?? '')).search(RegExp(s, 'i')) > -1;
s += '$';
// if(value == undefined) return false;
//console.log(s,value,(value||'').search(RegExp(s))>-1);
patternCache[pattern] = RegExp(s, 'i');
}
return ('' + (value ?? '')).search(patternCache[pattern]) > -1;
});

utils.glob = function (value, pattern) {
Expand Down
52 changes: 32 additions & 20 deletions src/17alasql.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,32 @@ alasql.parser.parseError = function (str, hash) {
};

/**
Jison parser
@param {string} sql SQL statement
@return {object} AST (Abstract Syntax Tree)
Jison parser
@param {string} sql SQL statement
@return {object} AST (Abstract Syntax Tree)
@todo Create class AST
@todo Add other parsers
@todo Create class AST
@todo Add other parsers
@example
alasql.parse = function(sql) {
@example
alasql.parse = function(sql) {
// My own parser here
}
}
*/
alasql.parse = function (sql) {
return alasqlparser.parse(alasql.utils.uncomment(sql));
};

/**
List of engines of external databases
@type {object}
@todo Create collection type
List of engines of external databases
@type {object}
@todo Create collection type
*/
alasql.engines = {};

/**
List of databases
@type {object}
List of databases
@type {object}
*/
alasql.databases = {};

Expand All @@ -51,7 +51,7 @@ alasql.databases = {};
alasql.databasenum = 0;

/**
Alasql options object
Alasql options object
*/
alasql.options = {
/** Log or throw error */
Expand Down Expand Up @@ -119,12 +119,15 @@ alasql.options = {
/** Check for NaN and convert it to undefined */
nan: false,

excel: {cellDates: true},
excel: { cellDates: true },

/** Option for SELECT * FROM a,b */
joinstar: 'overwrite',

loopbreak: 100000,

/** Whether GETDATE() and NOW() return dates as string. If false, then a Date object is returned */
dateAsString: true,
};

//alasql.options.worker = false;
Expand Down Expand Up @@ -207,7 +210,7 @@ alasql.autoval = function (tablename, colname, getNext, databaseid) {

return (
db.tables[tablename].identities[colname].value -
db.tables[tablename].identities[colname].step || null
db.tables[tablename].identities[colname].step || null
);
};

Expand Down Expand Up @@ -246,19 +249,28 @@ alasql.dexec = function (databaseid, sql, params, cb, scope) {
// if(db.databaseid != databaseid) console.trace('got!');
// console.log(3,db.databaseid,databaseid);

var hh;
var hh = hash(sql);

// Create hash
if (alasql.options.cache) {
hh = hash(sql);
var statement = db.sqlCache[hh];
// If database structure was not changed since last time return cache
if (statement && db.dbversion === statement.dbversion) {
return statement(params, cb);
}
}

// Create AST
var ast = alasql.parse(sql);
var ast = db.astCache[hh];
if (alasql.options.cache && !ast) {
// Create AST cache
ast = alasql.parse(sql);
if (ast) {
// add to AST cache
db.astCache[hh] = ast;
}
} else {
ast = alasql.parse(sql);
}
if (!ast.statements) {
return;
}
Expand Down
1 change: 1 addition & 0 deletions src/20database.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ var Database = (alasql.Database = function (databaseid) {
Database.prototype.resetSqlCache = function () {
this.sqlCache = {}; // Cache for compiled SQL statements
this.sqlCacheSize = 0;
this.astCache = {}; // Cache for AST objects
};

// Main SQL function
Expand Down
34 changes: 23 additions & 11 deletions src/50expression.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,14 +483,17 @@
s += '.indexOf(';
s += 'alasql.utils.getValueOf(' + leftJS() + '))>-1)';
} else if (Array.isArray(this.right)) {
// if(this.right.length == 0) return 'false';
s =
'([' +
this.right.map(ref).join(',') +
'].indexOf(alasql.utils.getValueOf(' +
leftJS() +
'))>-1)';
//console.log(s);
// Added patch to have a better performance for when you have a lot of entries in an IN statement
if (!alasql.sets) {
alasql.sets = {};
}
const allValues = this.right.map((value) => value.value);
const allValuesStr = allValues.join(",");
if (!alasql.sets[allValuesStr]) {
// leverage JS Set, which is faster for lookups than arrays
alasql.sets[allValuesStr] = new Set(allValues);
}
s = 'alasql.sets["' + allValuesStr + '"].has(alasql.utils.getValueOf(' + leftJS() + '))';
} else {
s = '(' + rightJS() + '.indexOf(' + leftJS() + ')>-1)';
//console.log('expression',350,s);
Expand All @@ -506,8 +509,17 @@
s += '.indexOf(';
s += 'alasql.utils.getValueOf(' + leftJS() + '))<0)';
} else if (Array.isArray(this.right)) {
s = '([' + this.right.map(ref).join(',') + '].indexOf(';
s += 'alasql.utils.getValueOf(' + leftJS() + '))<0)';
// Added patch to have a better performance for when you have a lot of entries in a NOT IN statement
if (!alasql.sets) {
alasql.sets = {};
}
const allValues = this.right.map((value) => value.value);
const allValuesStr = allValues.join(",");
if (!alasql.sets[allValuesStr]) {
// leverage JS Set, which is faster for lookups than arrays
alasql.sets[allValuesStr] = new Set(allValues);
}
s = '!alasql.sets["' + allValuesStr + '"].has(alasql.utils.getValueOf(' + leftJS() + '))';
} else {
s = '(' + rightJS() + '.indexOf(';
s += leftJS() + ')==-1)';
Expand Down Expand Up @@ -750,7 +762,7 @@

toString() {
var s;
const {op, right} = this;
const { op, right } = this;
const res = right.toString();

if (op === '~') {
Expand Down
37 changes: 20 additions & 17 deletions src/61date.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,27 +43,30 @@ stdfn.OBJECT_ID = function (objid) {
};

stdfn.DATE = function (d) {
if (/\d{8}/.test(d)) return new Date(+d.substr(0, 4), +d.substr(4, 2) - 1, +d.substr(6, 2));
if (!isNaN(d) && d.length === 8) return new Date(+d.substr(0, 4), +d.substr(4, 2) - 1, +d.substr(6, 2));
return newDate(d);
};

stdfn.NOW = function () {
var d = new Date();
var s =
d.getFullYear() +
'-' +
('0' + (d.getMonth() + 1)).substr(-2) +
'-' +
('0' + d.getDate()).substr(-2);
s +=
' ' +
('0' + d.getHours()).substr(-2) +
':' +
('0' + d.getMinutes()).substr(-2) +
':' +
('0' + d.getSeconds()).substr(-2);
s += '.' + ('00' + d.getMilliseconds()).substr(-3);
return s;
if (alasql.options.dateAsString) {
var d = new Date();
var s =
d.getFullYear() +
'-' +
('0' + (d.getMonth() + 1)).substr(-2) +
'-' +
('0' + d.getDate()).substr(-2);
s +=
' ' +
('0' + d.getHours()).substr(-2) +
':' +
('0' + d.getMinutes()).substr(-2) +
':' +
('0' + d.getSeconds()).substr(-2);
s += '.' + ('00' + d.getMilliseconds()).substr(-3);
return s;
}
return new Date();
};

stdfn.GETDATE = stdfn.NOW;
Expand Down
16 changes: 13 additions & 3 deletions test/test202.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,24 @@ if (typeof exports === 'object') {
}

describe('Test 202 GETTIME and CAST', function () {
it('1. GETDATE()', function (done) {
it('1a. GETDATE() as String', function (done) {
var res = alasql('SELECT ROW NOW(),GETDATE()');
// console.log(res);
assert(res[0].substr(0, 20) == res[1].substr(0, 20));
assert(res[0].substr(0, 20) === res[1].substr(0, 20));
done();
});

it('2. CONVERT(,,110)', function (done) {
it('1b. GETDATE() as Date', function (done) {
alasql.options.dateAsString = false;
var res = alasql('SELECT ROW NOW(),GETDATE()');
// console.log(res);
assert(res[0] instanceof Date);
assert(res[1] instanceof Date);
assert(res[1].toISOString() === res[0].toISOString());
done();
});

it('2. CONVERT(,,110) as String', function (done) {
var res = alasql('SELECT VALUE CONVERT(NVARCHAR(10),GETDATE(),110)');
// console.log(res);
assert(res.substr(-4) == new Date().getFullYear());
Expand Down
Loading

0 comments on commit 11e4fe7

Please sign in to comment.