Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Smarter filter and orderBy index optimizations #499

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Smarter filter and orderBy index optimizations #499

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 65 additions & 39 deletions lib/model.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,38 +307,51 @@ Model.prototype.getTableName = function() {
Model.prototype.ensureIndex = function(name, fn, opts) {
var self = this;

if ((opts === undefined) && (util.isPlainObject(fn))) {
opts = fn;
fn = undefined;
}

return self._createIndex(name, fn, opts)
.catch(function(error) {
self._getModel()._setError(error);
throw error;
});
}

Model.prototype.ensureCompoundIndex = function(name, fields) {
var self = this;

fields.forEach(function (f) {
if (typeof f !== 'string')
throw new Error('Second argument to `ensureCompoundIndex` must be an array of strings.');
});

return self.ensureIndex(name, { compound: fields });
}

Model.prototype._createIndex = function(name, fn, opts) {
var model = this._getModel();
var tableName = this.getTableName();
var r = model._thinky.r;
var table = r.table(this.getTableName());
var fields = [ name ];

if (opts === undefined && util.isPlainObject(fn)) {
opts = fn;
fn = undefined;
}

if (opts && Array.isArray(opts.compound)) {
fields = opts.compound;
fn = opts.compound.map(function (f) { return r.row(f); });
delete opts.compound;
}

var promise = this.tableReady().then(function() {
return new Promise(function(resolve, reject) {
return r.branch(
r.table(tableName).indexList().contains(name),
r.table(tableName).indexWait(name),
table.indexList().contains(name),
table.indexWait(name),
r.branch(
r.table(tableName).info()('primary_key').eq(name),
r.table(tableName).indexWait(name),
r.table(tableName).indexCreate(name, fn, opts).do(function() {
return r.table(tableName).indexWait(name);
table.info()('primary_key').eq(name),
table.indexWait(name),
table.indexCreate(name, fn, opts).do(function() {
return table.indexWait(name);
})
)
)
Expand All @@ -356,7 +369,7 @@ Model.prototype._createIndex = function(name, fn, opts) {
});
})
.then(function() {
model._indexes[name] = true;
model._indexes[name] = fields;
});

this._waitFor(promise);
Expand Down Expand Up @@ -622,8 +635,8 @@ Model.prototype.hasAndBelongsToMany = function(joinedModel, fieldDoc, leftKey, r
var linkPromise = linkModel.ready().then(function() {
return query.run()
.then(function() {
self._getModel()._indexes[leftKey] = true;
joinedModel._getModel()._indexes[rightKey] = true;
self._getModel()._indexes[leftKey] = [leftKey];
joinedModel._getModel()._indexes[rightKey] = [rightKey];
})
.error(function(error) {
if (error.message.match(/^Index `/)) {
Expand Down Expand Up @@ -670,43 +683,56 @@ Model.prototype.hasAndBelongsToMany = function(joinedModel, fieldDoc, leftKey, r
case 'orderBy':
Model.prototype[key] = function() {
var query = new Query(this);
if ((arguments.length === 1)
&& (typeof arguments[0] === 'string')
&& (this._getModel()._indexes[arguments[0]] === true)) {
var args = Array.prototype.slice.call(arguments);

query = query[key]({index: arguments[0]});
return query;
}
else {
query = query[key].apply(query, arguments);
return query;
}
// We find the first index matching all fields
util.loopKeys(this._getModel()._indexes, function(indexes, key) {
if (util.deepEqual(indexes[key], args)) {
args = {index: key};
return false;
}
});

return query[key].apply(query, [].concat(args));
}
break;
case 'filter':
Model.prototype[key] = function() {
var query = new Query(this);

// Optimize a filter with an object
if ((arguments.length === 1)
&& (util.isPlainObject(arguments[0]))) {

// Optimize a filter with an object
// We replace the first key that match an index name
var filter = arguments[0];

var keys = Object.keys(filter).sort(); // Lexicographical order
for(var i=0 ; i<keys.length; i++) {
var index = keys[i];
var indexes = this._getModel()._indexes;

if (this._getModel()._indexes[index] === true) { // Index found
query = query.getAll(filter[index], {index: index});
delete filter[index];
break;
}
var filter = arguments[0];
var filter_keys = Object.keys(filter).sort(); // Lexicographical order

// We find the index with the most overlapping fields
var optimal = Object.keys(indexes).reduce(function (optimal, name) {
var overlap = util.arrayOverlap(indexes[name], filter_keys);
// Ensure enough filter values were provided to fill the index
var filled = overlap === indexes[name].length;
return filled && (!optimal || (overlap > indexes[optimal].length)) ? name : optimal;
}, null);

if (optimal !== null) {
var args = indexes[optimal].map(function (f) {
var val = filter[f];
delete filter[f];
return val;
});

if (args.length === 1) args = args[0];
query = query.getAll(args, {index: optimal});

// Prevent appending a no-op empty filter
if (Object.keys(filter).length === 0) return query;
}
}

query = query[key].apply(query, arguments);
return query;
return query[key].apply(query, arguments);
}
break;
case 'get':
Expand Down
38 changes: 38 additions & 0 deletions lib/util.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,44 @@ function deepCopy(value) {
util.deepCopy = deepCopy;


/**
* Compare to objects for strict, deep equality.
*/
function deepEqual(a, b) {
var aArray = Array.isArray(a);
var bArray = Array.isArray(b);

if (aArray !== bArray) return false;
if (aArray) {
return a.reduce(function (acc, val, i) {
return acc && deepEqual(val, b[i]);
}, a.length === b.length);
}

if (isPlainObject(a)) {
var equal = isPlainObject(b);
loopKeys(a, function(_, key) {
return equal = equal && deepEqual(a[key], b[key]);
});
return equal;
}

return a === b;
}
util.deepEqual = deepEqual;


/**
* Compute the number of overlapping values between two arrays.
*/
function arrayOverlap(a, b) {
return a.reduce(function (sum, val) {
return ~b.indexOf(val) ? sum + 1 : sum;
}, 0)
}
util.arrayOverlap = arrayOverlap;


/**
* Wrap try/catch for v8
*/
Expand Down
9 changes: 9 additions & 0 deletions test/query.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1704,6 +1704,7 @@ describe('optimizer', function() {
Model.hasAndBelongsToMany(Model, 'manyToMany', 'hasAndBelongsToMany1', 'hasAndBelongsToMany2')

Model.ensureIndex('name1');
Model.ensureCompoundIndex('fullName', [ 'name1', 'name2' ]);
Model.once('ready', function() {
done();
})
Expand Down Expand Up @@ -1738,6 +1739,10 @@ describe('optimizer', function() {
var query = Model.orderBy('name1').toString();
assert(query.match(/index: "name1"/));
})
it('orderBy should be able to use an index - multiple fields - compound index', function() {
var query = Model.orderBy('name1', 'name2').toString();
assert(query.match(/index: "fullName"/));
})
it('filter should be able to use an index - single field', function() {
var query = Model.filter({name1: "Michel"}).toString();
assert(query.match(/index: "name1"/));
Expand All @@ -1746,6 +1751,10 @@ describe('optimizer', function() {
var query = Model.filter({name1: "Michel", foo: "bar"}).toString();
assert.equal(query.replace(/\s/g, ''), 'r.table("'+Model.getTableName()+'").getAll("Michel",{index:"name1"}).filter({foo:"bar"})')
})
it('filter should be able to use an index - multiple fields - compound index', function() {
var query = Model.filter({name1: "Michel", name2: "Bar"}).toString();
assert.equal(query.replace(/\s/g, ''), 'r.table("'+Model.getTableName()+'").getAll(["Michel","Bar"],{index:"fullName"})')
})
it('filter should not optimize a field without index', function() {
var query = Model.filter({name2: "Michel"}).toString();
assert.equal(query.match(/index: "name2"/), null);
Expand Down