Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Partition key from option #229

Merged
merged 1 commit into from
Nov 20, 2019
Merged

Partition key from option #229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions lib/cloudant.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,59 @@ Cloudant.prototype.ping = function(cb) {
else cb();
}
};

/**
* Apply find queries function.
* This function will perform a partitionedFind if `partitionKey`
* is provided in the options or from the query(to be supported).
*
* @param {Object} mo The selected model
* @param {Object} query The query to filter
* @param {Object[]} docs Model document/data
* @param {Object} include Include filter
* @param {Object} options Options for find method, like `partitionKey`
* @callback {Function} cb The callback function
*/
Cloudant.prototype._findRecursive = function(
mo, query, docs, include, options, cb,
) {
const self = this;
const db = this.cloudant.use(self.getDbName(self));
const partitionKey = options && options.partitionKey;
debug('Cloudant _findRecursive: partitionKey: %s', partitionKey);

const findCb = function(err, rst) {
debug('Cloudant.prototype.all (findRecursive) results: %j %j', err, rst);
if (err) return cb(err);

// only sort numeric id if the id type is of Number
const idName = self.getIdName(mo.mo.model.modelName);
if (!!idName && mo.mo.properties[idName].type.name === 'Number' &&
query.sort)
self._sortNumericId(rst.docs, query.sort);

// work around for issue
// https://github.com/strongloop/loopback-connector-Couchdb/issues/73
if (!rst.docs) {
const queryView = util.inspect(query, 4);
debug('findRecursive query: %s', queryView);
const errMsg = util.format('No documents returned for query: %s',
queryView);
return cb(new Error(g.f(errMsg)));
}
include(rst.docs, function(err) {
if (err) return cb(err);
self._extendDocs(rst, docs, query, mo, include, options, cb);
});
};

if (partitionKey) {
db.partitionedFind(partitionKey, query, findCb);
} else {
mo.db.find(query, findCb);
}
};

// mixins
require('./view')(Cloudant);
require('./geo')(Cloudant);
Expand Down
5 changes: 3 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,10 @@
"fs-extra": "^8.1.0",
"lodash": "^4.17.11",
"loopback-connector": "^4.0.0",
"loopback-connector-couchdb2": "^1.5.1",
"loopback-connector-couchdb2": "^1.5.2",
"request": "^2.81.0",
"strong-globalize": "^5.0.0"
"strong-globalize": "^5.0.0",
"uuid": "^3.3.3"
},
"devDependencies": {
"dockerode": "^2.4.3",
Expand Down
268 changes: 158 additions & 110 deletions test/partition.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
require('./init.js');
const _ = require('lodash');
const should = require('should');
const uuid = require('uuid/v4');
const DEFAULT_MODEL_VIEW = 'loopback__model__name';
let Product, db, connector;

Expand All @@ -28,141 +29,188 @@ describe('cloudant - partitioned db', () => {
db.automigrate(done);
});

it('property level - create global index by default', (done) => {
Product = db.define('Product', {
prodName: {type: String, index: true},
desc: {type: String},
});
db.autoupdate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName_index';
should.exist(indexes);
context('index tests', ()=> {
it('property level - create global index by default', (done) => {
Product = db.define('Product', {
prodName: {type: String, index: true},
desc: {type: String},
});
db.autoupdate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName_index';
should.exist(indexes);

const index = _.find(indexes, function(index) {
return index.name === indexName;
const index = _.find(indexes, function(index) {
return index.name === indexName;
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('asc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('asc');
// should be a global index
index.partitioned.should.equal(false);
done();
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('asc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('asc');
// should be a global index
index.partitioned.should.equal(false);
done();
});
});
});

it('index entry - create global index by default', (done) => {
Product = db.define('Product', {
prodName: {type: String},
desc: {type: String},
}, {
indexes: {
'prodName1_index': {
keys: {
prodName: -1,
it('index entry - create global index by default', (done) => {
Product = db.define('Product', {
prodName: {type: String},
desc: {type: String},
}, {
indexes: {
'prodName1_index': {
keys: {
prodName: -1,
},
},
},
},
});
db.autoupdate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName1_index';
should.exist(indexes);
});
db.autoupdate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName1_index';
should.exist(indexes);

const index = _.find(indexes, function(index) {
return index.name === indexName;
const index = _.find(indexes, function(index) {
return index.name === indexName;
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('desc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('desc');
// should be a global index
index.partitioned.should.equal(false);
done();
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('desc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('desc');
// should be a global index
index.partitioned.should.equal(false);
done();
});
});
});

it('index entry - ' +
'create partitioned index for when `partitioned` is configured as true',
(done) => {
Product = db.define('Product', {
prodName: {type: String},
desc: {type: String},
}, {
indexes: {
'prodName2_index': {
partitioned: true,
keys: {
prodName: 1,
it('index entry - ' +
'create partitioned index for when `partitioned` is configured as true',
(done) => {
Product = db.define('Product', {
prodName: {type: String},
desc: {type: String},
}, {
indexes: {
'prodName2_index': {
partitioned: true,
keys: {
prodName: 1,
},
},
},
},
});
db.autoupdate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName2_index';
should.exist(indexes);
});
db.autoupdate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName2_index';
should.exist(indexes);

const index = _.find(indexes, function(index) {
return index.name === indexName;
const index = _.find(indexes, function(index) {
return index.name === indexName;
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('asc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('asc');
// should be a global index
index.partitioned.should.equal(true);
done();
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('asc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('asc');
// should be a global index
index.partitioned.should.equal(true);
done();
});
});
});

it('index entry - ' +
'create global index for when `partitioned` is configured as false',
(done) => {
Product = db.define('Product', {
prodName: {type: String},
desc: {type: String},
}, {
indexes: {
'prodName3_index': {
partitioned: false,
keys: {
prodName: 1,
it('index entry - ' +
'create global index for when `partitioned` is configured as false',
(done) => {
Product = db.define('Product', {
prodName: {type: String},
desc: {type: String},
}, {
indexes: {
'prodName3_index': {
partitioned: false,
keys: {
prodName: 1,
},
},
},
},
});
db.automigrate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName3_index';
should.exist(indexes);

const index = _.find(indexes, function(index) {
return index.name === indexName;
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('asc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('asc');
// should be a global index
index.partitioned.should.equal(false);
done();
});
});
});
db.automigrate('Product', (err) => {
if (err) return done(err);
connector.getIndexes(connector.getDbName(connector), (e, results) => {
if (e) return done(e);
const indexes = results.indexes;
const indexName = 'prodName3_index';
should.exist(indexes);
});

const index = _.find(indexes, function(index) {
return index.name === indexName;
context('findAll tests', ()=> {
it('find all records by partition key from option', (done) => {
Product = db.define('Product', {
name: {type: String},
tag: {type: String},
}, {
forceId: false,
indexes: {
'product_name_index': {
partitioned: true,
keys: {
city: 1,
},
},
},
});
db.automigrate('Product', () => {
Product.create(SEED_DATA, function(err) {
if (err) return done(err);
Product.find({where: {tag: 'food'}}, {partitionKey: 'toronto'},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just out of curious, I guess the performance of partition + where(#L196) is better than doing where on find() normally?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@agnes512 Exactly, if you are interested to learn more about this feature, it's explained in https://docs.couchdb.org/en/master/partitioned-dbs/index.html.

An example: you have 200 documents in partition US and 300 documents in partition CA, when you do a global search, you find among 500 documents, but if you specify partitionKey as US and perform a partitioned search, you find among those 200 documents.

(err, results) => {
if (err) return done(err);
should.exist(results);
results.length.should.equal(2);
const resultTaggedFood = _.filter(results,
function(r) {
return r.tag === 'food';
});
resultTaggedFood.length.should.equal(2);
done();
});
});
should.exist(index);
index.name.should.equal(indexName);
index.def.fields[0]['prodName'].should.equal('asc');
index.def.fields[1][DEFAULT_MODEL_VIEW].should.equal('asc');
// should be a global index
index.partitioned.should.equal(false);
done();
});
});
});
});

const SEED_DATA = [
{id: `toronto: ${uuid()}`, name: 'beer', tag: 'drink'},
{id: `toronto: ${uuid()}`, name: 'salad', tag: 'food'},
{id: `toronto: ${uuid()}`, name: 'soup', tag: 'food'},
{id: `london: ${uuid()}`, name: 'beer', tag: 'drink'},
{id: `london: ${uuid()}`, name: 'salad', tag: 'food'},
{id: `london: ${uuid()}`, name: 'salad', tag: 'food'},
];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if I have a 7th record {id: toronto: ${uuid()}, name: 'dessert', tag: 'food'},, would it be included to the result of the above test case?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep, because its partitionKey is toronto(matches your partition key) and is tagged as food(match your query).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, the name partitionKey always makes me think of operations based on index.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep they are related :)

The benefit of partitioned databases is that secondary indices can be significantly more efficient when locating matching documents since their entries are contained within their partition. This means a given secondary index read will only scan a single partition range instead of having to read from a copy of every shard.

My understanding is index is a secondary optimization for search.