Skip to content

Commit

Permalink
datastore: the changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@stephenplusplus
stephenplusplus committed May 6, 2016
1 parent 4d6ebe3 commit c787f53
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 91 deletions.
10 changes: 3 additions & 7 deletions lib/common/stream-router.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,18 +94,14 @@ streamRouter.parseArguments_ = function(args) {
if (is.number(query.maxResults)) {
// `maxResults` is used API-wide.
maxResults = query.maxResults;
} else if (is.number(query.limitVal)) {
// `limitVal` is part of a Datastore query.
maxResults = query.limitVal;
} else if (is.number(query.pageSize)) {
// `pageSize` is Pub/Sub's `maxResults`.
maxResults = query.pageSize;
}

if (callback &&
(maxResults !== -1 || // The user specified a limit.
query.autoPaginate === false ||
query.autoPaginateVal === false)) {
query.autoPaginate === false)) {
autoPaginate = false;
}
}
Expand Down Expand Up @@ -159,8 +155,8 @@ streamRouter.router_ = function(parsedArguments, originalMethod) {
* This method simply calls the nextQuery recursively, emitting results to a
* stream. The stream ends when `nextQuery` is null.
*
* `maxResults` and `limitVal` (from Datastore) will act as a cap for how many
* results are fetched and emitted to the stream.
* `maxResults` will act as a cap for how many results are fetched and emitted
* to the stream.
*
* @param {object=|string=} parsedArguments.query - Query object. This is most
* commonly an object, but to make the API more simple, it can also be a
Expand Down
29 changes: 0 additions & 29 deletions lib/datastore/query.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,41 +67,12 @@ function Query(scope, namespace, kinds) {
this.selectVal = [];

// pagination
this.autoPaginateVal = true;
this.startVal = null;
this.endVal = null;
this.limitVal = -1;
this.offsetVal = -1;
}

/**
* @param {boolean=} autoPaginateVal - Have pagination handled automatically.
* Default: true.
* @return {module:datastore/query}
*
* @example
* //-
* // Retrieve a list of people related to "Dave", with auto-pagination
* // disabled.
* //-
* var query = datastore.createQuery('Person')
* .hasAncestor(datastore.key(['Person', 'Dave']))
* .autoPaginate(false);
*
* function callback(err, entities, nextQuery, apiResponse) {
* if (nextQuery) {
* // More results might exist, so we'll manually fetch them.
* datastore.runQuery(nextQuery, callback);
* }
* }
*
* datastore.runQuery(query, callback);
*/
Query.prototype.autoPaginate = function(autoPaginateVal) {
this.autoPaginateVal = autoPaginateVal !== false;
return this;
};

/**
* Datastore allows querying on properties. Supported comparison operators
* are `=`, `<`, `>`, `<=`, and `>=`. "Not equal" and `IN` operators are
Expand Down
53 changes: 16 additions & 37 deletions lib/datastore/request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,6 @@ var entity = require('./entity.js');
*/
var Query = require('./query.js');

/**
* @type {module:common/stream-router}
* @private
*/
var streamRouter = require('../common/stream-router.js');

/**
* @type {module:common/util}
* @private
Expand Down Expand Up @@ -489,7 +483,6 @@ DatastoreRequest.prototype.runQuery = function(query, options, callback) {
};
}

var originalLimitVal = query.limitVal;
var entities = [];

function onResponse(err, resp) {
Expand All @@ -498,43 +491,36 @@ DatastoreRequest.prototype.runQuery = function(query, options, callback) {
return;
}

var nextQuery = null;

if (resp.batch.entityResults) {
entities = entities.concat(entity.formatArray(resp.batch.entityResults));
}

var notFinished = resp.batch.moreResults === 'NOT_FINISHED';
var moreResults = resp.batch.moreResults === 'MORE_RESULTS_AFTER_LIMIT';
var info = {
moreResults: resp.batch.moreResults
};

if (notFinished || moreResults) {
var endCursor = resp.batch.endCursor;
var offset = query.offsetVal === -1 ? 0 : query.offsetVal;
var nextOffset = offset - resp.batch.skippedResults;
nextQuery = extend(true, new Query(), query);
nextQuery.start(endCursor.toString('base64')).offset(nextOffset);
if (resp.batch.endCursor) {
info.endCursor = resp.batch.endCursor;
}

if (notFinished) {
// Run the query again to make sure all of the requested entities are
// returned.
if (resp.batch.moreResults === 'NOT_FINISHED') {
var offset = query.offsetVal === -1 ? 0 : query.offsetVal;

var continuationQuery = extend(true, new Query(), query)
.start(info.endCursor)
.offset(offset - resp.batch.skippedResults);

var limit = reqOpts.query.limit && reqOpts.query.limit.value;
if (limit && limit > -1) {
// Update the limit on the nextQuery to return only the amount of
// results originally asked for.
nextQuery.limit(limit - resp.batch.entityResults.length);
continuationQuery.limit(limit - resp.batch.entityResults.length);
}
reqOpts.query = entity.queryToQueryProto(nextQuery);

reqOpts.query = entity.queryToQueryProto(continuationQuery);
self.request_(protoOpts, reqOpts, onResponse);
return;
}

if (nextQuery && originalLimitVal > -1) {
// Restore the original limit value for the query.
nextQuery.limit(originalLimitVal);
}

callback(null, entities, nextQuery, resp);
callback(null, entities, info, resp);
}

this.request_(protoOpts, reqOpts, onResponse);
Expand Down Expand Up @@ -880,11 +866,4 @@ DatastoreRequest.prototype.request_ = function(protoOpts, reqOpts, callback) {
this.request(protoOpts, reqOpts, callback);
};

/*! Developer Documentation
*
* This method can be used with either a callback or as a readable object
* stream. `streamRouter` is used to add this dual behavior.
*/
streamRouter.extend(DatastoreRequest, 'runQuery');

module.exports = DatastoreRequest;
39 changes: 21 additions & 18 deletions system-test/datastore.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ var env = require('./env.js');
var Datastore = require('../lib/datastore/index.js');
var entity = require('../lib/datastore/entity.js');

describe('Datastore', function() {
describe.only('Datastore', function() {
var testKinds = [];
var datastore = new Datastore(env);

Expand Down Expand Up @@ -431,19 +431,19 @@ describe('Datastore', function() {
});

it('should limit queries', function(done) {
var firstQ = datastore.createQuery('Character')
var q = datastore.createQuery('Character')
.hasAncestor(ancestor)
.limit(5)
.autoPaginate(false);
.limit(5);

datastore.runQuery(firstQ, function(err, firstEntities, secondQ) {
datastore.runQuery(q, function(err, firstEntities, info) {
assert.ifError(err);
assert.strictEqual(firstEntities.length, 5);

datastore.runQuery(secondQ, function(err, secondEntities, thirdQ) {
q.start(info.endCursor).limit(q.limitVal - firstEntities.length);

datastore.runQuery(q, function(err, secondEntities) {
assert.ifError(err);
assert.strictEqual(secondEntities.length, 3);
assert.strictEqual(thirdQ, null);
done();
});
});
Expand All @@ -462,7 +462,7 @@ describe('Datastore', function() {
});
});

it('should run a query as a stream', function(done) {
it.skip('should run a query as a stream', function(done) {
var q = datastore.createQuery('Character').hasAncestor(ancestor);

var resultsReturned = 0;
Expand All @@ -476,7 +476,7 @@ describe('Datastore', function() {
});
});

it('should not go over a limit with a stream', function(done) {
it.skip('should not go over a limit with a stream', function(done) {
var limit = 3;
var q = datastore.createQuery('Character')
.hasAncestor(ancestor)
Expand Down Expand Up @@ -584,18 +584,20 @@ describe('Datastore', function() {
.hasAncestor(ancestor)
.offset(2)
.limit(3)
.order('appearances')
.autoPaginate(false);
.order('appearances');

datastore.runQuery(q, function(err, entities, secondQuery) {
datastore.runQuery(q, function(err, entities, info) {
assert.ifError(err);

assert.strictEqual(entities.length, 3);
assert.strictEqual(entities[0].data.name, 'Robb');
assert.strictEqual(entities[2].data.name, 'Catelyn');

var offsetQuery = secondQuery.offset(0);
datastore.runQuery(offsetQuery, function(err, secondEntities) {
q.start(info.endCursor)
.limit(q.limitVal - entities.length)
.offset(0);

datastore.runQuery(q, function(err, secondEntities) {
assert.ifError(err);

assert.strictEqual(secondEntities.length, 3);
Expand All @@ -612,13 +614,14 @@ describe('Datastore', function() {
.hasAncestor(ancestor)
.offset(2)
.limit(2)
.order('appearances')
.autoPaginate(false);
.order('appearances');

datastore.runQuery(q, function(err, entities, nextQuery) {
datastore.runQuery(q, function(err, entities, info) {
assert.ifError(err);

datastore.runQuery(nextQuery.limit(-1), function(err, secondEntities) {
q.start(info.endCursor).limit(-1).offset(-1);

datastore.runQuery(q, function(err, secondEntities, info) {
assert.ifError(err);

assert.strictEqual(secondEntities.length, 4);
Expand Down

0 comments on commit c787f53

Please sign in to comment.