diff --git a/lib/algos/list/basic.js b/lib/algos/list/basic.js index cd57457a5..9543af015 100644 --- a/lib/algos/list/basic.js +++ b/lib/algos/list/basic.js @@ -28,14 +28,14 @@ class List extends Extension { } genMDParams() { - const params = { + const params = this.parameters ? { gt: this.parameters.gt, gte: this.parameters.gte || this.parameters.start, lt: this.parameters.lt, lte: this.parameters.lte || this.parameters.end, keys: this.parameters.keys, values: this.parameters.values, - }; + } : {}; Object.keys(params).forEach(key => { if (params[key] === null || params[key] === undefined) { delete params[key]; diff --git a/lib/algos/list/delimiterVersions.js b/lib/algos/list/delimiterVersions.js index 236e470db..470428a75 100644 --- a/lib/algos/list/delimiterVersions.js +++ b/lib/algos/list/delimiterVersions.js @@ -1,6 +1,5 @@ 'use strict'; // eslint-disable-line strict -const errors = require('../../errors'); const Delimiter = require('./delimiter').Delimiter; const Version = require('../../versioning/Version').Version; const VSConst = require('../../versioning/constants').VersioningConstants; @@ -8,7 +7,7 @@ const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools'); const VID_SEP = VSConst.VersionId.Separator; -const { BucketVersioningKeyFormat } = VSConst; +const { DbPrefixes, BucketVersioningKeyFormat } = VSConst; /** * Handle object listing with parameters @@ -34,13 +33,19 @@ class DelimiterVersions extends Delimiter { // listing results this.NextMarker = parameters.keyMarker; this.NextVersionIdMarker = undefined; - } - genMDParams() { - if (this.vFormat === BucketVersioningKeyFormat.v0) { - return this.genMDParamsV0(); - } - throw errors.NotImplemented; + Object.assign(this, { + [BucketVersioningKeyFormat.v0]: { + genMDParams: this.genMDParamsV0, + filter: this.filterV0, + skipping: this.skippingV0, + }, + [BucketVersioningKeyFormat.v1]: { + genMDParams: this.genMDParamsV1, + filter: this.filterV1, + skipping: this.skippingV1, + }, + }[this.vFormat]); } genMDParamsV0() { @@ -57,8 +62,9 @@ class DelimiterVersions extends Delimiter { if (this.parameters.versionIdMarker) { // versionIdMarker should always come with keyMarker // but may not be the other way around - params.gt = `${this.parameters.keyMarker}` + - `${VID_SEP}${this.parameters.versionIdMarker}`; + params.gt = this.parameters.keyMarker + + VID_SEP + + this.parameters.versionIdMarker; } else { params.gt = inc(this.parameters.keyMarker + VID_SEP); } @@ -66,6 +72,59 @@ class DelimiterVersions extends Delimiter { return params; } + genMDParamsV1() { + // return an array of two listing params sets to ask for + // synchronized listing of M and V ranges + const params = [{}, {}]; + if (this.parameters.prefix) { + params[0].gte = DbPrefixes.Master + this.parameters.prefix; + params[0].lt = DbPrefixes.Master + inc(this.parameters.prefix); + params[1].gte = DbPrefixes.Version + this.parameters.prefix; + params[1].lt = DbPrefixes.Version + inc(this.parameters.prefix); + } else { + params[0].gte = DbPrefixes.Master; + params[0].lt = inc(DbPrefixes.Master); // stop after the last master key + params[1].gte = DbPrefixes.Version; + params[1].lt = inc(DbPrefixes.Version); // stop after the last version key + } + if (this.parameters.keyMarker) { + if (params[1].gte <= DbPrefixes.Version + this.parameters.keyMarker) { + delete params[0].gte; + delete params[1].gte; + params[0].gt = DbPrefixes.Master + inc(this.parameters.keyMarker + VID_SEP); + if (this.parameters.versionIdMarker) { + // versionIdMarker should always come with keyMarker + // but may not be the other way around + params[1].gt = DbPrefixes.Version + + this.parameters.keyMarker + + VID_SEP + + this.parameters.versionIdMarker; + } else { + params[1].gt = DbPrefixes.Version + + inc(this.parameters.keyMarker + VID_SEP); + } + } + } + return params; + } + + /** + * Used to synchronize listing of M and V prefixes by object key + * + * @param {object} masterObj object listed from first range + * returned by genMDParamsV1() (the master keys range) + * @param {object} versionObj object listed from second range + * returned by genMDParamsV1() (the version keys range) + * @return {number} comparison result: + * * -1 if master key < version key + * * 1 if master key > version key + */ + compareObjects(masterObj, versionObj) { + const masterKey = masterObj.key.slice(DbPrefixes.Master.length); + const versionKey = versionObj.key.slice(DbPrefixes.Version.length); + return masterKey < versionKey ? -1 : 1; + } + /** * Add a (key, versionId, value) tuple to the listing. * Set the NextMarker to the current key @@ -92,7 +151,8 @@ class DelimiterVersions extends Delimiter { } /** - * Filter to apply on each iteration, based on: + * Filter to apply on each iteration if bucket is in v0 + * versioning key format, based on: * - prefix * - delimiter * - maxKeys @@ -102,14 +162,31 @@ class DelimiterVersions extends Delimiter { * @param {String} obj.value - The value of the element * @return {number} - indicates if iteration should continue */ - filter(obj) { - if (this.vFormat === BucketVersioningKeyFormat.v0) { - if (Version.isPHD(obj.value)) { - return FILTER_ACCEPT; // trick repd to not increase its streak - } - return this.filterCommon(obj.key, obj.value); + filterV0(obj) { + if (Version.isPHD(obj.value)) { + return FILTER_ACCEPT; // trick repd to not increase its streak } - throw errors.NotImplemented; + return this.filterCommon(obj.key, obj.value); + } + + /** + * Filter to apply on each iteration if bucket is in v1 + * versioning key format, based on: + * - prefix + * - delimiter + * - maxKeys + * The marker is being handled directly by levelDB + * @param {Object} obj - The key and value of the element + * @param {String} obj.key - The key of the element + * @param {String} obj.value - The value of the element + * @return {number} - indicates if iteration should continue + */ + filterV1(obj) { + // this function receives both M and V keys, but their prefix + // length is the same so we can remove their prefix without + // looking at the type of key + return this.filterCommon(obj.key.slice(DbPrefixes.Master.length), + obj.value); } filterCommon(key, value) { @@ -144,13 +221,6 @@ class DelimiterVersions extends Delimiter { return this.addContents({ key: nonversionedKey, value, versionId }); } - skipping() { - if (this.vFormat === BucketVersioningKeyFormat.v0) { - return this.skippingV0(); - } - throw errors.NotImplemented; - } - skippingV0() { if (this.NextMarker) { const index = this.NextMarker.lastIndexOf(this.delimiter); @@ -161,6 +231,16 @@ class DelimiterVersions extends Delimiter { return SKIP_NONE; } + skippingV1() { + const skipV0 = this.skippingV0(); + if (skipV0 === SKIP_NONE) { + return SKIP_NONE; + } + // skip to the same object key in both M and V range listings + return [DbPrefixes.Master + skipV0, + DbPrefixes.Version + skipV0]; + } + /** * Return an object containing all mandatory fields to use once the * iteration is done, doesn't show a NextMarker field if the output diff --git a/lib/algos/list/tools.js b/lib/algos/list/tools.js index 9239a6d55..349b7a545 100644 --- a/lib/algos/list/tools.js +++ b/lib/algos/list/tools.js @@ -1,3 +1,5 @@ +const { DbPrefixes } = require('../../versioning/constants').VersioningConstants; + // constants for extensions const SKIP_NONE = undefined; // to be inline with the values of NextMarker const FILTER_ACCEPT = 1; @@ -31,9 +33,36 @@ function inc(str) { String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str; } +/** + * Transform listing parameters for v0 versioning key format to make + * it compatible with v1 format + * + * @param {object} v0params - listing parameters for v0 format + * @return {object} - listing parameters for v1 format + */ +function listingParamsMasterKeysV0ToV1(v0params) { + const v1params = Object.assign({}, v0params); + if (v0params.gt !== undefined) { + v1params.gt = `${DbPrefixes.Master}${v0params.gt}`; + } else if (v0params.gte !== undefined) { + v1params.gte = `${DbPrefixes.Master}${v0params.gte}`; + } else { + v1params.gte = DbPrefixes.Master; + } + if (v0params.lt !== undefined) { + v1params.lt = `${DbPrefixes.Master}${v0params.lt}`; + } else if (v0params.lte !== undefined) { + v1params.lte = `${DbPrefixes.Master}${v0params.lte}`; + } else { + v1params.lt = inc(DbPrefixes.Master); // stop after the last master key + } + return v1params; +} + module.exports = { checkLimit, inc, + listingParamsMasterKeysV0ToV1, SKIP_NONE, FILTER_END, FILTER_SKIP, diff --git a/tests/unit/algos/list/delimiterVersions.js b/tests/unit/algos/list/delimiterVersions.js index c3928fc35..bdfa90417 100644 --- a/tests/unit/algos/list/delimiterVersions.js +++ b/tests/unit/algos/list/delimiterVersions.js @@ -7,11 +7,16 @@ const Werelogs = require('werelogs').Logger; const logger = new Werelogs('listTest'); const performListing = require('../../../utils/performListing'); const zpad = require('../../helpers').zpad; +const { inc } = require('../../../../lib/algos/list/tools'); +const VSConst = require('../../../../lib/versioning/constants').VersioningConstants; +const { DbPrefixes } = VSConst; +const VID_SEP = VSConst.VersionId.Separator; class Test { - constructor(name, input, output, filter) { + constructor(name, input, genMDParams, output, filter) { this.name = name; this.input = input; + this.genMDParams = genMDParams; this.output = output; this.filter = filter || this._defaultFilter; } @@ -27,40 +32,75 @@ const bar = '{"versionId":"bar"}'; const qux = '{"versionId":"qux"}'; const valuePHD = '{"isPHD":"true","versionId":"1234567890abcdefg"}'; const valueDeleteMarker = '{"hello":"world","isDeleteMarker":"true"}'; -const dataVersioned = [ - { key: 'Pâtisserie=中文-español-English', value: bar }, - { key: 'Pâtisserie=中文-español-English\0bar', value: bar }, - { key: 'Pâtisserie=中文-español-English\0foo', value: foo }, - { key: 'notes/spring/1.txt', value: bar }, - { key: 'notes/spring/1.txt\0bar', value: bar }, - { key: 'notes/spring/1.txt\0foo', value: foo }, - { key: 'notes/spring/1.txt\0qux', value: qux }, - { key: 'notes/spring/2.txt', value: valuePHD }, - { key: 'notes/spring/2.txt\0bar', value: valueDeleteMarker }, - { key: 'notes/spring/2.txt\0foo', value: foo }, - { key: 'notes/spring/march/1.txt', - value: '{"versionId":"null","isNull":true}' }, - { key: 'notes/spring/march/1.txt\0bar', value: bar }, - { key: 'notes/spring/march/1.txt\0foo', value: foo }, - { key: 'notes/summer/1.txt', value: bar }, - { key: 'notes/summer/1.txt\0bar', value: bar }, - { key: 'notes/summer/1.txt\0foo', value: foo }, - { key: 'notes/summer/2.txt', value: bar }, - { key: 'notes/summer/2.txt\0bar', value: bar }, - { key: 'notes/summer/4.txt', value: valuePHD }, - { key: 'notes/summer/4.txt\0bar', value: valueDeleteMarker }, - { key: 'notes/summer/4.txt\0foo', value: valueDeleteMarker }, - { key: 'notes/summer/4.txt\0qux', value: valueDeleteMarker }, - { key: 'notes/summer/44.txt', value: valuePHD }, - { key: 'notes/summer/444.txt', value: valueDeleteMarker }, - { key: 'notes/summer/4444.txt', value: valuePHD }, - { key: 'notes/summer/44444.txt', value: valueDeleteMarker }, - { key: 'notes/summer/444444.txt', value: valuePHD }, - { key: 'notes/summer/august/1.txt', value }, - { key: 'notes/year.txt', value }, - { key: 'notes/yore.rs', value }, - { key: 'notes/zaphod/Beeblebrox.txt', value }, -]; +const dataVersioned = { + v0: [ + { key: 'Pâtisserie=中文-español-English', value: bar }, + { key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar }, + { key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo }, + { key: 'notes/spring/1.txt', value: bar }, + { key: `notes/spring/1.txt${VID_SEP}bar`, value: bar }, + { key: `notes/spring/1.txt${VID_SEP}foo`, value: foo }, + { key: `notes/spring/1.txt${VID_SEP}qux`, value: qux }, + { key: 'notes/spring/2.txt', value: valuePHD }, + { key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker }, + { key: `notes/spring/2.txt${VID_SEP}foo`, value: foo }, + { key: 'notes/spring/march/1.txt', + value: '{"versionId":"null","isNull":true}' }, + { key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar }, + { key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo }, + { key: 'notes/summer/1.txt', value: bar }, + { key: `notes/summer/1.txt${VID_SEP}bar`, value: bar }, + { key: `notes/summer/1.txt${VID_SEP}foo`, value: foo }, + { key: 'notes/summer/2.txt', value: bar }, + { key: `notes/summer/2.txt${VID_SEP}bar`, value: bar }, + { key: 'notes/summer/4.txt', value: valuePHD }, + { key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker }, + { key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker }, + { key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker }, + { key: 'notes/summer/44.txt', value: valuePHD }, + { key: 'notes/summer/444.txt', value: valueDeleteMarker }, + { key: 'notes/summer/4444.txt', value: valuePHD }, + { key: 'notes/summer/44444.txt', value: valueDeleteMarker }, + { key: 'notes/summer/444444.txt', value: valuePHD }, + { key: 'notes/summer/august/1.txt', value }, + { key: 'notes/year.txt', value }, + { key: 'notes/yore.rs', value }, + { key: 'notes/zaphod/Beeblebrox.txt', value }, + ], + v1: [ // we add M and V prefixes in getTestListing() due to the + // test cases needing the original key to filter + { key: 'Pâtisserie=中文-español-English', value: bar }, + { key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar }, + { key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo }, + { key: 'notes/spring/1.txt', value: bar }, + { key: `notes/spring/1.txt${VID_SEP}bar`, value: bar }, + { key: `notes/spring/1.txt${VID_SEP}foo`, value: foo }, + { key: `notes/spring/1.txt${VID_SEP}qux`, value: qux }, + { key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker }, + { key: `notes/spring/2.txt${VID_SEP}foo`, value: foo }, + { key: 'notes/spring/march/1.txt', + value: '{"versionId":"null","isNull":true}' }, + { key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar }, + { key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo }, + { key: 'notes/summer/1.txt', value: bar }, + { key: `notes/summer/1.txt${VID_SEP}bar`, value: bar }, + { key: `notes/summer/1.txt${VID_SEP}foo`, value: foo }, + { key: 'notes/summer/2.txt', value: bar }, + { key: `notes/summer/2.txt${VID_SEP}bar`, value: bar }, + { key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker }, + { key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker }, + { key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker }, + // Compared to v0, the two following keys are version keys + // that we give a version ID, because delete markers do not + // have a master key in v1. + { key: `notes/summer/444.txt${VID_SEP}null`, value: valueDeleteMarker }, + { key: `notes/summer/44444.txt${VID_SEP}null`, value: valueDeleteMarker }, + { key: 'notes/summer/august/1.txt', value }, + { key: 'notes/year.txt', value }, + { key: 'notes/yore.rs', value }, + { key: 'notes/zaphod/Beeblebrox.txt', value }, + ], +}; const receivedData = [ { key: 'Pâtisserie=中文-español-English', value: bar, versionId: 'bar' }, { key: 'Pâtisserie=中文-español-English', value: foo, versionId: 'foo' }, @@ -90,6 +130,10 @@ const receivedData = [ ]; const tests = [ new Test('all versions', {}, { + v0: {}, + v1: [{ gte: DbPrefixes.Master, lt: inc(DbPrefixes.Master) }, + { gte: DbPrefixes.Version, lt: inc(DbPrefixes.Version) }], + }, { Versions: receivedData, CommonPrefixes: [], Delimiter: undefined, @@ -99,6 +143,17 @@ const tests = [ }), new Test('with valid key marker', { keyMarker: receivedData[3].key, + }, { + v0: { + gt: `${receivedData[3].key}\u0001`, + }, + v1: [{ + gt: `${DbPrefixes.Master}${receivedData[3].key}${inc(VID_SEP)}`, + lt: inc(DbPrefixes.Master), + }, { + gt: `${DbPrefixes.Version}${receivedData[3].key}${inc(VID_SEP)}`, + lt: inc(DbPrefixes.Version), + }], }, { Versions: receivedData.slice(5), CommonPrefixes: [], @@ -110,6 +165,17 @@ const tests = [ new Test('with bad key marker', { keyMarker: 'zzzz', delimiter: '/', + }, { + v0: { + gt: `zzzz${inc(VID_SEP)}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}zzzz${inc(VID_SEP)}`, + lt: inc(DbPrefixes.Master), + }, { + gt: `${DbPrefixes.Version}zzzz${inc(VID_SEP)}`, + lt: inc(DbPrefixes.Version), + }], }, { Versions: [], CommonPrefixes: [], @@ -120,6 +186,15 @@ const tests = [ }, (e, input) => e.key > input.keyMarker), new Test('with maxKeys', { maxKeys: 3, + }, { + v0: {}, + v1: [{ + gte: DbPrefixes.Master, + lt: inc(DbPrefixes.Master), + }, { + gte: DbPrefixes.Version, + lt: inc(DbPrefixes.Version), + }], }, { Versions: receivedData.slice(0, 3), CommonPrefixes: [], @@ -130,6 +205,15 @@ const tests = [ }), new Test('with big maxKeys', { maxKeys: 15000, + }, { + v0: {}, + v1: [{ + gte: DbPrefixes.Master, + lt: inc(DbPrefixes.Master), + }, { + gte: DbPrefixes.Version, + lt: inc(DbPrefixes.Version), + }], }, { Versions: receivedData, CommonPrefixes: [], @@ -140,6 +224,15 @@ const tests = [ }), new Test('with delimiter', { delimiter: '/', + }, { + v0: {}, + v1: [{ + gte: DbPrefixes.Master, + lt: inc(DbPrefixes.Master), + }, { + gte: DbPrefixes.Version, + lt: inc(DbPrefixes.Version), + }], }, { Versions: [ receivedData[0], @@ -153,6 +246,15 @@ const tests = [ }), new Test('with long delimiter', { delimiter: 'notes/summer', + }, { + v0: {}, + v1: [{ + gte: DbPrefixes.Master, + lt: inc(DbPrefixes.Master), + }, { + gte: DbPrefixes.Version, + lt: inc(DbPrefixes.Version), + }], }, { Versions: receivedData.filter(entry => entry.key.indexOf('notes/summer') < 0), @@ -166,6 +268,18 @@ const tests = [ delimiter: '/', prefix: 'notes/summer/', keyMarker: 'notes/summer0', + }, { + v0: { + gt: `notes/summer0${inc(VID_SEP)}`, + lt: `notes/summer${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/summer0${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes/summer${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/summer0${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes/summer${inc('/')}`, + }], }, { Versions: [], CommonPrefixes: [], @@ -177,6 +291,18 @@ const tests = [ new Test('delimiter and prefix (related to #147)', { delimiter: '/', prefix: 'notes/', + }, { + v0: { + gte: 'notes/', + lt: `notes${inc('/')}`, + }, + v1: [{ + gte: `${DbPrefixes.Master}notes/`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gte: `${DbPrefixes.Version}notes/`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [ receivedData[19], @@ -196,6 +322,18 @@ const tests = [ delimiter: '/', prefix: 'notes/', keyMarker: 'notes/year.txt', + }, { + v0: { + gt: `notes/year.txt${inc(VID_SEP)}`, + lt: `notes${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/year.txt${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [ receivedData[20], @@ -213,6 +351,18 @@ const tests = [ prefix: 'notes/', keyMarker: 'notes/', maxKeys: 1, + }, { + v0: { + gt: `notes/${inc(VID_SEP)}`, + lt: `notes${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [], CommonPrefixes: ['notes/spring/'], @@ -227,6 +377,18 @@ const tests = [ prefix: 'notes/', // prefix keyMarker: 'notes/spring/', maxKeys: 1, + }, { + v0: { + gt: `notes/spring/${inc(VID_SEP)}`, + lt: `notes${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/spring/${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/spring/${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [], CommonPrefixes: ['notes/summer/'], @@ -241,6 +403,18 @@ const tests = [ prefix: 'notes/', // prefix keyMarker: 'notes/summer/', maxKeys: 1, + }, { + v0: { + gt: `notes/summer/${inc(VID_SEP)}`, + lt: `notes${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/summer/${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/summer/${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [ receivedData[19], @@ -257,6 +431,18 @@ const tests = [ prefix: 'notes/', // prefix keyMarker: 'notes/year.txt', maxKeys: 1, + }, { + v0: { + gt: `notes/year.txt${inc(VID_SEP)}`, + lt: `notes${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/year.txt${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [ receivedData[20], @@ -273,6 +459,18 @@ const tests = [ prefix: 'notes/', keyMarker: 'notes/yore.rs', maxKeys: 1, + }, { + v0: { + gt: `notes/yore.rs${inc(VID_SEP)}`, + lt: `notes${inc('/')}`, + }, + v1: [{ + gt: `${DbPrefixes.Master}notes/yore.rs${inc(VID_SEP)}`, + lt: `${DbPrefixes.Master}notes${inc('/')}`, + }, { + gt: `${DbPrefixes.Version}notes/yore.rs${inc(VID_SEP)}`, + lt: `${DbPrefixes.Version}notes${inc('/')}`, + }], }, { Versions: [], CommonPrefixes: ['notes/zaphod/'], @@ -283,24 +481,51 @@ const tests = [ }, (e, input) => e.key > input.keyMarker), ]; -describe('Delimiter All Versions listing algorithm', () => { - it('Should return good skipping value for DelimiterVersions', done => { - const delimiter = new DelimiterVersions({ delimiter: '/' }); - for (let i = 0; i < 100; i++) { - delimiter.filter({ key: `foo/${zpad(i)}`, value: '{}' }); - } - assert.strictEqual(delimiter.skipping(), 'foo/'); - done(); - }); +function getTestListing(test, data, vFormat) { + return data + .filter(e => test.filter(e, test.input)) + .map(e => { + if (vFormat === 'v0') { + return e; + } + if (vFormat === 'v1') { + const keyPrefix = e.key.includes(VID_SEP) ? + DbPrefixes.Version : DbPrefixes.Master; + return { + key: `${keyPrefix}${e.key}`, + value: e.value, + }; + } + return assert.fail(`bad format ${vFormat}`); + }); +} + +['v0', 'v1'].forEach(vFormat => { + describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => { + it('Should return good skipping value for DelimiterVersions', () => { + const delimiter = new DelimiterVersions({ delimiter: '/' }); + for (let i = 0; i < 100; i++) { + delimiter.filter({ + key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`, + value: '{}', + }); + } + assert.strictEqual(delimiter.skipping(), + `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`); + }); - tests.forEach(test => { - it(`Should list ${test.name}`, done => { - // Simulate skip scan done by LevelDB - const d = dataVersioned.filter(e => test.filter(e, test.input)); - const res = - performListing(d, DelimiterVersions, test.input, logger); - assert.deepStrictEqual(res, test.output); - done(); + tests.forEach(test => { + it(`Should return metadata listing params to list ${test.name}`, () => { + const listing = new DelimiterVersions(test.input, logger, vFormat); + const params = listing.genMDParams(); + assert.deepStrictEqual(params, test.genMDParams[vFormat]); + }); + it(`Should list ${test.name}`, () => { + // Simulate skip scan done by LevelDB + const d = getTestListing(test, dataVersioned[vFormat], vFormat); + const res = performListing(d, DelimiterVersions, test.input, logger, vFormat); + assert.deepStrictEqual(res, test.output); + }); }); }); }); diff --git a/tests/unit/algos/list/tools.js b/tests/unit/algos/list/tools.js index c7f1d47b5..9f61d20f0 100644 --- a/tests/unit/algos/list/tools.js +++ b/tests/unit/algos/list/tools.js @@ -2,7 +2,10 @@ const assert = require('assert'); -const checkLimit = require('../../../../lib/algos/list/tools').checkLimit; +const { checkLimit, inc, listingParamsMasterKeysV0ToV1 } = + require('../../../../lib/algos/list/tools'); +const VSConst = require('../../../../lib/versioning/constants').VersioningConstants; +const { DbPrefixes } = VSConst; describe('checkLimit function', () => { const tests = [ @@ -23,3 +26,79 @@ describe('checkLimit function', () => { }); }); }); + +describe('listingParamsMasterKeysV0ToV1', () => { + const testCases = [ + { + v0params: {}, + v1params: { + gte: DbPrefixes.Master, + lt: inc(DbPrefixes.Master), + }, + }, + { + v0params: { + gt: 'foo/bar', + }, + v1params: { + gt: `${DbPrefixes.Master}foo/bar`, + lt: inc(DbPrefixes.Master), + }, + }, + { + v0params: { + gte: 'foo/bar', + }, + v1params: { + gte: `${DbPrefixes.Master}foo/bar`, + lt: inc(DbPrefixes.Master), + }, + }, + { + v0params: { + lt: 'foo/bar', + }, + v1params: { + gte: DbPrefixes.Master, + lt: `${DbPrefixes.Master}foo/bar`, + }, + }, + { + v0params: { + lte: 'foo/bar', + }, + v1params: { + gte: DbPrefixes.Master, + lte: `${DbPrefixes.Master}foo/bar`, + }, + }, + { + v0params: { + gt: 'baz/qux', + lt: 'foo/bar', + }, + v1params: { + gt: `${DbPrefixes.Master}baz/qux`, + lt: `${DbPrefixes.Master}foo/bar`, + }, + }, + { + v0params: { + gte: 'baz/qux', + lte: 'foo/bar', + limit: 5, + }, + v1params: { + gte: `${DbPrefixes.Master}baz/qux`, + lte: `${DbPrefixes.Master}foo/bar`, + limit: 5, + }, + }, + ]; + testCases.forEach(testCase => { + it(`${JSON.stringify(testCase.v0params)} => ${JSON.stringify(testCase.v1params)}`, () => { + const converted = listingParamsMasterKeysV0ToV1(testCase.v0params); + assert.deepStrictEqual(converted, testCase.v1params); + }); + }); +}); diff --git a/tests/utils/performListing.js b/tests/utils/performListing.js index c8b629a2a..e47feec23 100644 --- a/tests/utils/performListing.js +++ b/tests/utils/performListing.js @@ -1,5 +1,9 @@ -module.exports = function performListing(data, Extension, params, logger) { - const listing = new Extension(params, logger); +const assert = require('assert'); + +module.exports = function performListing(data, Extension, params, logger, vFormat) { + const listing = new Extension(params, logger, vFormat); + const mdParams = listing.genMDParams(); + assert.strictEqual(typeof mdParams, 'object'); data.every(e => listing.filter(e) >= 0); return listing.result(); };