From 4d5691eff11e81c84b5494e7e252b4aae2c869a2 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Wed, 14 Aug 2019 10:28:45 +0200 Subject: [PATCH 1/3] Drop support of key types other than string and Buffer --- README.md | 13 +++++++------ UPGRADING.md | 4 ++++ memdown.js | 7 ++++--- test.js | 29 +++-------------------------- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index b5d9d7f..75df822 100644 --- a/README.md +++ b/README.md @@ -40,31 +40,32 @@ Your data is discarded when the process ends or you release a reference to the s ## Data types -Unlike [`leveldown`], `memdown` does not stringify keys or values. This means that in addition to Buffers, you can store any JS type without the need for [`encoding-down`]. For keys for example, you could use Buffers or strings, which sort lexicographically, or numbers, even Dates, which sort naturally. The only exceptions are `null` and `undefined`. Keys and values of that type are rejected. +Keys can be strings or Buffers. Any other key type will be irreversibly stringified. Unlike [`leveldown`] though, `memdown` does not stringify values. This means that in addition to Buffers, you can store any JS value without the need for [`encoding-down`]. The only exceptions are `null` and `undefined`. Keys and values of that type are rejected. ```js const db = levelup(memdown()) -db.put(12, true, (err) => { +db.put('example', 123, (err) => { if (err) throw err db.createReadStream({ keyAsBuffer: false, valueAsBuffer: false }).on('data', (entry) => { - console.log(typeof entry.key) // 'number' - console.log(typeof entry.value) // 'boolean' + console.log(typeof entry.key) // 'string' + console.log(typeof entry.value) // 'number' }) }) ``` -If you desire normalization for keys and values (e.g. to stringify numbers), wrap `memdown` with [`encoding-down`]. Alternatively install [`level-mem`] which conveniently bundles [`levelup`], `memdown` and [`encoding-down`]. Such an approach is also recommended if you want to achieve universal (isomorphic) behavior. For example, you could have [`leveldown`] in a backend and `memdown` in the frontend. +If you desire normalization for values (e.g. to stringify numbers), wrap `memdown` with [`encoding-down`]. Alternatively install [`level-mem`] which conveniently bundles [`levelup`], `memdown` and [`encoding-down`]. Such an approach is also recommended if you want to achieve universal (isomorphic) behavior. For example, you could have [`leveldown`] in a backend and `memdown` in the frontend. ```js const encode = require('encoding-down') const db = levelup(encode(memdown())) -db.put(12, true, (err) => { +// The default value encoding is utf8, which stringifies input. +db.put('example', 123, (err) => { if (err) throw err db.createReadStream({ diff --git a/UPGRADING.md b/UPGRADING.md index f196bb6..1ce4ed6 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -2,6 +2,10 @@ This document describes breaking changes and how to upgrade. For a complete list of changes including minor and patch releases, please refer to the [`CHANGELOG`][changelog]. +## v5 (unreleased) + +Support of keys other than strings and Buffers has been dropped. Internally `memdown` now stores keys as Buffers which solves a number of compatibility issues ([#186](https://github.com/Level/memdown/issues/186)). If you pass in a key that isn't a string or Buffer, it will be irreversibly stringified. + ## v4 This is an upgrade to `abstract-leveldown@6` which solves long-standing issues around serialization and type support. diff --git a/memdown.js b/memdown.js index ce36243..10cbbf0 100644 --- a/memdown.js +++ b/memdown.js @@ -10,6 +10,7 @@ var Buffer = require('safe-buffer').Buffer var setImmediate = require('./immediate') var NONE = {} +// TODO (perf): replace ltgt.compare with a simpler, buffer-only comparator function gt (value) { return ltgt.compare(value, this._upperBound) > 0 } @@ -97,8 +98,8 @@ MemIterator.prototype._next = function (callback) { if (!this._test(key)) return setImmediate(callback) - if (this.keyAsBuffer && !Buffer.isBuffer(key)) { - key = Buffer.from(String(key)) + if (!this.keyAsBuffer) { + key = key.toString() } if (this.valueAsBuffer && !Buffer.isBuffer(value)) { @@ -167,7 +168,7 @@ MemDOWN.prototype._open = function (options, callback) { } MemDOWN.prototype._serializeKey = function (key) { - return key + return Buffer.isBuffer(key) ? key : Buffer.from(String(key)) } MemDOWN.prototype._serializeValue = function (value) { diff --git a/test.js b/test.js index dc55f82..0bac904 100644 --- a/test.js +++ b/test.js @@ -368,7 +368,8 @@ test('number keys', function (t) { t.plan(4) var db = testCommon.factory() - var numbers = [-Infinity, 0, 2, 12, +Infinity] + var numbers = [-Infinity, 0, 12, 2, +Infinity] + var strings = numbers.map(String) var buffers = numbers.map(stringBuffer) db.open(noop) @@ -379,31 +380,7 @@ test('number keys', function (t) { concat(iterator1, function (err, entries) { t.ifError(err, 'no iterator error') - t.same(entries.map(getKey), numbers, 'sorts naturally') - }) - - concat(iterator2, function (err, entries) { - t.ifError(err, 'no iterator error') - t.same(entries.map(getKey), buffers, 'buffer input is stringified') - }) -}) - -test('date keys', function (t) { - t.plan(4) - - var db = testCommon.factory() - var dates = [new Date(0), new Date(1)] - var buffers = dates.map(stringBuffer) - - db.open(noop) - db.batch(dates.map(putKey), noop) - - var iterator = db.iterator({ keyAsBuffer: false }) - var iterator2 = db.iterator({ keyAsBuffer: true }) - - concat(iterator, function (err, entries) { - t.ifError(err, 'no iterator error') - t.same(entries.map(getKey), dates, 'sorts naturally') + t.same(entries.map(getKey), strings, 'sorts lexicographically') }) concat(iterator2, function (err, entries) { From d9be16be21ff720351004fef07494c4b3ed5c4c2 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 17 Aug 2019 14:03:37 +0200 Subject: [PATCH 2/3] Test mixed use of string and Buffer key --- test.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test.js b/test.js index 0bac904..64a68ac 100644 --- a/test.js +++ b/test.js @@ -364,6 +364,34 @@ test('put multiple times', function (t) { }) }) +test('put key as string, get as buffer and vice versa', function (t) { + t.plan(7) + + var db = testCommon.factory() + + db.open(function (err) { + t.ifError(err, 'no error from open') + + db.put('a', 'a', function (err) { + t.ifError(err, 'no put error') + + db.get(Buffer.from('a'), { asBuffer: false }, function (err, value) { + t.ifError(err, 'no get error') + t.is(value, 'a', 'got value') + }) + }) + + db.put(Buffer.from('b'), 'b', function (err) { + t.ifError(err, 'no put error') + + db.get('b', { asBuffer: false }, function (err, value) { + t.ifError(err, 'no get error') + t.is(value, 'b', 'got value') + }) + }) + }) +}) + test('number keys', function (t) { t.plan(4) From fd27efbe2c9418c5f93b6a081b37f793e6c50acd Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 17 Aug 2019 14:17:18 +0200 Subject: [PATCH 3/3] Test mixed use of string and Buffer key with iterator --- test.js | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test.js b/test.js index 64a68ac..56ca491 100644 --- a/test.js +++ b/test.js @@ -392,6 +392,44 @@ test('put key as string, get as buffer and vice versa', function (t) { }) }) +test('put key as string, iterate as buffer', function (t) { + t.plan(4) + + var db = testCommon.factory() + + db.open(function (err) { + t.ifError(err, 'no error from open') + + db.put('a', 'a', function (err) { + t.ifError(err, 'no put error') + + concat(db.iterator({ keyAsBuffer: true, valueAsBuffer: false }), function (err, entries) { + t.ifError(err, 'no concat error') + t.same(entries, [{ key: Buffer.from('a'), value: 'a' }]) + }) + }) + }) +}) + +test('put key as buffer, iterate as string', function (t) { + t.plan(4) + + var db = testCommon.factory() + + db.open(function (err) { + t.ifError(err, 'no error from open') + + db.put(Buffer.from('a'), 'a', function (err) { + t.ifError(err, 'no put error') + + concat(db.iterator({ keyAsBuffer: false, valueAsBuffer: false }), function (err, entries) { + t.ifError(err, 'no concat error') + t.same(entries, [{ key: 'a', value: 'a' }]) + }) + }) + }) +}) + test('number keys', function (t) { t.plan(4)