From 2679991a8c63b3f7a23c5a2147b89e56303e55c9 Mon Sep 17 00:00:00 2001 From: "Peter A. Bigot" Date: Sun, 29 Nov 2015 08:43:47 -0600 Subject: [PATCH] Layout: add 64-bit integral encodings In many cases 64-bit integral encodings hold values that are less than 2^53 and so can be exactly represented as a JavaScript number. Simplify use of these fields by having a Number representation for them, with a slightly different naming convention to highlight that they are not exactly conversions. Closes #13 --- README.md | 23 ++++++ lib/Layout.js | 182 +++++++++++++++++++++++++++++++++++++++++++++ test/LayoutTest.js | 134 +++++++++++++++++++++++++++++++++ test/examples.js | 11 +++ test/n64.c | 71 ++++++++++++++++++ 5 files changed, 421 insertions(+) create mode 100644 test/n64.c diff --git a/README.md b/README.md index ccc3164..3fd9067 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ Layout support is provided for these types of data: * Signed and unsigned integral values from 1 to 6 bytes in length, in little-endian or big-endian format; +* Signed and unsigned 64-bit integral values decoded as integral + Numbers; * Float and double values (also little-endian or big-endian); * Sequences of instances of an arbitrary layout; * Structures with named fields containing arbitrary layouts; @@ -164,6 +166,27 @@ The buffer-layout way: See [BitStructure](http://pabigot.github.io/buffer-layout/module-Layout-BitStructure.html). +### 64-bit values as Numbers + +The C definition: + + uint64_t v = 0x0102030405060708ULL; + +The buffer-layout way: + + var ds = lo.nu64be(), + b = Buffer('0102030405060708', 'hex'), + v = 72623859790382856, + nv = v - 6; + assert.equal(v, nv); + assert.equal(ds.decode(b), nv); + +Note that because the exact value is not less than 2^53 it cannot be +represented as a JavaScript Number, and is instead approximated by a +nearby representable integer that is equivalent within Numbers. + +See [NearUInt64](http://pabigot.github.io/buffer-layout/module-Layout-NearUInt64BE.html). + ### A NUL-terminated C string The C definition: diff --git a/lib/Layout.js b/lib/Layout.js index 6634db5..0096e6c 100644 --- a/lib/Layout.js +++ b/lib/Layout.js @@ -51,6 +51,12 @@ * module:Layout.s24be|24-bit}, {@link module:Layout.s32be|32-bit}, * {@link module:Layout.s40be|40-bit}, and {@link * module:Layout.s48be|48-bit} representation ranges; + * * 64-bit integral values that decode to an exact (if magnitude is + * less than 2^53) or nearby integral Number in {@link + * module:Layout.nu64|unsigned little-endian}, {@link + * module:Layout.nu64be|unsigned big-endian}, {@link + * module:Layout.ns64|signed little-endian}, and {@link + * module:Layout.ns64be|unsigned big-endian} encodings; * * 32-bit floating point values with {@link * module:Layout.f32|little-endian} and {@link * module:Layout.f32be|big-endian} representations; @@ -92,6 +98,10 @@ * @local UIntBE * @local Int * @local IntBE + * @local NearUInt64 + * @local NearUInt64BE + * @local NearInt64 + * @local NearInt64BE * @local Float * @local FloatBE * @local Double @@ -531,6 +541,162 @@ IntBE.prototype.encode = function (src, b, offset) { b.writeIntBE(src, offset, this.span); }; +var V2E32 = Math.pow(2, 32); +/* True modulus high and low 32-bit words, where low word is always + * non-negative. */ +function divmodInt64 (src) { + var hi32 = Math.floor(src / V2E32), + lo32 = src - (hi32 * V2E32); + //assert.equal(roundedInt64(hi32, lo32), src); + //assert(0 <= lo32); + return { hi32: hi32, + lo32: lo32 }; +} +/* Reconstruct Number from quotient and non-negative remainder */ +function roundedInt64 (hi32, lo32) { + return hi32 * V2E32 + lo32; +} + +/** Represent an unsigned 64-bit integer in little-endian format when + * encoded and as a near integral JavaScript Number when decoded. + * + * *Factory*: {@link module:Layout.nu64|nu64} + * + * **NOTE** Values with magnitude greater than 2^52 may not decode to + * the exact value of the encoded representation. + * + * @constructor + * @augments {Layout} */ +function NearUInt64 (property) { + Layout.call(this, 8, property); + Object.freeze(this); +} +NearUInt64.prototype = Object.create(Layout.prototype); +NearUInt64.prototype.constructor = NearUInt64; +/** Implement {@link Layout#decode|decode} for {@link NearUInt64|NearUInt64}. */ +NearUInt64.prototype.decode = function (b, offset) { + if (undefined === offset) { + offset = 0; + } + var lo32 = b.readUInt32LE(offset), + hi32 = b.readUInt32LE(offset+4); + return roundedInt64(hi32, lo32); +}; +/** Implement {@link Layout#encode|encode} for {@link NearUInt64|NearUInt64}. */ +NearUInt64.prototype.encode = function (src, b, offset) { + if (undefined === offset) { + offset = 0; + } + var split = divmodInt64(src); + b.writeUInt32LE(split.lo32, offset); + b.writeUInt32LE(split.hi32, offset+4); +}; + +/** Represent an unsigned 64-bit integer in big-endian format when + * encoded and as a near integral JavaScript Number when decoded. + * + * *Factory*: {@link module:Layout.nu64be|nu64be} + * + * **NOTE** Values with magnitude greater than 2^52 may not decode to + * the exact value of the encoded representation. + * + * @constructor + * @augments {Layout} */ +function NearUInt64BE (property) { + Layout.call(this, 8, property); + Object.freeze(this); +} +NearUInt64BE.prototype = Object.create(Layout.prototype); +NearUInt64BE.prototype.constructor = NearUInt64BE; +/** Implement {@link Layout#decode|decode} for {@link NearUInt64BE|NearUInt64BE}. */ +NearUInt64BE.prototype.decode = function (b, offset) { + if (undefined === offset) { + offset = 0; + } + var hi32 = b.readUInt32BE(offset), + lo32 = b.readUInt32BE(offset+4); + return roundedInt64(hi32, lo32); +}; +/** Implement {@link Layout#encode|encode} for {@link NearUInt64BE|NearUInt64BE}. */ +NearUInt64BE.prototype.encode = function (src, b, offset) { + if (undefined === offset) { + offset = 0; + } + var split = divmodInt64(src); + b.writeUInt32BE(split.hi32, offset); + b.writeUInt32BE(split.lo32, offset+4); +}; + +/** Represent a signed 64-bit integer in little-endian format when + * encoded and as a near integral JavaScript Number when decoded. + * + * *Factory*: {@link module:Layout.ns64|ns64} + * + * **NOTE** Values with magnitude greater than 2^52 may not decode to + * the exact value of the encoded representation. + * + * @constructor + * @augments {Layout} */ +function NearInt64 (property) { + Layout.call(this, 8, property); + Object.freeze(this); +} +NearInt64.prototype = Object.create(Layout.prototype); +NearInt64.prototype.constructor = NearInt64; +/** Implement {@link Layout#decode|decode} for {@link NearInt64|NearInt64}. */ +NearInt64.prototype.decode = function (b, offset) { + if (undefined === offset) { + offset = 0; + } + var lo32 = b.readUInt32LE(offset), + hi32 = b.readInt32LE(offset+4); + return roundedInt64(hi32, lo32); +}; +/** Implement {@link Layout#encode|encode} for {@link NearInt64|NearInt64}. */ +NearInt64.prototype.encode = function (src, b, offset) { + if (undefined === offset) { + offset = 0; + } + var split = divmodInt64(src); + b.writeUInt32LE(split.lo32, offset); + b.writeInt32LE(split.hi32, offset+4); +}; + +/** Represent a signed 64-bit integer in big-endian format when + * encoded and as a near integral JavaScript Number when decoded. + * + * *Factory*: {@link module:Layout.ns64be|ns64be} + * + * **NOTE** Values with magnitude greater than 2^52 may not decode to + * the exact value of the encoded representation. + * + * @constructor + * @augments {Layout} */ +function NearInt64BE (property) { + Layout.call(this, 8, property); + Object.freeze(this); +} +NearInt64BE.prototype = Object.create(Layout.prototype); +NearInt64BE.prototype.constructor = NearInt64BE; +/** Implement {@link Layout#decode|decode} for {@link NearInt64BE|NearInt64BE}. */ +NearInt64BE.prototype.decode = function (b, offset) { + if (undefined === offset) { + offset = 0; + } + var hi32 = b.readInt32BE(offset), + lo32 = b.readUInt32BE(offset+4); + return roundedInt64(hi32, lo32); +}; +/** Implement {@link Layout#encode|encode} for {@link NearInt64BE|NearInt64BE}. */ +NearInt64BE.prototype.encode = function (src, b, offset) { + if (undefined === offset) { + offset = 0; + } + var split = divmodInt64(src); + b.writeInt32BE(split.hi32, offset); + b.writeUInt32BE(split.lo32, offset+4); +}; + /** Represent a 32-bit floating point number in little-endian format. * * *Factory*: {@link module:Layout.f32|f32} @@ -1892,6 +2058,10 @@ exports.u40 = function (property) { return new UInt(5, property); }; * spanning six bytes. */ exports.u48 = function (property) { return new UInt(6, property); }; +/** Factory for {@link NearUInt64|little-endian unsigned int + * layouts} interpreted as Numbers. */ +exports.nu64 = function (property) { return new NearUInt64(property); }; + /** Factory for {@link UInt|big-endian unsigned int layouts} * spanning two bytes. */ exports.u16be = function (property) { return new UIntBE(2, property); }; @@ -1912,6 +2082,10 @@ exports.u40be = function (property) { return new UIntBE(5, property); }; * spanning six bytes. */ exports.u48be = function (property) { return new UIntBE(6, property); }; +/** Factory for {@link NearUInt64BE|big-endian unsigned int + * layouts} interpreted as Numbers. */ +exports.nu64be = function (property) { return new NearUInt64BE(property); }; + /** Factory for {@link Int|signed int layouts} spanning one * byte. */ exports.s8 = function (property) { return new Int(1, property); }; @@ -1936,6 +2110,10 @@ exports.s40 = function (property) { return new Int(5, property); }; * spanning six bytes. */ exports.s48 = function (property) { return new Int(6, property); }; +/** Factory for {@link NearInt64|little-endian signed int layouts} + * interpreted as Numbers. */ +exports.ns64 = function (property) { return new NearInt64(property); }; + /** Factory for {@link Int|big-endian signed int layouts} * spanning two bytes. */ exports.s16be = function (property) { return new IntBE(2, property); }; @@ -1956,6 +2134,10 @@ exports.s40be = function (property) { return new IntBE(5, property); }; * spanning six bytes. */ exports.s48be = function (property) { return new IntBE(6, property); }; +/** Factory for {@link NearInt64BE|big-endian signed int layouts} + * interpreted as Numbers. */ +exports.ns64be = function (property) { return new NearInt64BE(property); }; + /** Factory for {@link Float|little-endian 32-bit floating point} values. */ exports.f32 = function (property) { return new Float(property); }; diff --git a/test/LayoutTest.js b/test/LayoutTest.js index 143a353..491d5e3 100644 --- a/test/LayoutTest.js +++ b/test/LayoutTest.js @@ -2,7 +2,19 @@ var assert = require("assert"), _ = require("lodash"), lo = require("../lib/Layout"); +/* Some versions of Node have an undocumented in-place reverse. + * That's not what we want. */ +function reversedBuffer (b) +{ + var ba = Array.prototype.slice.call(b); + return new Buffer(ba.reverse()); +} + suite("Layout", function () { + test("#reversedBuffer", function () { + var b = Buffer('0102030405', 'hex'); + assert.equal(Buffer('0504030201', 'hex').compare(reversedBuffer(b)), 0); + }); suite("Buffer", function () { test("issue 3992", function () { var buf = new Buffer(4); @@ -223,6 +235,128 @@ suite("Layout", function () { assert.throws(function () { new lo.IntBE(8, 'u64'); }, TypeError); }); }); + test("RoundedUInt64", function () { + var be = lo.nu64be('be'), + le = lo.nu64('le'); + assert.equal(be.span, 8); + assert.equal(le.span, 8); + assert.equal(be.property, 'be'); + assert.equal(le.property, 'le'); + + var b = Buffer('0000003b2a2a873b', 'hex'), + rb = reversedBuffer(b), + v = 254110500667, + ev = v, + eb = new Buffer(be.span); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(b.compare(eb), 0); + le.encode(v, eb); + assert.equal(rb.compare(eb), 0); + + b = Buffer('001d9515553fdcbb', 'hex'); + rb = reversedBuffer(b); + v = 8326693181709499; + ev = v; + assert.equal(ev, v); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(b.compare(eb), 0); + assert.equal(be.decode(eb), ev); + le.encode(v, eb); + assert.equal(rb.compare(eb), 0); + assert.equal(le.decode(eb), ev); + + /* The logic changes for the remaining cases since the exact + * value cannot be represented in a Number: the encoded buffer + * will not bitwise-match the original buffer. */ + b = Buffer('003b2a2aaa7fdcbb', 'hex'); + rb = reversedBuffer(b); + v = 16653386363428027; + ev = v + 1; + assert.equal(ev, v); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(be.decode(eb), ev); + le.encode(v, eb); + assert.equal(le.decode(eb), ev); + + b = Buffer('eca8aaa9ffffdcbb', 'hex'); + rb = reversedBuffer(b); + v = 17053067636159536315; + ev = v + 837; + assert.equal(ev, v); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(be.decode(eb), ev); + le.encode(v, eb); + assert.equal(le.decode(eb), ev); + }); + test("RoundedInt64", function () { + var be = lo.ns64be('be'), + le = lo.ns64('le'); + assert.equal(be.span, 8); + assert.equal(le.span, 8); + assert.equal(be.property, 'be'); + assert.equal(le.property, 'le'); + + var b = Buffer('ffffffff89abcdf0', 'hex'), + rb = reversedBuffer(b), + v = -1985229328, + ev = v, + eb = new Buffer(be.span); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(b.compare(eb), 0); + le.encode(v, eb); + assert.equal(rb.compare(eb), 0); + + b = Buffer('ffffc4d5d555a345', 'hex'); + rb = reversedBuffer(b); + v = -65052290473147; + ev = v; + assert.equal(ev, v); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(b.compare(eb), 0); + assert.equal(be.decode(eb), ev); + le.encode(v, eb); + assert.equal(rb.compare(eb), 0); + assert.equal(le.decode(eb), ev); + + /* The logic changes for the remaining cases since the exact + * value cannot be represented in a Number: the encoded buffer + * will not bitwise-match the original buffer. */ + b = Buffer('ff13575556002345', 'hex'); + rb = reversedBuffer(b); + v = -66613545453739195; + ev = v + 3; + assert.equal(ev, v); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(be.decode(eb), ev); + le.encode(v, eb); + assert.equal(le.decode(eb), ev); + + b = Buffer('e26aeaaac0002345', 'hex'); + rb = reversedBuffer(b); + v = -2131633454519934139; + ev = v - 69; + assert.equal(ev, v); + assert.equal(be.decode(b), ev); + assert.equal(le.decode(rb), ev); + be.encode(v, eb); + assert.equal(be.decode(eb), ev); + le.encode(v, eb); + assert.equal(le.decode(eb), ev); + }); test("Float", function () { var be = lo.f32be('eff'), le = lo.f32('ffe'), diff --git a/test/examples.js b/test/examples.js index b6d1122..c14a059 100644 --- a/test/examples.js +++ b/test/examples.js @@ -87,6 +87,17 @@ suite("Examples", function () { assert.equal(Buffer('8b010040', 'hex').compare(b), 0); assert.deepEqual(ds.decode(b), {b00l03:3, b03l01:1, b04l18:24, b1Cl04:4}); }); + test("64-bit values", function () { + /* + uint64_t v = 0x0102030405060708ULL; + */ + var ds = lo.nu64be(), + b = Buffer('0102030405060708', 'hex'), + v = 72623859790382856, + nv = v - 6; + assert.equal(v, nv); + assert.equal(ds.decode(b), nv); + }); test("C string", function () { /* const char str[] = "hi!"; diff --git a/test/n64.c b/test/n64.c new file mode 100644 index 0000000..52d8f8c --- /dev/null +++ b/test/n64.c @@ -0,0 +1,71 @@ +/* Show 64-bit integer values with round-trip through + * double-precision. + * + * Compile with: gcc -std=c11 */ +#include +#include +#include +#include + +static const uint32_t usval = 0x76543210; +static const uint32_t offset = 0x2345; + +static void +display_u64 (uint64_t val) +{ + double dval = val; + uint64_t rval = dval; + printf("%016" PRIx64 " : %" PRIu64 "\n" + "\t%a : %.*g : err %" PRId64 "\n" + "\t%016" PRIx64 " : %" PRIu64 "\n", + val, val, + dval, DBL_DECIMAL_DIG, dval, (int64_t)(rval - val), + rval, rval); +} + +static void +display_s64 (int64_t val) +{ + double dval = val; + int64_t rval = dval; + printf("%016" PRIx64 " : %" PRId64 "\n" + "\t%a : %.*g : err %" PRId64 "\n" + "\t%016" PRIx64 " : %" PRId64 "\n", + val, val, + dval, DBL_DECIMAL_DIG, dval, (int64_t)(rval - val), + rval, rval); +} + + +static void +gen_s (void) +{ + printf("Shifted:\n"); + uint64_t val = usval; + unsigned int sc = 0; + while (33 >= sc) { + display_u64(val); + val = (2 * val) + offset; + ++sc; + } +} + +static void +gen_sn (void) +{ + printf("Shifted Negated:\n"); + uint64_t val = usval; + unsigned int sc = 0; + while (33 >= sc) { + display_s64(-val); + val = (2 * val) + offset; + ++sc; + } +} + +int main (void) +{ + gen_s(); + gen_sn(); + return EXIT_SUCCESS; +}